From 572f6c399d9fe4b4d513ad65a276b32847ebd6ba Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jun 2017 13:20:04 +0100 Subject: [PATCH] Include the state before the event in roomserver output (#143) * Include the state before the event in roomserver output * Fix roomserver integration test * Remove VisibilityEventIDs from the JSON * More comments * Remove spurious TODO --- .../cmd/roomserver-integration-tests/main.go | 3 +- .../dendrite/roomserver/api/output.go | 71 +++++++++++++------ .../roomserver/input/latest_events.go | 35 ++++++++- 3 files changed, 83 insertions(+), 26 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go b/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go index 940293f8..b303a06b 100644 --- a/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go @@ -340,7 +340,8 @@ func main() { "state_key":"@richvdh:matrix.org", "type":"m.room.member" }, - "VisibilityEventIDs":null, + "StateBeforeRemovesEventIDs":["$1463671339126270PnVwC:matrix.org"], + "StateBeforeAddsEventIDs":null, "LatestEventIDs":["$1463671339126270PnVwC:matrix.org"], "AddsStateEventIDs":["$1463671337126266wrSBX:matrix.org", "$1463671339126270PnVwC:matrix.org"], "RemovesStateEventIDs":null, diff --git a/src/github.com/matrix-org/dendrite/roomserver/api/output.go b/src/github.com/matrix-org/dendrite/roomserver/api/output.go index 29e76ed8..0b2aee64 100644 --- a/src/github.com/matrix-org/dendrite/roomserver/api/output.go +++ b/src/github.com/matrix-org/dendrite/roomserver/api/output.go @@ -19,12 +19,17 @@ import ( ) // An OutputRoomEvent is written when the roomserver receives a new event. +// It contains the full matrix room event and enough information for a +// consumer to construct the current state of the room and the state before the +// event. +// +// When we talk about state in a matrix room we are talking about the state +// after a list of events. The current state is the state after the latest +// event IDs in the room. The state before an event is the state after its +// prev_events. type OutputRoomEvent struct { // The JSON bytes of the event. Event []byte - // The state event IDs needed to determine who can see this event. - // This can be used to tell which users to send the event to. - VisibilityEventIDs []string // The latest events in the room after this event. // This can be used to set the prev events for new events in the room. // This also can be used to get the full current state after this event. @@ -40,9 +45,27 @@ type OutputRoomEvent struct { // This is used by consumers to check if they can safely update their // current state using the delta supplied in AddsStateEventIDs and // RemovesStateEventIDs. + // // If the LastSentEventID doesn't match what they were expecting it to be // they can use the LatestEventIDs to request the full current state. LastSentEventID string + // The state event IDs that are part of the state at the event, but not + // part of the current state. Together with the StateBeforeRemovesEventIDs + // this can be used to construct the state before the event from the + // current state. The StateBeforeAddsEventIDs and StateBeforeRemovesEventIDs + // delta is applied after the AddsStateEventIDs and RemovesStateEventIDs. + // + // Consumers need to know the state at each event in order to determine + // which users and servers are allowed to see the event. This information + // is needed to apply the history visibility rules and to tell which + // servers we need to push events to over federation. + // + // The state is given as a delta against the current state because they are + // usually either the same state, or differ by just a couple of events. + StateBeforeAddsEventIDs []string + // The state event IDs that are part of the current state, but not part + // of the state at the event. + StateBeforeRemovesEventIDs []string } // UnmarshalJSON implements json.Unmarshaller @@ -52,12 +75,13 @@ func (ore *OutputRoomEvent) UnmarshalJSON(data []byte) error { // We use json.RawMessage so that the event JSON is sent as JSON rather than // being base64 encoded which is the default for []byte. var content struct { - Event *json.RawMessage - VisibilityEventIDs []string - LatestEventIDs []string - AddsStateEventIDs []string - RemovesStateEventIDs []string - LastSentEventID string + Event *json.RawMessage + LatestEventIDs []string + AddsStateEventIDs []string + RemovesStateEventIDs []string + LastSentEventID string + StateBeforeAddsEventIDs []string + StateBeforeRemovesEventIDs []string } if err := json.Unmarshal(data, &content); err != nil { return err @@ -65,11 +89,12 @@ func (ore *OutputRoomEvent) UnmarshalJSON(data []byte) error { if content.Event != nil { ore.Event = []byte(*content.Event) } - ore.VisibilityEventIDs = content.VisibilityEventIDs ore.LatestEventIDs = content.LatestEventIDs ore.AddsStateEventIDs = content.AddsStateEventIDs ore.RemovesStateEventIDs = content.RemovesStateEventIDs ore.LastSentEventID = content.LastSentEventID + ore.StateBeforeAddsEventIDs = content.StateBeforeAddsEventIDs + ore.StateBeforeRemovesEventIDs = content.StateBeforeRemovesEventIDs return nil } @@ -81,19 +106,21 @@ func (ore OutputRoomEvent) MarshalJSON() ([]byte, error) { // being base64 encoded which is the default for []byte. event := json.RawMessage(ore.Event) content := struct { - Event *json.RawMessage - VisibilityEventIDs []string - LatestEventIDs []string - AddsStateEventIDs []string - RemovesStateEventIDs []string - LastSentEventID string + Event *json.RawMessage + LatestEventIDs []string + AddsStateEventIDs []string + RemovesStateEventIDs []string + LastSentEventID string + StateBeforeAddsEventIDs []string + StateBeforeRemovesEventIDs []string }{ - Event: &event, - VisibilityEventIDs: ore.VisibilityEventIDs, - LatestEventIDs: ore.LatestEventIDs, - AddsStateEventIDs: ore.AddsStateEventIDs, - RemovesStateEventIDs: ore.RemovesStateEventIDs, - LastSentEventID: ore.LastSentEventID, + Event: &event, + LatestEventIDs: ore.LatestEventIDs, + AddsStateEventIDs: ore.AddsStateEventIDs, + RemovesStateEventIDs: ore.RemovesStateEventIDs, + LastSentEventID: ore.LastSentEventID, + StateBeforeAddsEventIDs: ore.StateBeforeAddsEventIDs, + StateBeforeRemovesEventIDs: ore.StateBeforeRemovesEventIDs, } return json.Marshal(&content) } diff --git a/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go b/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go index ad5c15a7..ce7ac0bf 100644 --- a/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go +++ b/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" ) // updateLatestEvents updates the list of latest events for this room in the database and writes the @@ -110,6 +111,13 @@ func doUpdateLatestEvents( return err } + stateBeforeEventRemoves, stateBeforeEventAdds, err := state.DifferenceBetweeenStateSnapshots( + db, newStateNID, stateAtEvent.BeforeStateSnapshotNID, + ) + if err != nil { + return err + } + // Send the event to the output logs. // We do this inside the database transaction to ensure that we only mark an event as sent if we sent it. // (n.b. this means that it's possible that the same event will be sent twice if the transaction fails but @@ -118,7 +126,10 @@ func doUpdateLatestEvents( // send the event asynchronously but we would need to ensure that 1) the events are written to the log in // the correct order, 2) that pending writes are resent across restarts. In order to avoid writing all the // necessary bookkeeping we'll keep the event sending synchronous for now. - if err = writeEvent(db, ow, lastEventIDSent, event, newLatest, removed, added); err != nil { + if err = writeEvent( + db, ow, lastEventIDSent, event, newLatest, removed, added, + stateBeforeEventRemoves, stateBeforeEventAdds, + ); err != nil { return err } @@ -170,6 +181,7 @@ func writeEvent( db RoomEventDatabase, ow OutputRoomEventWriter, lastEventIDSent string, event gomatrixserverlib.Event, latest []types.StateAtEventAndReference, removed, added []types.StateEntry, + stateBeforeEventRemoves, stateBeforeEventAdds []types.StateEntry, ) error { latestEventIDs := make([]string, len(latest)) @@ -190,6 +202,13 @@ func writeEvent( for _, entry := range removed { stateEventNIDs = append(stateEventNIDs, entry.EventNID) } + for _, entry := range stateBeforeEventRemoves { + stateEventNIDs = append(stateEventNIDs, entry.EventNID) + } + for _, entry := range stateBeforeEventAdds { + stateEventNIDs = append(stateEventNIDs, entry.EventNID) + } + stateEventNIDs = stateEventNIDs[:util.SortAndUnique(eventNIDSorter(stateEventNIDs))] eventIDMap, err := db.EventIDs(stateEventNIDs) if err != nil { return err @@ -200,7 +219,17 @@ func writeEvent( for _, entry := range removed { ore.RemovesStateEventIDs = append(ore.RemovesStateEventIDs, eventIDMap[entry.EventNID]) } - - // TODO: Fill out VisibilityStateIDs + for _, entry := range stateBeforeEventRemoves { + ore.StateBeforeRemovesEventIDs = append(ore.StateBeforeRemovesEventIDs, eventIDMap[entry.EventNID]) + } + for _, entry := range stateBeforeEventAdds { + ore.StateBeforeAddsEventIDs = append(ore.StateBeforeAddsEventIDs, eventIDMap[entry.EventNID]) + } return ow.WriteOutputRoomEvent(ore) } + +type eventNIDSorter []types.EventNID + +func (s eventNIDSorter) Len() int { return len(s) } +func (s eventNIDSorter) Less(i, j int) bool { return s[i] < s[j] } +func (s eventNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }