From a202d88fe527c83179d2ebf57ac0d146c3173f9d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 24 Apr 2020 10:38:58 +0100 Subject: [PATCH] Use a single storage.Database interface (#978) --- roomserver/input/authevents.go | 5 +- roomserver/input/events.go | 59 ++------------------- roomserver/input/input.go | 3 +- roomserver/input/latest_events.go | 5 +- roomserver/input/membership.go | 3 +- roomserver/query/query.go | 75 +++------------------------ roomserver/query/query_test.go | 4 +- roomserver/state/database/database.go | 67 ------------------------ roomserver/state/shared/shared.go | 1 - roomserver/state/state.go | 6 +-- roomserver/storage/interface.go | 41 ++++++++++++++- 11 files changed, 65 insertions(+), 204 deletions(-) delete mode 100644 roomserver/state/database/database.go delete mode 100644 roomserver/state/shared/shared.go diff --git a/roomserver/input/authevents.go b/roomserver/input/authevents.go index 456a01c7..2c2e14b3 100644 --- a/roomserver/input/authevents.go +++ b/roomserver/input/authevents.go @@ -18,6 +18,7 @@ import ( "context" "sort" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -26,7 +27,7 @@ import ( // Returns the numeric IDs for the auth events. func checkAuthEvents( ctx context.Context, - db RoomEventDatabase, + db storage.Database, event gomatrixserverlib.HeaderedEvent, authEventIDs []string, ) ([]types.EventNID, error) { @@ -127,7 +128,7 @@ func (ae *authEvents) lookupEvent(typeNID types.EventTypeNID, stateKey string) * // loadAuthEvents loads the events needed for authentication from the supplied room state. func loadAuthEvents( ctx context.Context, - db RoomEventDatabase, + db storage.Database, needed gomatrixserverlib.StateNeeded, state []types.StateEntry, ) (result authEvents, err error) { diff --git a/roomserver/input/events.go b/roomserver/input/events.go index 2bb0d0a0..393c1f41 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -23,63 +23,12 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" - "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) -// A RoomEventDatabase has the storage APIs needed to store a room event. -type RoomEventDatabase interface { - database.RoomStateDatabase - // Stores a matrix room event in the database - StoreEvent( - ctx context.Context, - event gomatrixserverlib.Event, - txnAndSessionID *api.TransactionID, - authEventNIDs []types.EventNID, - ) (types.RoomNID, types.StateAtEvent, error) - // Look up the state entries for a list of string event IDs - // Returns an error if the there is an error talking to the database - // Returns a types.MissingEventError if the event IDs aren't in the database. - StateEntriesForEventIDs( - ctx context.Context, eventIDs []string, - ) ([]types.StateEntry, error) - // Set the state at an event. - SetState( - ctx context.Context, - eventNID types.EventNID, - stateNID types.StateSnapshotNID, - ) error - // Look up the latest events in a room in preparation for an update. - // The RoomRecentEventsUpdater must have Commit or Rollback called on it if this doesn't return an error. - // Returns the latest events in the room and the last eventID sent to the log along with an updater. - // If this returns an error then no further action is required. - GetLatestEventsForUpdate( - ctx context.Context, roomNID types.RoomNID, - ) (updater types.RoomRecentEventsUpdater, err error) - // Look up the string event IDs for a list of numeric event IDs - EventIDs( - ctx context.Context, eventNIDs []types.EventNID, - ) (map[types.EventNID]string, error) - // Build a membership updater for the target user in a room. - MembershipUpdater( - ctx context.Context, roomID, targerUserID string, - roomVersion gomatrixserverlib.RoomVersion, - ) (types.MembershipUpdater, error) - // Look up event ID by transaction's info. - // This is used to determine if the room event is processed/processing already. - // Returns an empty string if no such event exists. - GetTransactionEventID( - ctx context.Context, transactionID string, - sessionID int64, userID string, - ) (string, error) - // Look up the room version for a given room. - GetRoomVersionForRoom( - ctx context.Context, roomID string, - ) (gomatrixserverlib.RoomVersion, error) -} - // OutputRoomEventWriter has the APIs needed to write an event to the output logs. type OutputRoomEventWriter interface { // Write a list of events for a room @@ -93,7 +42,7 @@ type OutputRoomEventWriter interface { // state deltas when sending to kafka streams func processRoomEvent( ctx context.Context, - db RoomEventDatabase, + db storage.Database, ow OutputRoomEventWriter, input api.InputRoomEvent, ) (eventID string, err error) { @@ -153,7 +102,7 @@ func processRoomEvent( func calculateAndSetState( ctx context.Context, - db RoomEventDatabase, + db storage.Database, input api.InputRoomEvent, roomNID types.RoomNID, stateAtEvent *types.StateAtEvent, @@ -184,7 +133,7 @@ func calculateAndSetState( func processInviteEvent( ctx context.Context, - db RoomEventDatabase, + db storage.Database, ow OutputRoomEventWriter, input api.InputInviteEvent, ) (err error) { diff --git a/roomserver/input/input.go b/roomserver/input/input.go index 3b519ecb..cb588380 100644 --- a/roomserver/input/input.go +++ b/roomserver/input/input.go @@ -24,12 +24,13 @@ import ( "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/util" ) // RoomserverInputAPI implements api.RoomserverInputAPI type RoomserverInputAPI struct { - DB RoomEventDatabase + DB storage.Database Producer sarama.SyncProducer // The kafkaesque topic to output new room events to. // This is the name used in kafka to identify the stream to write events to. diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index 525a6f51..cac3968d 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -47,7 +48,7 @@ import ( // Can only be called once at a time func updateLatestEvents( ctx context.Context, - db RoomEventDatabase, + db storage.Database, ow OutputRoomEventWriter, roomNID types.RoomNID, stateAtEvent types.StateAtEvent, @@ -86,7 +87,7 @@ func updateLatestEvents( // when there are so many variables to pass around. type latestEventsUpdater struct { ctx context.Context - db RoomEventDatabase + db storage.Database updater types.RoomRecentEventsUpdater ow OutputRoomEventWriter roomNID types.RoomNID diff --git a/roomserver/input/membership.go b/roomserver/input/membership.go index ee39ff5e..8629cb23 100644 --- a/roomserver/input/membership.go +++ b/roomserver/input/membership.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -29,7 +30,7 @@ import ( // consumers about the invites added or retired by the change in current state. func updateMemberships( ctx context.Context, - db RoomEventDatabase, + db storage.Database, updater types.RoomRecentEventsUpdater, removed, added []types.StateEntry, ) ([]api.OutputEvent, error) { diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 224d9fa2..e9286b4e 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -26,79 +26,16 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/state" - "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -// RoomserverQueryAPIEventDB has a convenience API to fetch events directly by -// EventIDs. -type RoomserverQueryAPIEventDB interface { - // Look up the Events for a list of event IDs. Does not error if event was - // not found. - // Returns an error if the retrieval went wrong. - EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) -} - -// RoomserverQueryAPIDatabase has the storage APIs needed to implement the query API. -type RoomserverQueryAPIDatabase interface { - database.RoomStateDatabase - RoomserverQueryAPIEventDB - // Look up the numeric ID for the room. - // Returns 0 if the room doesn't exists. - // Returns an error if there was a problem talking to the database. - RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) - // Look up event references for the latest events in the room and the current state snapshot. - // Returns the latest events, the current state and the maximum depth of the latest events plus 1. - // Returns an error if there was a problem talking to the database. - LatestEventIDs( - ctx context.Context, roomNID types.RoomNID, - ) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) - // Look up the numeric IDs for a list of events. - // Returns an error if there was a problem talking to the database. - EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) - // Lookup the event IDs for a batch of event numeric IDs. - // Returns an error if the retrieval went wrong. - EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) - // Lookup the membership of a given user in a given room. - // Returns the numeric ID of the latest membership event sent from this user - // in this room, along a boolean set to true if the user is still in this room, - // false if not. - // Returns an error if there was a problem talking to the database. - GetMembership( - ctx context.Context, roomNID types.RoomNID, requestSenderUserID string, - ) (membershipEventNID types.EventNID, stillInRoom bool, err error) - // Lookup the membership event numeric IDs for all user that are or have - // been members of a given room. Only lookup events of "join" membership if - // joinOnly is set to true. - // Returns an error if there was a problem talking to the database. - GetMembershipEventNIDsForRoom( - ctx context.Context, roomNID types.RoomNID, joinOnly bool, - ) ([]types.EventNID, error) - // Look up the active invites targeting a user in a room and return the - // numeric state key IDs for the user IDs who sent them. - // Returns an error if there was a problem talking to the database. - GetInvitesForUser( - ctx context.Context, - roomNID types.RoomNID, - targetUserNID types.EventStateKeyNID, - ) (senderUserNIDs []types.EventStateKeyNID, err error) - // Look up the string event state keys for a list of numeric event state keys - // Returns an error if there was a problem talking to the database. - EventStateKeys( - context.Context, []types.EventStateKeyNID, - ) (map[types.EventStateKeyNID]string, error) - // Look up the room version for a given room. - GetRoomVersionForRoom( - ctx context.Context, roomID string, - ) (gomatrixserverlib.RoomVersion, error) -} - // RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI type RoomserverQueryAPI struct { - DB RoomserverQueryAPIDatabase + DB storage.Database ImmutableCache caching.ImmutableCache } @@ -741,7 +678,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } authEventIDs = util.UniqueStrings(authEventIDs) // de-dupe - authEvents, err := getAuthChain(ctx, r.DB, authEventIDs) + authEvents, err := getAuthChain(ctx, r.DB.EventsFromIDs, authEventIDs) if err != nil { return err } @@ -788,12 +725,14 @@ func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs [ return r.loadStateEvents(ctx, stateEntries) } +type eventsFromIDs func(context.Context, []string) ([]types.Event, error) + // getAuthChain fetches the auth chain for the given auth events. An auth chain // is the list of all events that are referenced in the auth_events section, and // all their auth_events, recursively. The returned set of events contain the // given events. Will *not* error if we don't have all auth events. func getAuthChain( - ctx context.Context, dB RoomserverQueryAPIEventDB, authEventIDs []string, + ctx context.Context, fn eventsFromIDs, authEventIDs []string, ) ([]gomatrixserverlib.Event, error) { // List of event IDs to fetch. On each pass, these events will be requested // from the database and the `eventsToFetch` will be updated with any new @@ -804,7 +743,7 @@ func getAuthChain( for len(eventsToFetch) > 0 { // Try to retrieve the events from the database. - events, err := dB.EventsFromIDs(ctx, eventsToFetch) + events, err := fn(ctx, eventsToFetch) if err != nil { return nil, err } diff --git a/roomserver/query/query_test.go b/roomserver/query/query_test.go index 7e040c6f..8fb6a082 100644 --- a/roomserver/query/query_test.go +++ b/roomserver/query/query_test.go @@ -106,7 +106,7 @@ func TestGetAuthChainSingle(t *testing.T) { t.Fatalf("Failed to add events to db: %v", err) } - result, err := getAuthChain(context.TODO(), db, []string{"e"}) + result, err := getAuthChain(context.TODO(), db.EventsFromIDs, []string{"e"}) if err != nil { t.Fatalf("getAuthChain failed: %v", err) } @@ -139,7 +139,7 @@ func TestGetAuthChainMultiple(t *testing.T) { t.Fatalf("Failed to add events to db: %v", err) } - result, err := getAuthChain(context.TODO(), db, []string{"e", "f"}) + result, err := getAuthChain(context.TODO(), db.EventsFromIDs, []string{"e", "f"}) if err != nil { t.Fatalf("getAuthChain failed: %v", err) } diff --git a/roomserver/state/database/database.go b/roomserver/state/database/database.go deleted file mode 100644 index 80f1b14f..00000000 --- a/roomserver/state/database/database.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -import ( - "context" - - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" -) - -// A RoomStateDatabase has the storage APIs needed to load state from the database -type RoomStateDatabase interface { - // Store the room state at an event in the database - AddState( - ctx context.Context, - roomNID types.RoomNID, - stateBlockNIDs []types.StateBlockNID, - state []types.StateEntry, - ) (types.StateSnapshotNID, error) - // Look up the state of a room at each event for a list of string event IDs. - // Returns an error if there is an error talking to the database - // Returns a types.MissingEventError if the room state for the event IDs aren't in the database - StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) - // Look up the numeric IDs for a list of string event types. - // Returns a map from string event type to numeric ID for the event type. - EventTypeNIDs(ctx context.Context, eventTypes []string) (map[string]types.EventTypeNID, error) - // Look up the numeric IDs for a list of string event state keys. - // Returns a map from string state key to numeric ID for the state key. - EventStateKeyNIDs(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) - // Look up the numeric state data IDs for each numeric state snapshot ID - // The returned slice is sorted by numeric state snapshot ID. - StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) - // Look up the state data for each numeric state data ID - // The returned slice is sorted by numeric state data ID. - StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) - // Look up the state data for the state key tuples for each numeric state block ID - // This is used to fetch a subset of the room state at a snapshot. - // If a block doesn't contain any of the requested tuples then it can be discarded from the result. - // The returned slice is sorted by numeric state block ID. - StateEntriesForTuples( - ctx context.Context, - stateBlockNIDs []types.StateBlockNID, - stateKeyTuples []types.StateKeyTuple, - ) ([]types.StateEntryList, error) - // Look up the Events for a list of numeric event IDs. - // Returns a sorted list of events. - Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) - // Look up snapshot NID for an event ID string - SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) - // Look up a room version from the room NID. - GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) -} diff --git a/roomserver/state/shared/shared.go b/roomserver/state/shared/shared.go deleted file mode 100644 index a29b5e40..00000000 --- a/roomserver/state/shared/shared.go +++ /dev/null @@ -1 +0,0 @@ -package shared diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 3f68e074..389c9440 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -22,7 +22,7 @@ import ( "sort" "time" - "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -31,10 +31,10 @@ import ( ) type StateResolution struct { - db database.RoomStateDatabase + db storage.Database } -func NewStateResolution(db database.RoomStateDatabase) StateResolution { +func NewStateResolution(db storage.Database) StateResolution { return StateResolution{ db: db, } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 50369d80..0235e51e 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -18,13 +18,50 @@ import ( "context" "github.com/matrix-org/dendrite/roomserver/api" - statedb "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { - statedb.RoomStateDatabase + // Store the room state at an event in the database + AddState( + ctx context.Context, + roomNID types.RoomNID, + stateBlockNIDs []types.StateBlockNID, + state []types.StateEntry, + ) (types.StateSnapshotNID, error) + // Look up the state of a room at each event for a list of string event IDs. + // Returns an error if there is an error talking to the database + // Returns a types.MissingEventError if the room state for the event IDs aren't in the database + StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) + // Look up the numeric IDs for a list of string event types. + // Returns a map from string event type to numeric ID for the event type. + EventTypeNIDs(ctx context.Context, eventTypes []string) (map[string]types.EventTypeNID, error) + // Look up the numeric IDs for a list of string event state keys. + // Returns a map from string state key to numeric ID for the state key. + EventStateKeyNIDs(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) + // Look up the numeric state data IDs for each numeric state snapshot ID + // The returned slice is sorted by numeric state snapshot ID. + StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) + // Look up the state data for each numeric state data ID + // The returned slice is sorted by numeric state data ID. + StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) + // Look up the state data for the state key tuples for each numeric state block ID + // This is used to fetch a subset of the room state at a snapshot. + // If a block doesn't contain any of the requested tuples then it can be discarded from the result. + // The returned slice is sorted by numeric state block ID. + StateEntriesForTuples( + ctx context.Context, + stateBlockNIDs []types.StateBlockNID, + stateKeyTuples []types.StateKeyTuple, + ) ([]types.StateEntryList, error) + // Look up the Events for a list of numeric event IDs. + // Returns a sorted list of events. + Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) + // Look up snapshot NID for an event ID string + SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) + // Look up a room version from the room NID. + GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error)