Send-to-device support (#1072)
* Groundwork for send-to-device messaging
* Update sample config
* Add unstable routing for now
* Send to device consumer in sync API
* Start the send-to-device consumer
* fix indentation in dendrite-config.yaml
* Create send-to-device database tables, other tweaks
* Add some logic for send-to-device messages, add them into sync stream
* Handle incoming send-to-device messages, count them with EDU stream pos
* Undo changes to test
* pq.Array
* Fix sync
* Logging
* Fix a couple of transaction things, fix client API
* Add send-to-device test, hopefully fix bugs
* Comments
* Refactor a bit
* Fix schema
* Fix queries
* Debug logging
* Fix storing and retrieving of send-to-device messages
* Try to avoid database locks
* Update sync position
* Use latest sync position
* Jiggle about sync a bit
* Fix tests
* Break out the retrieval from the update/delete behaviour
* Comments
* nolint on getResponseWithPDUsForCompleteSync
* Try to line up sync tokens again
* Implement wildcard
* Add all send-to-device tests to whitelist, what could possibly go wrong?
* Only care about wildcard when targeted locally
* Deduplicate transactions
* Handle tokens properly, return immediately if waiting send-to-device messages
* Fix sync
* Update sytest-whitelist
* Fix copyright notice (need to do more of this)
* Comments, copyrights
* Return errors from Do, fix dendritejs
* Review comments
* Comments
* Constructor for TransactionWriter
* defletions
* Update gomatrixserverlib, sytest-blacklist
2020-06-01 16:50:19 +00:00
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2020-05-13 16:28:42 +00:00
package tables
import (
"context"
"database/sql"
2020-11-09 18:46:11 +00:00
eduAPI "github.com/matrix-org/dendrite/eduserver/api"
2020-05-14 08:53:55 +00:00
"github.com/matrix-org/dendrite/roomserver/api"
2020-05-13 16:28:42 +00:00
"github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/gomatrixserverlib"
)
2020-05-14 08:53:55 +00:00
type AccountData interface {
InsertAccountData ( ctx context . Context , txn * sql . Tx , userID , roomID , dataType string ) ( pos types . StreamPosition , err error )
2020-05-15 08:41:12 +00:00
// SelectAccountDataInRange returns a map of room ID to a list of `dataType`.
SelectAccountDataInRange ( ctx context . Context , userID string , r types . Range , accountDataEventFilter * gomatrixserverlib . EventFilter ) ( data map [ string ] [ ] string , err error )
2020-05-14 08:53:55 +00:00
SelectMaxAccountDataID ( ctx context . Context , txn * sql . Tx ) ( id int64 , err error )
}
2020-05-13 16:28:42 +00:00
type Invites interface {
InsertInviteEvent ( ctx context . Context , txn * sql . Tx , inviteEvent gomatrixserverlib . HeaderedEvent ) ( streamPos types . StreamPosition , err error )
2020-09-01 10:28:35 +00:00
DeleteInviteEvent ( ctx context . Context , txn * sql . Tx , inviteEventID string ) ( types . StreamPosition , error )
2020-08-25 17:59:00 +00:00
// SelectInviteEventsInRange returns a map of room ID to invite events. If multiple invite/retired invites exist in the given range, return the latest value
// for the room.
2020-06-26 10:07:52 +00:00
SelectInviteEventsInRange ( ctx context . Context , txn * sql . Tx , targetUserID string , r types . Range ) ( invites map [ string ] gomatrixserverlib . HeaderedEvent , retired map [ string ] gomatrixserverlib . HeaderedEvent , err error )
2020-05-13 16:28:42 +00:00
SelectMaxInviteID ( ctx context . Context , txn * sql . Tx ) ( id int64 , err error )
}
2020-05-14 08:53:55 +00:00
2020-09-10 13:39:18 +00:00
type Peeks interface {
InsertPeek ( ctx context . Context , txn * sql . Tx , roomID , userID , deviceID string ) ( streamPos types . StreamPosition , err error )
DeletePeek ( ctx context . Context , txn * sql . Tx , roomID , userID , deviceID string ) ( streamPos types . StreamPosition , err error )
DeletePeeks ( ctx context . Context , txn * sql . Tx , roomID , userID string ) ( streamPos types . StreamPosition , err error )
SelectPeeksInRange ( ctxt context . Context , txn * sql . Tx , userID , deviceID string , r types . Range ) ( peeks [ ] types . Peek , err error )
SelectPeekingDevices ( ctxt context . Context ) ( peekingDevices map [ string ] [ ] types . PeekingDevice , err error )
SelectMaxPeekID ( ctx context . Context , txn * sql . Tx ) ( id int64 , err error )
}
2020-05-14 08:53:55 +00:00
type Events interface {
2020-05-15 08:41:12 +00:00
SelectStateInRange ( ctx context . Context , txn * sql . Tx , r types . Range , stateFilter * gomatrixserverlib . StateFilter ) ( map [ string ] map [ string ] bool , map [ string ] types . StreamEvent , error )
2020-05-14 08:53:55 +00:00
SelectMaxEventID ( ctx context . Context , txn * sql . Tx ) ( id int64 , err error )
InsertEvent ( ctx context . Context , txn * sql . Tx , event * gomatrixserverlib . HeaderedEvent , addState , removeState [ ] string , transactionID * api . TransactionID , excludeFromSync bool ) ( streamPos types . StreamPosition , err error )
2020-05-15 08:41:12 +00:00
// SelectRecentEvents returns events between the two stream positions: exclusive of low and inclusive of high.
2020-05-14 16:30:16 +00:00
// If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude from sync.
2020-06-26 14:34:41 +00:00
// Returns up to `limit` events. Returns `limited=true` if there are more events in this range but we hit the `limit`.
SelectRecentEvents ( ctx context . Context , txn * sql . Tx , roomID string , r types . Range , limit int , chronologicalOrder bool , onlySyncEvents bool ) ( [ ] types . StreamEvent , bool , error )
2020-05-15 08:41:12 +00:00
// SelectEarlyEvents returns the earliest events in the given room.
SelectEarlyEvents ( ctx context . Context , txn * sql . Tx , roomID string , r types . Range , limit int ) ( [ ] types . StreamEvent , error )
2020-05-14 08:53:55 +00:00
SelectEvents ( ctx context . Context , txn * sql . Tx , eventIDs [ ] string ) ( [ ] types . StreamEvent , error )
2020-07-08 16:45:39 +00:00
UpdateEventJSON ( ctx context . Context , event * gomatrixserverlib . HeaderedEvent ) error
2020-09-15 10:17:46 +00:00
// DeleteEventsForRoom removes all event information for a room. This should only be done when removing the room entirely.
DeleteEventsForRoom ( ctx context . Context , txn * sql . Tx , roomID string ) ( err error )
2020-05-14 08:53:55 +00:00
}
2020-05-14 15:11:37 +00:00
2020-05-14 16:30:16 +00:00
// Topology keeps track of the depths and stream positions for all events.
// These positions are used as types.TopologyToken when backfilling events locally.
2020-05-14 15:11:37 +00:00
type Topology interface {
2020-05-14 16:30:16 +00:00
// InsertEventInTopology inserts the given event in the room's topology, based on the event's depth.
// `pos` is the stream position of this event in the events table, and is used to order events which have the same depth.
2020-05-14 15:11:37 +00:00
InsertEventInTopology ( ctx context . Context , txn * sql . Tx , event * gomatrixserverlib . HeaderedEvent , pos types . StreamPosition ) ( err error )
2020-05-14 16:30:16 +00:00
// SelectEventIDsInRange selects the IDs of events whose depths are within a given range in a given room's topological order.
// Events with `minDepth` are *exclusive*, as is the event which has exactly `minDepth`,`maxStreamPos`.
// `maxStreamPos` is only used when events have the same depth as `maxDepth`, which results in events less than `maxStreamPos` being returned.
2020-05-14 15:11:37 +00:00
// Returns an empty slice if no events match the given range.
2020-05-14 16:30:16 +00:00
SelectEventIDsInRange ( ctx context . Context , txn * sql . Tx , roomID string , minDepth , maxDepth , maxStreamPos types . StreamPosition , limit int , chronologicalOrder bool ) ( eventIDs [ ] string , err error )
// SelectPositionInTopology returns the depth and stream position of a given event in the topology of the room it belongs to.
SelectPositionInTopology ( ctx context . Context , txn * sql . Tx , eventID string ) ( depth , spos types . StreamPosition , err error )
// SelectMaxPositionInTopology returns the event which has the highest depth, and if there are multiple, the event with the highest stream position.
SelectMaxPositionInTopology ( ctx context . Context , txn * sql . Tx , roomID string ) ( depth types . StreamPosition , spos types . StreamPosition , err error )
2020-09-15 10:17:46 +00:00
// DeleteTopologyForRoom removes all topological information for a room. This should only be done when removing the room entirely.
DeleteTopologyForRoom ( ctx context . Context , txn * sql . Tx , roomID string ) ( err error )
2020-05-14 15:11:37 +00:00
}
type CurrentRoomState interface {
SelectStateEvent ( ctx context . Context , roomID , evType , stateKey string ) ( * gomatrixserverlib . HeaderedEvent , error )
SelectEventsWithEventIDs ( ctx context . Context , txn * sql . Tx , eventIDs [ ] string ) ( [ ] types . StreamEvent , error )
UpsertRoomState ( ctx context . Context , txn * sql . Tx , event gomatrixserverlib . HeaderedEvent , membership * string , addedAt types . StreamPosition ) error
DeleteRoomStateByEventID ( ctx context . Context , txn * sql . Tx , eventID string ) error
2020-09-15 10:17:46 +00:00
DeleteRoomStateForRoom ( ctx context . Context , txn * sql . Tx , roomID string ) error
2020-05-14 15:11:37 +00:00
// SelectCurrentState returns all the current state events for the given room.
SelectCurrentState ( ctx context . Context , txn * sql . Tx , roomID string , stateFilter * gomatrixserverlib . StateFilter ) ( [ ] gomatrixserverlib . HeaderedEvent , error )
// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
SelectRoomIDsWithMembership ( ctx context . Context , txn * sql . Tx , userID string , membership string ) ( [ ] string , error )
// SelectJoinedUsers returns a map of room ID to a list of joined user IDs.
SelectJoinedUsers ( ctx context . Context ) ( map [ string ] [ ] string , error )
}
// BackwardsExtremities keeps track of backwards extremities for a room.
// Backwards extremities are the earliest (DAG-wise) known events which we have
// the entire event JSON. These event IDs are used in federation requests to fetch
// even earlier events.
//
// We persist the previous event IDs as well, one per row, so when we do fetch even
// earlier events we can simply delete rows which referenced it. Consider the graph:
// A
// | Event C has 1 prev_event ID: A.
// B C
// |___| Event D has 2 prev_event IDs: B and C.
// |
// D
// The earliest known event we have is D, so this table has 2 rows.
// A backfill request gives us C but not B. We delete rows where prev_event=C. This
// still means that D is a backwards extremity as we do not have event B. However, event
// C is *also* a backwards extremity at this point as we do not have event A. Later,
// when we fetch event B, we delete rows where prev_event=B. This then removes D as
// a backwards extremity because there are no more rows with event_id=B.
type BackwardsExtremities interface {
// InsertsBackwardExtremity inserts a new backwards extremity.
InsertsBackwardExtremity ( ctx context . Context , txn * sql . Tx , roomID , eventID string , prevEventID string ) ( err error )
2020-05-20 15:04:31 +00:00
// SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room, as a map of event_id to list of prev_event_ids.
SelectBackwardExtremitiesForRoom ( ctx context . Context , roomID string ) ( bwExtrems map [ string ] [ ] string , err error )
2020-05-14 15:11:37 +00:00
// DeleteBackwardExtremity removes a backwards extremity for a room, if one existed.
DeleteBackwardExtremity ( ctx context . Context , txn * sql . Tx , roomID , knownEventID string ) ( err error )
2020-09-15 10:17:46 +00:00
// DeleteBackwardExtremitiesFoorRoomID removes all backward extremities for a room. This should only be done when removing the room entirely.
DeleteBackwardExtremitiesForRoom ( ctx context . Context , txn * sql . Tx , roomID string ) ( err error )
2020-05-14 15:11:37 +00:00
}
Send-to-device support (#1072)
* Groundwork for send-to-device messaging
* Update sample config
* Add unstable routing for now
* Send to device consumer in sync API
* Start the send-to-device consumer
* fix indentation in dendrite-config.yaml
* Create send-to-device database tables, other tweaks
* Add some logic for send-to-device messages, add them into sync stream
* Handle incoming send-to-device messages, count them with EDU stream pos
* Undo changes to test
* pq.Array
* Fix sync
* Logging
* Fix a couple of transaction things, fix client API
* Add send-to-device test, hopefully fix bugs
* Comments
* Refactor a bit
* Fix schema
* Fix queries
* Debug logging
* Fix storing and retrieving of send-to-device messages
* Try to avoid database locks
* Update sync position
* Use latest sync position
* Jiggle about sync a bit
* Fix tests
* Break out the retrieval from the update/delete behaviour
* Comments
* nolint on getResponseWithPDUsForCompleteSync
* Try to line up sync tokens again
* Implement wildcard
* Add all send-to-device tests to whitelist, what could possibly go wrong?
* Only care about wildcard when targeted locally
* Deduplicate transactions
* Handle tokens properly, return immediately if waiting send-to-device messages
* Fix sync
* Update sytest-whitelist
* Fix copyright notice (need to do more of this)
* Comments, copyrights
* Return errors from Do, fix dendritejs
* Review comments
* Comments
* Constructor for TransactionWriter
* defletions
* Update gomatrixserverlib, sytest-blacklist
2020-06-01 16:50:19 +00:00
// SendToDevice tracks send-to-device messages which are sent to individual
// clients. Each message gets inserted into this table at the point that we
// receive it from the EDU server.
//
// We're supposed to try and do our best to deliver send-to-device messages
// once, but the only way that we can really guarantee that they have been
// delivered is if the client successfully requests the next sync as given
// in the next_batch. Each time the device syncs, we will request all of the
// updates that either haven't been sent yet, along with all updates that we
// *have* sent but we haven't confirmed to have been received yet. If it's the
// first time we're sending a given update then we update the table to say
// what the "since" parameter was when we tried to send it.
//
// When the client syncs again, if their "since" parameter is *later* than
// the recorded one, we drop the entry from the DB as it's "sent". If the
// sync parameter isn't later then we will keep including the updates in the
// sync response, as the client is seemingly trying to repeat the same /sync.
type SendToDevice interface {
InsertSendToDeviceMessage ( ctx context . Context , txn * sql . Tx , userID , deviceID , content string ) ( err error )
SelectSendToDeviceMessages ( ctx context . Context , txn * sql . Tx , userID , deviceID string ) ( events [ ] types . SendToDeviceEvent , err error )
UpdateSentSendToDeviceMessages ( ctx context . Context , txn * sql . Tx , token string , nids [ ] types . SendToDeviceNID ) ( err error )
DeleteSendToDeviceMessages ( ctx context . Context , txn * sql . Tx , nids [ ] types . SendToDeviceNID ) ( err error )
CountSendToDeviceMessages ( ctx context . Context , txn * sql . Tx , userID , deviceID string ) ( count int , err error )
}
2020-06-26 14:34:41 +00:00
type Filter interface {
SelectFilter ( ctx context . Context , localpart string , filterID string ) ( * gomatrixserverlib . Filter , error )
InsertFilter ( ctx context . Context , filter * gomatrixserverlib . Filter , localpart string ) ( filterID string , err error )
}
2020-11-09 18:46:11 +00:00
type Receipts interface {
UpsertReceipt ( ctx context . Context , txn * sql . Tx , roomId , receiptType , userId , eventId string , timestamp gomatrixserverlib . Timestamp ) ( pos types . StreamPosition , err error )
SelectRoomReceiptsAfter ( ctx context . Context , roomIDs [ ] string , streamPos types . StreamPosition ) ( [ ] eduAPI . OutputReceiptEvent , error )
}