2020-05-01 12:01:50 +00:00
|
|
|
package internal
|
2020-04-29 10:34:31 +00:00
|
|
|
|
|
|
|
import (
|
2020-08-20 16:03:07 +00:00
|
|
|
"context"
|
2020-09-22 10:05:45 +00:00
|
|
|
"sync"
|
2020-08-20 16:03:07 +00:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/matrix-org/dendrite/federationsender/api"
|
2020-06-01 17:34:08 +00:00
|
|
|
"github.com/matrix-org/dendrite/federationsender/queue"
|
2020-07-22 16:01:29 +00:00
|
|
|
"github.com/matrix-org/dendrite/federationsender/statistics"
|
2020-04-29 10:34:31 +00:00
|
|
|
"github.com/matrix-org/dendrite/federationsender/storage"
|
2020-05-21 13:40:13 +00:00
|
|
|
"github.com/matrix-org/dendrite/internal/config"
|
2020-08-20 16:03:07 +00:00
|
|
|
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
|
|
|
|
"github.com/matrix-org/gomatrix"
|
2020-04-29 14:29:39 +00:00
|
|
|
"github.com/matrix-org/gomatrixserverlib"
|
2020-04-29 10:34:31 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI
|
|
|
|
type FederationSenderInternalAPI struct {
|
2020-04-29 14:29:39 +00:00
|
|
|
db storage.Database
|
2020-08-10 13:18:04 +00:00
|
|
|
cfg *config.FederationSender
|
2020-07-22 16:01:29 +00:00
|
|
|
statistics *statistics.Statistics
|
2020-08-20 16:03:07 +00:00
|
|
|
rsAPI roomserverAPI.RoomserverInternalAPI
|
2020-04-29 14:29:39 +00:00
|
|
|
federation *gomatrixserverlib.FederationClient
|
|
|
|
keyRing *gomatrixserverlib.KeyRing
|
2020-06-01 17:34:08 +00:00
|
|
|
queues *queue.OutgoingQueues
|
2020-09-22 10:05:45 +00:00
|
|
|
joins sync.Map // joins currently in progress
|
2020-04-29 14:29:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func NewFederationSenderInternalAPI(
|
2020-08-10 13:18:04 +00:00
|
|
|
db storage.Database, cfg *config.FederationSender,
|
2020-08-20 16:03:07 +00:00
|
|
|
rsAPI roomserverAPI.RoomserverInternalAPI,
|
2020-04-29 14:29:39 +00:00
|
|
|
federation *gomatrixserverlib.FederationClient,
|
|
|
|
keyRing *gomatrixserverlib.KeyRing,
|
2020-07-22 16:01:29 +00:00
|
|
|
statistics *statistics.Statistics,
|
2020-06-01 17:34:08 +00:00
|
|
|
queues *queue.OutgoingQueues,
|
2020-04-29 14:29:39 +00:00
|
|
|
) *FederationSenderInternalAPI {
|
|
|
|
return &FederationSenderInternalAPI{
|
|
|
|
db: db,
|
|
|
|
cfg: cfg,
|
2020-06-10 15:54:43 +00:00
|
|
|
rsAPI: rsAPI,
|
2020-04-29 14:29:39 +00:00
|
|
|
federation: federation,
|
|
|
|
keyRing: keyRing,
|
Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)
* Improve federation sender performance and behaviour, add backoff
* Tweaks
* Tweaks
* Tweaks
* Take copies of events before passing to destination queues
* Don't accidentally drop queued messages
* Don't take copies again
* Tidy up a bit
* Break out statistics (tracked component-wide), report success and failures from Perform actions
* Fix comment, use atomic add
* Improve logic a bit, don't block on wakeup, move idle check
* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc
* Dedupe destinations, fix other bug hopefully
* Dispatch sends again
* Federation sender to ignore invites that are destined locally
* Loopback invite events
* Remodel a bit with channels
* Linter
* Only loopback invite event if we know the room
* We should tell other resident servers about the invite if we know about the room
* Correct invite signing
* Fix invite loopback
* Check HTTP response codes, push new invites to front of queue
* Review comments
2020-05-07 11:42:06 +00:00
|
|
|
statistics: statistics,
|
2020-06-01 17:34:08 +00:00
|
|
|
queues: queues,
|
2020-04-29 14:29:39 +00:00
|
|
|
}
|
2020-04-29 10:34:31 +00:00
|
|
|
}
|
2020-08-20 16:03:07 +00:00
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) isBlacklistedOrBackingOff(s gomatrixserverlib.ServerName) (*statistics.ServerStatistics, error) {
|
|
|
|
stats := a.statistics.ForServer(s)
|
|
|
|
until, blacklisted := stats.BackoffInfo()
|
|
|
|
if blacklisted {
|
|
|
|
return stats, &api.FederationClientError{
|
|
|
|
Blacklisted: true,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
now := time.Now()
|
|
|
|
if until != nil && now.Before(*until) {
|
|
|
|
return stats, &api.FederationClientError{
|
|
|
|
RetryAfter: time.Until(*until),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return stats, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func failBlacklistableError(err error, stats *statistics.ServerStatistics) (until time.Time, blacklisted bool) {
|
|
|
|
if err == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
mxerr, ok := err.(gomatrix.HTTPError)
|
|
|
|
if !ok {
|
|
|
|
return stats.Failure()
|
|
|
|
}
|
2020-09-08 12:41:08 +00:00
|
|
|
if mxerr.Code == 401 { // invalid signature in X-Matrix header
|
|
|
|
return stats.Failure()
|
|
|
|
}
|
|
|
|
if mxerr.Code >= 500 && mxerr.Code < 600 { // internal server errors
|
2020-08-20 16:03:07 +00:00
|
|
|
return stats.Failure()
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) doRequest(
|
|
|
|
s gomatrixserverlib.ServerName, request func() (interface{}, error),
|
|
|
|
) (interface{}, error) {
|
|
|
|
stats, err := a.isBlacklistedOrBackingOff(s)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
res, err := request()
|
|
|
|
if err != nil {
|
|
|
|
until, blacklisted := failBlacklistableError(err, stats)
|
|
|
|
now := time.Now()
|
|
|
|
var retryAfter time.Duration
|
|
|
|
if until.After(now) {
|
|
|
|
retryAfter = time.Until(until)
|
|
|
|
}
|
|
|
|
return res, &api.FederationClientError{
|
|
|
|
Err: err.Error(),
|
|
|
|
Blacklisted: blacklisted,
|
|
|
|
RetryAfter: retryAfter,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
stats.Success()
|
|
|
|
return res, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) GetUserDevices(
|
|
|
|
ctx context.Context, s gomatrixserverlib.ServerName, userID string,
|
|
|
|
) (gomatrixserverlib.RespUserDevices, error) {
|
2020-10-09 16:08:32 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
|
|
|
|
defer cancel()
|
2020-08-20 16:03:07 +00:00
|
|
|
ires, err := a.doRequest(s, func() (interface{}, error) {
|
|
|
|
return a.federation.GetUserDevices(ctx, s, userID)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return gomatrixserverlib.RespUserDevices{}, err
|
|
|
|
}
|
|
|
|
return ires.(gomatrixserverlib.RespUserDevices), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) ClaimKeys(
|
|
|
|
ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string,
|
|
|
|
) (gomatrixserverlib.RespClaimKeys, error) {
|
2020-10-09 16:08:32 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
|
|
|
|
defer cancel()
|
2020-08-20 16:03:07 +00:00
|
|
|
ires, err := a.doRequest(s, func() (interface{}, error) {
|
|
|
|
return a.federation.ClaimKeys(ctx, s, oneTimeKeys)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return gomatrixserverlib.RespClaimKeys{}, err
|
|
|
|
}
|
|
|
|
return ires.(gomatrixserverlib.RespClaimKeys), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) QueryKeys(
|
|
|
|
ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string,
|
|
|
|
) (gomatrixserverlib.RespQueryKeys, error) {
|
|
|
|
ires, err := a.doRequest(s, func() (interface{}, error) {
|
|
|
|
return a.federation.QueryKeys(ctx, s, keys)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return gomatrixserverlib.RespQueryKeys{}, err
|
|
|
|
}
|
|
|
|
return ires.(gomatrixserverlib.RespQueryKeys), nil
|
|
|
|
}
|
2020-09-02 14:26:30 +00:00
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) Backfill(
|
|
|
|
ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string,
|
|
|
|
) (res gomatrixserverlib.Transaction, err error) {
|
2020-10-09 16:08:32 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
|
|
|
|
defer cancel()
|
2020-09-02 14:26:30 +00:00
|
|
|
ires, err := a.doRequest(s, func() (interface{}, error) {
|
|
|
|
return a.federation.Backfill(ctx, s, roomID, limit, eventIDs)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return gomatrixserverlib.Transaction{}, err
|
|
|
|
}
|
|
|
|
return ires.(gomatrixserverlib.Transaction), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) LookupState(
|
|
|
|
ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion,
|
|
|
|
) (res gomatrixserverlib.RespState, err error) {
|
2020-10-09 16:08:32 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
|
|
|
|
defer cancel()
|
2020-09-02 14:26:30 +00:00
|
|
|
ires, err := a.doRequest(s, func() (interface{}, error) {
|
|
|
|
return a.federation.LookupState(ctx, s, roomID, eventID, roomVersion)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return gomatrixserverlib.RespState{}, err
|
|
|
|
}
|
|
|
|
return ires.(gomatrixserverlib.RespState), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) LookupStateIDs(
|
|
|
|
ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string,
|
|
|
|
) (res gomatrixserverlib.RespStateIDs, err error) {
|
2020-10-09 16:08:32 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
|
|
|
|
defer cancel()
|
2020-09-02 14:26:30 +00:00
|
|
|
ires, err := a.doRequest(s, func() (interface{}, error) {
|
|
|
|
return a.federation.LookupStateIDs(ctx, s, roomID, eventID)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return gomatrixserverlib.RespStateIDs{}, err
|
|
|
|
}
|
|
|
|
return ires.(gomatrixserverlib.RespStateIDs), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) GetEvent(
|
|
|
|
ctx context.Context, s gomatrixserverlib.ServerName, eventID string,
|
|
|
|
) (res gomatrixserverlib.Transaction, err error) {
|
2020-10-09 16:08:32 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
|
|
|
|
defer cancel()
|
2020-09-02 14:26:30 +00:00
|
|
|
ires, err := a.doRequest(s, func() (interface{}, error) {
|
|
|
|
return a.federation.GetEvent(ctx, s, eventID)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return gomatrixserverlib.Transaction{}, err
|
|
|
|
}
|
|
|
|
return ires.(gomatrixserverlib.Transaction), nil
|
|
|
|
}
|
2020-09-22 13:40:54 +00:00
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) GetServerKeys(
|
|
|
|
ctx context.Context, s gomatrixserverlib.ServerName,
|
|
|
|
) (gomatrixserverlib.ServerKeys, error) {
|
2020-10-09 16:08:32 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
|
|
|
|
defer cancel()
|
2020-09-22 13:40:54 +00:00
|
|
|
ires, err := a.doRequest(s, func() (interface{}, error) {
|
|
|
|
return a.federation.GetServerKeys(ctx, s)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return gomatrixserverlib.ServerKeys{}, err
|
|
|
|
}
|
|
|
|
return ires.(gomatrixserverlib.ServerKeys), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *FederationSenderInternalAPI) LookupServerKeys(
|
|
|
|
ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
|
|
|
|
) ([]gomatrixserverlib.ServerKeys, error) {
|
2020-10-09 16:08:32 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, time.Minute)
|
|
|
|
defer cancel()
|
2020-09-22 13:40:54 +00:00
|
|
|
ires, err := a.doRequest(s, func() (interface{}, error) {
|
|
|
|
return a.federation.LookupServerKeys(ctx, s, keyRequests)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return []gomatrixserverlib.ServerKeys{}, err
|
|
|
|
}
|
|
|
|
return ires.([]gomatrixserverlib.ServerKeys), nil
|
|
|
|
}
|