From b1d5360335ef9b26f62504282669403f6f3e6df5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 11:55:17 +0100 Subject: [PATCH] Update HTTP clients (#1457) * Update gomatrixserverlib * Use separate HTTP client for API calls, set User-Agent for outbound HTTP requests --- cmd/dendrite-media-api-server/main.go | 3 +- cmd/dendrite-monolith-server/main.go | 3 +- go.mod | 2 +- go.sum | 10 ++----- internal/setup/base.go | 41 ++++++++++++++++++--------- 5 files changed, 33 insertions(+), 26 deletions(-) diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index f442abfa..2c2fe3b3 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -17,7 +17,6 @@ package main import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi" - "github.com/matrix-org/gomatrixserverlib" ) func main() { @@ -26,7 +25,7 @@ func main() { defer base.Close() // nolint: errcheck userAPI := base.UserAPIClient() - client := gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation) + client := base.CreateClient() mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, userAPI, client) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 759f1c9f..28a349a7 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -29,7 +29,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/serverkeyapi" "github.com/matrix-org/dendrite/userapi" - "github.com/matrix-org/gomatrixserverlib" ) var ( @@ -125,7 +124,7 @@ func main() { monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, - Client: gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation), + Client: base.CreateClient(), FedClient: federation, KeyRing: keyRing, KafkaConsumer: base.KafkaConsumer, diff --git a/go.mod b/go.mod index c0dda07c..b4e16c0c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a + github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 64880728..461b3d3c 100644 --- a/go.sum +++ b/go.sum @@ -569,14 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929152221-6fe6457127ad h1:n0P/Oy8ZqqTPzum6FEayAjamsmvJTuIcA10WQ8GcK70= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929152221-6fe6457127ad/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154026-a52e7a5f0553 h1:tiel2c3I9xr0SRS05g3UvOjj6Sgg1I3Yn2/oGA1GgLk= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154026-a52e7a5f0553/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154241-9414c4d0b5f2 h1:a07U5eFT521mFiUtA/A8NwiZp5vfRU59/QKs+pa3Fkc= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154241-9414c4d0b5f2/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a h1:kIwbS7eY7P/MX0oN4wRHGkoc4eTTnwOcdCawBZ3SrJI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae h1:9vxYG3+pXD3G74lZ8s67/afs2RkvyrJy6HEcqfHEVgw= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/internal/setup/base.go b/internal/setup/base.go index ef956dd2..f0009d19 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -69,6 +69,7 @@ type BaseDendrite struct { PublicMediaAPIMux *mux.Router InternalAPIMux *mux.Router UseHTTPAPIs bool + apiHttpClient *http.Client httpClient *http.Client Cfg *config.Dendrite Caches *caching.Caches @@ -118,6 +119,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo logrus.WithError(err).Warnf("Failed to create cache") } + apiClient := http.Client{} client := http.Client{Timeout: HTTPClientTimeout} if cfg.FederationSender.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ @@ -148,6 +150,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo PublicKeyAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath(), PublicMediaAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(), InternalAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), + apiHttpClient: &apiClient, httpClient: &client, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, @@ -161,7 +164,7 @@ func (b *BaseDendrite) Close() error { // AppserviceHTTPClient returns the AppServiceQueryAPI for hitting the appservice component over HTTP. func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { - a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.httpClient) + a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.apiHttpClient) if err != nil { logrus.WithError(err).Panic("CreateHTTPAppServiceAPIs failed") } @@ -170,27 +173,27 @@ func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { // RoomserverHTTPClient returns RoomserverInternalAPI for hitting the roomserver over HTTP. func (b *BaseDendrite) RoomserverHTTPClient() roomserverAPI.RoomserverInternalAPI { - rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.httpClient, b.Caches) + rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.apiHttpClient, b.Caches) if err != nil { - logrus.WithError(err).Panic("RoomserverHTTPClient failed", b.httpClient) + logrus.WithError(err).Panic("RoomserverHTTPClient failed", b.apiHttpClient) } return rsAPI } // UserAPIClient returns UserInternalAPI for hitting the userapi over HTTP. func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI { - userAPI, err := userapiinthttp.NewUserAPIClient(b.Cfg.UserAPIURL(), b.httpClient) + userAPI, err := userapiinthttp.NewUserAPIClient(b.Cfg.UserAPIURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("UserAPIClient failed", b.httpClient) + logrus.WithError(err).Panic("UserAPIClient failed", b.apiHttpClient) } return userAPI } // EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { - e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.httpClient) + e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("EDUServerClient failed", b.httpClient) + logrus.WithError(err).Panic("EDUServerClient failed", b.apiHttpClient) } return e } @@ -198,9 +201,9 @@ func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { // FederationSenderHTTPClient returns FederationSenderInternalAPI for hitting // the federation sender over HTTP func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.FederationSenderInternalAPI { - f, err := fsinthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.httpClient) + f, err := fsinthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("FederationSenderHTTPClient failed", b.httpClient) + logrus.WithError(err).Panic("FederationSenderHTTPClient failed", b.apiHttpClient) } return f } @@ -209,7 +212,7 @@ func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.Federati func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { f, err := skinthttp.NewServerKeyClient( b.Cfg.ServerKeyAPIURL(), - b.httpClient, + b.apiHttpClient, b.Caches, ) if err != nil { @@ -220,9 +223,9 @@ func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { // KeyServerHTTPClient returns KeyInternalAPI for hitting the key server over HTTP func (b *BaseDendrite) KeyServerHTTPClient() keyserverAPI.KeyInternalAPI { - f, err := keyinthttp.NewKeyServerClient(b.Cfg.KeyServerURL(), b.httpClient) + f, err := keyinthttp.NewKeyServerClient(b.Cfg.KeyServerURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("KeyServerHTTPClient failed", b.httpClient) + logrus.WithError(err).Panic("KeyServerHTTPClient failed", b.apiHttpClient) } return f } @@ -238,13 +241,25 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { return db } +// CreateClient creates a new client (normally used for media fetch requests). +// Should only be called once per component. +func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client { + client := gomatrixserverlib.NewClient( + b.Cfg.FederationSender.DisableTLSValidation, + ) + client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) + return client +} + // CreateFederationClient creates a new federation client. Should only be called // once per component. func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient { - return gomatrixserverlib.NewFederationClient( + client := gomatrixserverlib.NewFederationClient( b.Cfg.Global.ServerName, b.Cfg.Global.KeyID, b.Cfg.Global.PrivateKey, b.Cfg.FederationSender.DisableTLSValidation, ) + client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) + return client } // SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on