From 9c1f38621c4d787761092bc841e06ca424fbbf35 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 2 Jul 2020 17:11:33 +0100 Subject: [PATCH] Remove publicroomsapi (#1176) * Remove all of publicroomsapi * Remove references to publicroomsapi * Remove doc references to publicroomsapi --- build/docker/config/dendrite-config.yaml | 4 +- build/docker/docker-compose.polylith.yml | 10 +- build/docker/images-build.sh | 2 +- build/docker/images-pull.sh | 2 +- build/docker/images-push.sh | 2 +- build/docker/postgres/create_db.sh | 2 +- build/gobind/monolith.go | 9 - .../types/types.go => clientapi/api/api.go | 6 +- clientapi/clientapi.go | 4 +- clientapi/routing/directory_public.go | 4 +- clientapi/routing/routing.go | 97 +++--- cmd/client-api-proxy/main.go | 28 +- cmd/dendrite-client-api-server/main.go | 2 +- cmd/dendrite-current-state-server/main.go | 33 +++ cmd/dendrite-demo-libp2p/main.go | 6 +- .../storage/postgreswithdht/storage.go | 164 ---------- .../storage/postgreswithpubsub/storage.go | 179 ----------- cmd/dendrite-demo-libp2p/storage/storage.go | 62 ---- cmd/dendrite-demo-yggdrasil/main.go | 9 - cmd/dendrite-monolith-server/main.go | 8 - cmd/dendrite-public-rooms-api-server/main.go | 41 --- cmd/dendritejs/main.go | 9 - dendrite-config.yaml | 2 - docs/INSTALL.md | 32 +- docs/WIRING-Current.md | 50 ++-- go.mod | 2 + go.sum | 9 + internal/config/config.go | 5 - internal/setup/monolith.go | 17 +- internal/test/config.go | 6 +- internal/test/server.go | 1 - publicroomsapi/README.md | 5 - publicroomsapi/consumers/roomserver.go | 99 ------- publicroomsapi/directory/directory.go | 120 -------- publicroomsapi/directory/public_rooms.go | 262 ---------------- publicroomsapi/publicroomsapi.go | 51 ---- publicroomsapi/routing/routing.go | 79 ----- publicroomsapi/storage/interface.go | 32 -- publicroomsapi/storage/postgres/prepare.go | 36 --- .../storage/postgres/public_rooms_table.go | 280 ------------------ publicroomsapi/storage/postgres/storage.go | 259 ---------------- publicroomsapi/storage/sqlite3/prepare.go | 36 --- .../storage/sqlite3/public_rooms_table.go | 273 ----------------- publicroomsapi/storage/sqlite3/storage.go | 265 ----------------- publicroomsapi/storage/storage.go | 45 --- publicroomsapi/storage/storage_wasm.go | 39 --- 46 files changed, 160 insertions(+), 2528 deletions(-) rename publicroomsapi/types/types.go => clientapi/api/api.go (81%) create mode 100644 cmd/dendrite-current-state-server/main.go delete mode 100644 cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go delete mode 100644 cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go delete mode 100644 cmd/dendrite-demo-libp2p/storage/storage.go delete mode 100644 cmd/dendrite-public-rooms-api-server/main.go delete mode 100644 publicroomsapi/README.md delete mode 100644 publicroomsapi/consumers/roomserver.go delete mode 100644 publicroomsapi/directory/directory.go delete mode 100644 publicroomsapi/directory/public_rooms.go delete mode 100644 publicroomsapi/publicroomsapi.go delete mode 100644 publicroomsapi/routing/routing.go delete mode 100644 publicroomsapi/storage/interface.go delete mode 100644 publicroomsapi/storage/postgres/prepare.go delete mode 100644 publicroomsapi/storage/postgres/public_rooms_table.go delete mode 100644 publicroomsapi/storage/postgres/storage.go delete mode 100644 publicroomsapi/storage/sqlite3/prepare.go delete mode 100644 publicroomsapi/storage/sqlite3/public_rooms_table.go delete mode 100644 publicroomsapi/storage/sqlite3/storage.go delete mode 100644 publicroomsapi/storage/storage.go delete mode 100644 publicroomsapi/storage/storage_wasm.go diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index 53d9f7b0..c8302c0d 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -98,7 +98,7 @@ database: room_server: "postgres://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable" server_key: "postgres://dendrite:itsasecret@postgres/dendrite_serverkey?sslmode=disable" federation_sender: "postgres://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable" - public_rooms_api: "postgres://dendrite:itsasecret@postgres/dendrite_publicroomsapi?sslmode=disable" + current_state: "postgres://dendrite:itsasecret@postgres/dendrite_currentstate?sslmode=disable" appservice: "postgres://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable" # If using naffka you need to specify a naffka database #naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable" @@ -113,7 +113,7 @@ listen: server_key_api: "server_key_api:7778" sync_api: "sync_api:7773" media_api: "media_api:7774" - public_rooms_api: "public_rooms_api:7775" + current_state_server: "current_state_server:7775" federation_sender: "federation_sender:7776" edu_server: "edu_server:7777" key_server: "key_server:7779" diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index d424d43b..62ca6763 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -7,8 +7,7 @@ services: "--bind-address=:8008", "--client-api-server-url=http://client_api:7771", "--sync-api-server-url=http://sync_api:7773", - "--media-api-server-url=http://media_api:7774", - "--public-rooms-api-server-url=http://public_rooms_api:7775" + "--media-api-server-url=http://media_api:7774" ] volumes: - ./config:/etc/dendrite @@ -18,7 +17,6 @@ services: - sync_api - client_api - media_api - - public_rooms_api ports: - "8008:8008" @@ -45,9 +43,9 @@ services: networks: - internal - public_rooms_api: - hostname: public_rooms_api - image: matrixdotorg/dendrite:publicroomsapi + current_state_server: + hostname: current_state_server + image: matrixdotorg/dendrite:currentstateserver command: [ "--config=dendrite.yaml" ] diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index 9ee5a09d..443f3092 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -15,7 +15,7 @@ docker build -t matrixdotorg/dendrite:federationsender --build-arg component=de docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=dendrite-public-rooms-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:currentstateserver --build-arg component=dendrite-current-state-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:serverkeyapi --build-arg component=dendrite-server-key-api-server -f build/docker/Dockerfile.component . diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index da08a732..b4a4b2fc 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -11,7 +11,7 @@ docker pull matrixdotorg/dendrite:federationsender docker pull matrixdotorg/dendrite:federationproxy docker pull matrixdotorg/dendrite:keyserver docker pull matrixdotorg/dendrite:mediaapi -docker pull matrixdotorg/dendrite:publicroomsapi +docker pull matrixdotorg/dendrite:currentstateserver docker pull matrixdotorg/dendrite:roomserver docker pull matrixdotorg/dendrite:syncapi docker pull matrixdotorg/dendrite:userapi diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index 1ac60b92..ec1e860f 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -11,7 +11,7 @@ docker push matrixdotorg/dendrite:federationsender docker push matrixdotorg/dendrite:federationproxy docker push matrixdotorg/dendrite:keyserver docker push matrixdotorg/dendrite:mediaapi -docker push matrixdotorg/dendrite:publicroomsapi +docker push matrixdotorg/dendrite:currentstateserver docker push matrixdotorg/dendrite:roomserver docker push matrixdotorg/dendrite:syncapi docker push matrixdotorg/dendrite:serverkeyapi diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh index 8ed11db1..a884b5cc 100644 --- a/build/docker/postgres/create_db.sh +++ b/build/docker/postgres/create_db.sh @@ -1,5 +1,5 @@ #!/bin/bash -for db in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka; do +for db in account device mediaapi syncapi roomserver serverkey federationsender currentstate appservice naffka; do createdb -U dendrite -O dendrite dendrite_$db done diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go index 62ca1744..ea9c62f2 100644 --- a/build/gobind/monolith.go +++ b/build/gobind/monolith.go @@ -18,7 +18,6 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -75,7 +74,6 @@ func (m *DendriteMonolith) Start() { cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory)) cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory)) cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory)) - cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory)) cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s/dendrite-currentstate.db", m.StorageDirectory)) cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory)) if err = cfg.Derive(); err != nil { @@ -111,11 +109,6 @@ func (m *DendriteMonolith) Start() { // This is different to rsAPI which can be the http client which doesn't need this dependency rsAPI.SetFederationSenderAPI(fsAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } - stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) monolith := setup.Monolith{ @@ -135,8 +128,6 @@ func (m *DendriteMonolith) Start() { UserAPI: userAPI, StateAPI: stateAPI, //ServerKeyAPI: serverKeyAPI, - - PublicRoomsDB: publicRoomsDB, } monolith.AddAllPublicRoutes(base.PublicAPIMux) diff --git a/publicroomsapi/types/types.go b/clientapi/api/api.go similarity index 81% rename from publicroomsapi/types/types.go rename to clientapi/api/api.go index 11cb0d20..dae462c0 100644 --- a/publicroomsapi/types/types.go +++ b/clientapi/api/api.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,10 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -package types +package api -// ExternalPublicRoomsProvider provides a list of homeservers who should be queried -// periodically for a list of public rooms on their server. type ExternalPublicRoomsProvider interface { // The list of homeserver domains to query. These servers will receive a request // via this API: https://matrix.org/docs/spec/server_server/latest#public-room-directory diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 8ea84249..bbce6dcc 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -18,6 +18,7 @@ import ( "github.com/Shopify/sarama" "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/routing" currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" @@ -47,6 +48,7 @@ func AddPublicRoutes( transactionsCache *transactions.Cache, fsAPI federationSenderAPI.FederationSenderInternalAPI, userAPI userapi.UserInternalAPI, + extRoomsProvider api.ExternalPublicRoomsProvider, ) { syncProducer := &producers.SyncAPIProducer{ Producer: producer, @@ -56,6 +58,6 @@ func AddPublicRoutes( routing.Setup( router, cfg, eduInputAPI, rsAPI, asAPI, accountsDB, deviceDB, userAPI, federation, - syncProducer, transactionsCache, fsAPI, stateAPI, + syncProducer, transactionsCache, fsAPI, stateAPI, extRoomsProvider, ) } diff --git a/clientapi/routing/directory_public.go b/clientapi/routing/directory_public.go index 6d0db579..64600cb4 100644 --- a/clientapi/routing/directory_public.go +++ b/clientapi/routing/directory_public.go @@ -24,10 +24,10 @@ import ( "sync" "time" + "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" - "github.com/matrix-org/dendrite/publicroomsapi/types" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -64,7 +64,7 @@ func GetPostPublicRooms( // GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied. func GetPostPublicRoomsWithExternal( req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, - fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, + fedClient *gomatrixserverlib.FederationClient, extRoomsProvider api.ExternalPublicRoomsProvider, ) util.JSONResponse { var request PublicRoomReq if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 57bb921d..85c5c0d9 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -21,6 +21,7 @@ import ( "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" @@ -30,7 +31,6 @@ import ( "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/dendrite/userapi/storage/devices" @@ -55,12 +55,13 @@ func Setup( asAPI appserviceAPI.AppServiceQueryAPI, accountDB accounts.Database, deviceDB devices.Database, - userAPI api.UserInternalAPI, + userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, syncProducer *producers.SyncAPIProducer, transactionsCache *transactions.Cache, federationSender federationSenderAPI.FederationSenderInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, + extRoomsProvider api.ExternalPublicRoomsProvider, ) { publicAPIMux.Handle("/client/versions", @@ -84,12 +85,12 @@ func Setup( unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter() r0mux.Handle("/createRoom", - httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", - httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -100,12 +101,12 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/joined_rooms", - httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return GetJoinedRooms(req, device, stateAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/join", - httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -116,7 +117,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/leave", - httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -127,7 +128,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/ban", - httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -136,7 +137,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/invite", - httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -145,7 +146,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/kick", - httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -154,7 +155,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/unban", - httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -163,7 +164,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", - httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -172,7 +173,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", - httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -183,7 +184,7 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", - httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -192,7 +193,7 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -200,7 +201,7 @@ func Setup( return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -214,7 +215,7 @@ func Setup( return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], eventType, "", eventFormat) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -224,7 +225,7 @@ func Setup( })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", - httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -240,7 +241,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}", - httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -273,7 +274,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -283,7 +284,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -321,19 +322,19 @@ func Setup( ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) r0mux.Handle("/logout", - httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return Logout(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/logout/all", - httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return LogoutAll(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/typing/{userID}", - httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -343,7 +344,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/sendToDevice/{eventType}/{txnID}", - httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -357,7 +358,7 @@ func Setup( // rather than r0. It's an exact duplicate of the above handler. // TODO: Remove this if/when sytest is fixed! unstableMux.Handle("/sendToDevice/{eventType}/{txnID}", - httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -368,7 +369,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/account/whoami", - httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return Whoami(req, device) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -430,7 +431,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", - httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -452,7 +453,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/displayname", - httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -464,19 +465,19 @@ func Setup( // PUT requests, so we need to allow this method r0mux.Handle("/account/3pid", - httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return GetAssociated3PIDs(req, accountDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/account/3pid", - httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return CheckAndSave3PIDAssociation(req, accountDB, device, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) unstableMux.Handle("/account/3pid/delete", - httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return Forget3PID(req, accountDB) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -499,7 +500,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/voip/turnServer", - httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return RequestTurnServer(req, device, cfg) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -525,7 +526,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -535,7 +536,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -545,7 +546,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -555,7 +556,7 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -565,7 +566,7 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/rooms/{roomID}/members", - httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -575,7 +576,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/joined_members", - httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -592,13 +593,13 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/devices", - httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return GetDevicesByLocalpart(req, deviceDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -608,7 +609,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -618,7 +619,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -628,7 +629,7 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/delete_devices", - httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return DeleteDevices(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -653,7 +654,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags", - httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -663,7 +664,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -673,7 +674,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -683,7 +684,7 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/capabilities", - httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) diff --git a/cmd/client-api-proxy/main.go b/cmd/client-api-proxy/main.go index ebc0629f..742ec3e3 100644 --- a/cmd/client-api-proxy/main.go +++ b/cmd/client-api-proxy/main.go @@ -48,13 +48,12 @@ Arguments: ` var ( - syncServerURL = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'") - clientAPIURL = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'") - mediaAPIURL = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'") - publicRoomsAPIURL = flag.String("public-rooms-api-server-url", "", "The base URL of the listening 'dendrite-public-rooms-api-server' process. E.g. 'http://localhost:7775'") - bindAddress = flag.String("bind-address", ":8008", "The listening port for the proxy.") - certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") - keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") + syncServerURL = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'") + clientAPIURL = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'") + mediaAPIURL = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'") + bindAddress = flag.String("bind-address", ":8008", "The listening port for the proxy.") + certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") + keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") ) func makeProxy(targetURL string) (*httputil.ReverseProxy, error) { @@ -121,13 +120,6 @@ func main() { fmt.Fprintln(os.Stderr, "no --media-api-server-url specified.") os.Exit(1) } - - if *publicRoomsAPIURL == "" { - flag.Usage() - fmt.Fprintln(os.Stderr, "no --public-rooms-api-server-url specified.") - os.Exit(1) - } - syncProxy, err := makeProxy(*syncServerURL) if err != nil { panic(err) @@ -140,14 +132,8 @@ func main() { if err != nil { panic(err) } - publicRoomsProxy, err := makeProxy(*publicRoomsAPIURL) - if err != nil { - panic(err) - } http.Handle("/_matrix/client/r0/sync", syncProxy) - http.Handle("/_matrix/client/r0/directory/list/", publicRoomsProxy) - http.Handle("/_matrix/client/r0/publicRooms", publicRoomsProxy) http.Handle("/_matrix/media/v1/", mediaProxy) http.Handle("/", clientProxy) @@ -159,8 +145,6 @@ func main() { fmt.Println("Proxying requests to:") fmt.Println(" /_matrix/client/r0/sync => ", *syncServerURL+"/api/_matrix/client/r0/sync") - fmt.Println(" /_matrix/client/r0/directory/list => ", *publicRoomsAPIURL+"/_matrix/client/r0/directory/list") - fmt.Println(" /_matrix/client/r0/publicRooms => ", *publicRoomsAPIURL+"/_matrix/media/client/r0/publicRooms") fmt.Println(" /_matrix/media/v1 => ", *mediaAPIURL+"/api/_matrix/media/v1") fmt.Println(" /* => ", *clientAPIURL+"/api/*") fmt.Println("Listening on ", *bindAddress) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index f46dae50..58c029fe 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -39,7 +39,7 @@ func main() { clientapi.AddPublicRoutes( base.PublicAPIMux, base.Cfg, base.KafkaProducer, deviceDB, accountDB, federation, - rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI, + rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI, nil, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-current-state-server/main.go b/cmd/dendrite-current-state-server/main.go new file mode 100644 index 00000000..0d4eae7b --- /dev/null +++ b/cmd/dendrite-current-state-server/main.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/matrix-org/dendrite/currentstateserver" + "github.com/matrix-org/dendrite/internal/setup" +) + +func main() { + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "CurrentStateServer", true) + defer base.Close() // nolint: errcheck + + stateAPI := currentstateserver.NewInternalAPI(cfg, base.KafkaConsumer) + + currentstateserver.AddInternalRoutes(base.InternalAPIMux, stateAPI) + + base.SetupAndServeHTTP(string(base.Cfg.Bind.CurrentState), string(base.Cfg.Listen.CurrentState)) + +} diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 4bb7a96c..988f4aa7 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -29,7 +29,6 @@ import ( p2phttp "github.com/libp2p/go-libp2p-http" p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage" "github.com/matrix-org/dendrite/currentstateserver" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/federationsender" @@ -130,7 +129,6 @@ func main() { cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) - cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicrooms.db", *instanceName)) cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName)) if err = cfg.Derive(); err != nil { @@ -164,10 +162,12 @@ func main() { &base.Base, federation, rsAPI, keyRing, ) rsAPI.SetFederationSenderAPI(fsAPI) + /* TODO: publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub, cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } + */ stateAPI := currentstateserver.NewInternalAPI(base.Base.Cfg, base.Base.KafkaConsumer) monolith := setup.Monolith{ @@ -187,8 +187,6 @@ func main() { ServerKeyAPI: serverKeyAPI, StateAPI: stateAPI, UserAPI: userAPI, - - PublicRoomsDB: publicRoomsDB, } monolith.AddAllPublicRoutes(base.Base.PublicAPIMux) diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go deleted file mode 100644 index d2cb36a8..00000000 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgreswithdht - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "sync/atomic" - "time" - - "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" - "github.com/matrix-org/gomatrixserverlib" - - dht "github.com/libp2p/go-libp2p-kad-dht" -) - -const DHTInterval = time.Second * 10 - -// PublicRoomsServerDatabase represents a public rooms server database. -type PublicRoomsServerDatabase struct { - dht *dht.IpfsDHT - postgres.PublicRoomsServerDatabase - ourRoomsContext context.Context // our current value in the DHT - ourRoomsCancel context.CancelFunc // cancel when we want to expire our value - foundRooms map[string]gomatrixserverlib.PublicRoom // additional rooms we have learned about from the DHT - foundRoomsMutex sync.RWMutex // protects foundRooms - maintenanceTimer *time.Timer // - roomsAdvertised atomic.Value // stores int - roomsDiscovered atomic.Value // stores int -} - -// NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName) - if err != nil { - return nil, err - } - provider := PublicRoomsServerDatabase{ - dht: dht, - PublicRoomsServerDatabase: *pg, - } - go provider.ResetDHTMaintenance() - provider.roomsAdvertised.Store(0) - provider.roomsDiscovered.Store(0) - return &provider, nil -} - -func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) { - return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID) -} - -func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error { - d.ResetDHTMaintenance() - return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID) -} - -func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { - count, err := d.PublicRoomsServerDatabase.CountPublicRooms(ctx) - if err != nil { - return 0, err - } - d.foundRoomsMutex.RLock() - defer d.foundRoomsMutex.RUnlock() - return count + int64(len(d.foundRooms)), nil -} - -func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) { - realfilter := filter - if realfilter == "__local__" { - realfilter = "" - } - rooms, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, realfilter) - if err != nil { - return []gomatrixserverlib.PublicRoom{}, err - } - if filter != "__local__" { - d.foundRoomsMutex.RLock() - defer d.foundRoomsMutex.RUnlock() - for _, room := range d.foundRooms { - rooms = append(rooms, room) - } - } - return rooms, nil -} - -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error { - return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove) -} - -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error { - return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event) -} - -func (d *PublicRoomsServerDatabase) ResetDHTMaintenance() { - if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() { - <-d.maintenanceTimer.C - } - d.Interval() -} - -func (d *PublicRoomsServerDatabase) Interval() { - if err := d.AdvertiseRoomsIntoDHT(); err != nil { - // fmt.Println("Failed to advertise room in DHT:", err) - } - if err := d.FindRoomsInDHT(); err != nil { - // fmt.Println("Failed to find rooms in DHT:", err) - } - fmt.Println("Found", d.roomsDiscovered.Load(), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)") - d.maintenanceTimer = time.AfterFunc(DHTInterval, d.Interval) -} - -func (d *PublicRoomsServerDatabase) AdvertiseRoomsIntoDHT() error { - dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second) - _ = dbCancel - ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__") - if err != nil { - return err - } - if j, err := json.Marshal(ourRooms); err == nil { - d.roomsAdvertised.Store(len(ourRooms)) - d.ourRoomsContext, d.ourRoomsCancel = context.WithCancel(context.Background()) - if err := d.dht.PutValue(d.ourRoomsContext, "/matrix/publicRooms", j); err != nil { - return err - } - } - return nil -} - -func (d *PublicRoomsServerDatabase) FindRoomsInDHT() error { - d.foundRoomsMutex.Lock() - searchCtx, searchCancel := context.WithTimeout(context.Background(), 10*time.Second) - defer searchCancel() - defer d.foundRoomsMutex.Unlock() - results, err := d.dht.GetValues(searchCtx, "/matrix/publicRooms", 1024) - if err != nil { - return err - } - d.foundRooms = make(map[string]gomatrixserverlib.PublicRoom) - for _, result := range results { - var received []gomatrixserverlib.PublicRoom - if err := json.Unmarshal(result.Val, &received); err != nil { - return err - } - for _, room := range received { - d.foundRooms[room.RoomID] = room - } - } - d.roomsDiscovered.Store(len(d.foundRooms)) - return nil -} diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go deleted file mode 100644 index cf642eb3..00000000 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgreswithpubsub - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "sync/atomic" - "time" - - "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" - "github.com/matrix-org/gomatrixserverlib" - - pubsub "github.com/libp2p/go-libp2p-pubsub" -) - -const MaintenanceInterval = time.Second * 10 - -type discoveredRoom struct { - time time.Time - room gomatrixserverlib.PublicRoom -} - -// PublicRoomsServerDatabase represents a public rooms server database. -type PublicRoomsServerDatabase struct { - postgres.PublicRoomsServerDatabase // - pubsub *pubsub.PubSub // - subscription *pubsub.Subscription // - foundRooms map[string]discoveredRoom // additional rooms we have learned about from the DHT - foundRoomsMutex sync.RWMutex // protects foundRooms - maintenanceTimer *time.Timer // - roomsAdvertised atomic.Value // stores int -} - -// NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName) - if err != nil { - return nil, err - } - provider := PublicRoomsServerDatabase{ - pubsub: pubsub, - PublicRoomsServerDatabase: *pg, - foundRooms: make(map[string]discoveredRoom), - } - if topic, err := pubsub.Join("/matrix/publicRooms"); err != nil { - return nil, err - } else if sub, err := topic.Subscribe(); err == nil { - provider.subscription = sub - go provider.MaintenanceTimer() - go provider.FindRooms() - provider.roomsAdvertised.Store(0) - return &provider, nil - } else { - return nil, err - } -} - -func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) { - return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID) -} - -func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error { - d.MaintenanceTimer() - return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID) -} - -func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { - d.foundRoomsMutex.RLock() - defer d.foundRoomsMutex.RUnlock() - return int64(len(d.foundRooms)), nil -} - -func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) { - var rooms []gomatrixserverlib.PublicRoom - if filter == "__local__" { - if r, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, ""); err == nil { - rooms = append(rooms, r...) - } else { - return []gomatrixserverlib.PublicRoom{}, err - } - } else { - d.foundRoomsMutex.RLock() - defer d.foundRoomsMutex.RUnlock() - for _, room := range d.foundRooms { - rooms = append(rooms, room.room) - } - } - return rooms, nil -} - -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error { - return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove) -} - -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error { - return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event) -} - -func (d *PublicRoomsServerDatabase) MaintenanceTimer() { - if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() { - <-d.maintenanceTimer.C - } - d.Interval() -} - -func (d *PublicRoomsServerDatabase) Interval() { - d.foundRoomsMutex.Lock() - for k, v := range d.foundRooms { - if time.Since(v.time) > time.Minute { - delete(d.foundRooms, k) - } - } - d.foundRoomsMutex.Unlock() - if err := d.AdvertiseRooms(); err != nil { - fmt.Println("Failed to advertise room in DHT:", err) - } - d.foundRoomsMutex.RLock() - defer d.foundRoomsMutex.RUnlock() - fmt.Println("Found", len(d.foundRooms), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)") - d.maintenanceTimer = time.AfterFunc(MaintenanceInterval, d.Interval) -} - -func (d *PublicRoomsServerDatabase) AdvertiseRooms() error { - dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second) - _ = dbCancel - ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__") - if err != nil { - return err - } - advertised := 0 - for _, room := range ourRooms { - if j, err := json.Marshal(room); err == nil { - if topic, err := d.pubsub.Join("/matrix/publicRooms"); err != nil { - fmt.Println("Failed to subscribe to topic:", err) - } else if err := topic.Publish(context.TODO(), j); err != nil { - fmt.Println("Failed to publish public room:", err) - } else { - advertised++ - } - } - } - - d.roomsAdvertised.Store(advertised) - return nil -} - -func (d *PublicRoomsServerDatabase) FindRooms() { - for { - msg, err := d.subscription.Next(context.Background()) - if err != nil { - continue - } - received := discoveredRoom{ - time: time.Now(), - } - if err := json.Unmarshal(msg.Data, &received.room); err != nil { - fmt.Println("Unmarshal error:", err) - continue - } - d.foundRoomsMutex.Lock() - d.foundRooms[received.room.RoomID] = received - d.foundRoomsMutex.Unlock() - } -} diff --git a/cmd/dendrite-demo-libp2p/storage/storage.go b/cmd/dendrite-demo-libp2p/storage/storage.go deleted file mode 100644 index 2d8dc181..00000000 --- a/cmd/dendrite-demo-libp2p/storage/storage.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "net/url" - - dht "github.com/libp2p/go-libp2p-kad-dht" - pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithdht" - "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" -) - -const schemePostgres = "postgres" -const schemeFile = "file" - -// NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (storage.Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName) - } - switch uri.Scheme { - case schemePostgres: - return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName) - case schemeFile: - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) - default: - return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName) - } -} - -// NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (storage.Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName) - } - switch uri.Scheme { - case schemePostgres: - return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName) - case schemeFile: - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) - default: - return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName) - } -} diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index cef34c7e..7476d686 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -34,7 +34,6 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -74,7 +73,6 @@ func main() { cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) - cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicrooms.db", *instanceName)) cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName)) cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) if err = cfg.Derive(); err != nil { @@ -110,11 +108,6 @@ func main() { rsComponent.SetFederationSenderAPI(fsAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } - embed.Embed(base.BaseMux, *instancePort, "Yggdrasil Demo") stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) @@ -136,8 +129,6 @@ func main() { UserAPI: userAPI, StateAPI: stateAPI, //ServerKeyAPI: serverKeyAPI, - - PublicRoomsDB: publicRoomsDB, } monolith.AddAllPublicRoutes(base.PublicAPIMux) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 905eda2b..9ac6941b 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -27,7 +27,6 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/serverkeyapi" @@ -118,11 +117,6 @@ func main() { // This is different to rsAPI which can be the http client which doesn't need this dependency rsImpl.SetFederationSenderAPI(fsAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } - stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) monolith := setup.Monolith{ @@ -142,8 +136,6 @@ func main() { ServerKeyAPI: serverKeyAPI, StateAPI: stateAPI, UserAPI: userAPI, - - PublicRoomsDB: publicRoomsDB, } monolith.AddAllPublicRoutes(base.PublicAPIMux) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go deleted file mode 100644 index 23866b75..00000000 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/publicroomsapi" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/sirupsen/logrus" -) - -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "PublicRoomsAPI", true) - defer base.Close() // nolint: errcheck - - userAPI := base.UserAPIClient() - - rsAPI := base.RoomserverHTTPClient() - - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } - publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, userAPI, publicRoomsDB, rsAPI, nil, nil) - - base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) - -} diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 36ce9f65..6e2bdafe 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -29,7 +29,6 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/userapi" go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" @@ -169,7 +168,6 @@ func main() { cfg.Database.FederationSender = "file:/idb/dendritejs_fedsender.db" cfg.Database.MediaAPI = "file:/idb/dendritejs_mediaapi.db" cfg.Database.Naffka = "file:/idb/dendritejs_naffka.db" - cfg.Database.PublicRoomsAPI = "file:/idb/dendritejs_publicrooms.db" cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db" cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db" cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db" @@ -215,11 +213,6 @@ func main() { rsAPI.SetFederationSenderAPI(fedSenderAPI) p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } - stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) monolith := setup.Monolith{ @@ -239,8 +232,6 @@ func main() { StateAPI: stateAPI, UserAPI: userAPI, //ServerKeyAPI: serverKeyAPI, - - PublicRoomsDB: publicRoomsDB, ExtPublicRoomsProvider: p2pPublicRoomProvider, } monolith.AddAllPublicRoutes(base.PublicAPIMux) diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 70c8f795..2b95c102 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -120,7 +120,6 @@ database: server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable" federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable" appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable" - public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable" current_state: "postgres://dendrite:itsasecret@localhost/dendrite_currentstate?sslmode=disable" max_open_conns: 100 max_idle_conns: 2 @@ -137,7 +136,6 @@ listen: federation_api: "localhost:7772" sync_api: "localhost:7773" media_api: "localhost:7774" - public_rooms_api: "localhost:7775" federation_sender: "localhost:7776" appservice_api: "localhost:7777" edu_server: "localhost:7778" diff --git a/docs/INSTALL.md b/docs/INSTALL.md index b4c81a42..c9735180 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -108,7 +108,7 @@ Assuming that Postgres 9.5 (or later) is installed: * Create the component databases: ```bash - for i in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka; do + for i in account device mediaapi syncapi roomserver serverkey federationsender currentstate appservice naffka; do sudo -u postgres createdb -O dendrite dendrite_$i done ``` @@ -176,17 +176,17 @@ The following contains scripts which will run all the required processes in orde | | :7774 | | | | - | | /directory +----------------------------------+ - | | +--------->| dendrite-public-rooms-api-server |<========++ - | | | +----------------------------------+ || - | | | :7775 | || - | | | +<-----------+ || - | | | | || - | | | /sync +--------------------------+ || + | | + | | + | | + | | + | | + | | + | | /sync +--------------------------+ | | +--------->| dendrite-sync-api-server |<================++ - | | | | +--------------------------+ || - | | | | :7773 | ^^ || -Matrix +------------------+ | | | | || client_data || + | | | +--------------------------+ || + | | | :7773 | ^^ || +Matrix +------------------+ | | | || client_data || Clients --->| client-api-proxy |-------+ +<-----------+ ++=============++ || +------------------+ | | | || || :8008 | | CS API +----------------------------+ || || @@ -232,7 +232,6 @@ your client at `http://localhost:8008`. --client-api-server-url "http://localhost:7771" \ --sync-api-server-url "http://localhost:7773" \ --media-api-server-url "http://localhost:7774" \ ---public-rooms-api-server-url "http://localhost:7775" \ ``` ### Federation proxy @@ -282,15 +281,6 @@ order to upload and retrieve media. ./bin/dendrite-media-api-server --config dendrite.yaml ``` -### Public room server - -This implements `/directory` requests. Clients talk to this via the proxy -in order to retrieve room directory listings. - -```bash -./bin/dendrite-public-rooms-api-server --config dendrite.yaml -``` - ### Federation API server This implements federation requests. Servers talk to this via the proxy in diff --git a/docs/WIRING-Current.md b/docs/WIRING-Current.md index ec539d4e..b74f341e 100644 --- a/docs/WIRING-Current.md +++ b/docs/WIRING-Current.md @@ -9,23 +9,22 @@ a request/response model like HTTP or RPC. Therefore, components can expose "int Note in Monolith mode these are actually direct function calls and are not serialised HTTP requests. ``` - Tier 1 Sync PublicRooms FederationAPI ClientAPI MediaAPI -Public Facing | .-----1------` | | | | | | | | | - 2 | .-------3-----------------` | | | `--------|-|-|-|--11--------------------. - | | | .--------4----------------------------------` | | | | - | | | | .---5-----------` | | | | | | - | | | | | .---6----------------------------` | | | - | | | | | | | .-----7----------` | | - | | | | | | 8 | | 10 | - | | | | | | | | `---9----. | | - V V V V V V V V V V V + Tier 1 Sync FederationAPI ClientAPI MediaAPI +Public Facing | | | | | | | | | | + 2 .-------3-----------------` | | | `--------|-|-|-|--11--------------------. + | | .--------4----------------------------------` | | | | + | | | .---5-----------` | | | | | | + | | | | .---6----------------------------` | | | + | | | | | | .-----7----------` | | + | | | | | 8 | | 10 | + | | | | | | | `---9----. | | + V V V V V V V V V V Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI Internal only | `------------------------12----------^ ^ `------------------------------------------------------------13----------` Client ---> Server ``` -- 1 (PublicRooms -> Roomserver): Calculating current auth for changing visibility - 2 (Sync -> Roomserver): When making backfill requests - 3 (FedAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing backfill/state/state_ids requests - 4 (ClientAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing /state requests @@ -46,20 +45,20 @@ In addition to this, all public facing components (Tier 1) talk to the `UserAPI` ``` .----1--------------------------------------------. V | - Tier 1 Sync PublicRooms FederationAPI ClientAPI MediaAPI -Public Facing ^ ^ ^ ^ - | | | | - 2 | | | + Tier 1 Sync FederationAPI ClientAPI MediaAPI +Public Facing ^ ^ ^ + | | | + 2 | | | `-3------------. | - | | | | - | | | | - | .------4------` | | - | | .--------5-----|------------------------------` - | | | | + | | | + | | | + | | | + | .--------4-----|------------------------------` + | | | Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI Internal only | | ^ ^ - | `-----6----------` | - `--------------------7--------` + | `-----5----------` | + `--------------------6--------` Producer ----> Consumer @@ -67,7 +66,6 @@ Producer ----> Consumer - 1 (ClientAPI -> Sync): For tracking account data - 2 (Roomserver -> Sync): For all data to send to clients - 3 (EDUServer -> Sync): For typing/send-to-device data to send to clients -- 4 (Roomserver -> PublicRooms): For tracking the current room name/topic/joined count/etc. -- 5 (Roomserver -> ClientAPI): For tracking memberships for profile updates. -- 6 (EDUServer -> FedSender): For sending EDUs over federation -- 7 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts. +- 4 (Roomserver -> ClientAPI): For tracking memberships for profile updates. +- 5 (EDUServer -> FedSender): For sending EDUs over federation +- 6 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts. diff --git a/go.mod b/go.mod index 4add4c3f..b278ec4b 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/matrix-org/dendrite require ( github.com/Shopify/sarama v1.26.1 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/go-metrics v0.0.1 // indirect github.com/gologme/log v1.2.0 github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 diff --git a/go.sum b/go.sum index 52a55697..a98ae851 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,10 @@ github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhY github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -189,6 +193,7 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= @@ -497,6 +502,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -504,10 +510,12 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= @@ -650,6 +658,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/p golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/config/config.go b/internal/config/config.go index 8275fc47..777bd6a3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -186,9 +186,6 @@ type Dendrite struct { // The AppServices database stores information used by the AppService component. // It is only accessed by the AppService component. AppService DataSource `yaml:"appservice"` - // The PublicRoomsAPI database stores information used to compute the public - // room directory. It is only accessed by the PublicRoomsAPI server. - PublicRoomsAPI DataSource `yaml:"public_rooms_api"` // The Naffka database is used internally by the naffka library, if used. Naffka DataSource `yaml:"naffka,omitempty"` // Maximum open connections to the DB (0 = use default, negative means unlimited) @@ -233,7 +230,6 @@ type Dendrite struct { UserAPI Address `yaml:"user_api"` RoomServer Address `yaml:"room_server"` FederationSender Address `yaml:"federation_sender"` - PublicRoomsAPI Address `yaml:"public_rooms_api"` EDUServer Address `yaml:"edu_server"` KeyServer Address `yaml:"key_server"` } `yaml:"bind"` @@ -250,7 +246,6 @@ type Dendrite struct { UserAPI Address `yaml:"user_api"` RoomServer Address `yaml:"room_server"` FederationSender Address `yaml:"federation_sender"` - PublicRoomsAPI Address `yaml:"public_rooms_api"` EDUServer Address `yaml:"edu_server"` KeyServer Address `yaml:"key_server"` } `yaml:"listen"` diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index d4ae0915..ca90986e 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -19,6 +19,7 @@ import ( "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/clientapi/api" currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi" @@ -27,8 +28,6 @@ import ( "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/mediaapi" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/publicroomsapi/types" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/dendrite/syncapi" @@ -58,13 +57,8 @@ type Monolith struct { UserAPI userapi.UserInternalAPI StateAPI currentstateAPI.CurrentStateInternalAPI - // TODO: can we remove this? It's weird that we are required the database - // yet every other component can do that on its own. libp2p-demo uses a custom - // database though annoyingly. - PublicRoomsDB storage.Database - // Optional - ExtPublicRoomsProvider types.ExternalPublicRoomsProvider + ExtPublicRoomsProvider api.ExternalPublicRoomsProvider } // AddAllPublicRoutes attaches all public paths to the given router @@ -73,7 +67,7 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { publicMux, m.Config, m.KafkaProducer, m.DeviceDB, m.AccountDB, m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, m.StateAPI, transactions.New(), - m.FederationSenderAPI, m.UserAPI, + m.FederationSenderAPI, m.UserAPI, m.ExtPublicRoomsProvider, ) keyserver.AddPublicRoutes(publicMux, m.Config, m.UserAPI) @@ -83,11 +77,6 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { m.EDUInternalAPI, m.StateAPI, ) mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client) - /* - publicroomsapi.AddPublicRoutes( - publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient, - m.ExtPublicRoomsProvider, - ) */ syncapi.AddPublicRoutes( publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, m.FedClient, m.Config, ) diff --git a/internal/test/config.go b/internal/test/config.go index 951f65a1..bbcc9bed 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -96,7 +96,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Database.RoomServer = config.DataSource(database) cfg.Database.ServerKey = config.DataSource(database) cfg.Database.SyncAPI = config.DataSource(database) - cfg.Database.PublicRoomsAPI = config.DataSource(database) + cfg.Database.CurrentState = config.DataSource(database) cfg.Listen.ClientAPI = assignAddress() cfg.Listen.AppServiceAPI = assignAddress() @@ -104,7 +104,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Listen.MediaAPI = assignAddress() cfg.Listen.RoomServer = assignAddress() cfg.Listen.SyncAPI = assignAddress() - cfg.Listen.PublicRoomsAPI = assignAddress() + cfg.Listen.CurrentState = assignAddress() cfg.Listen.EDUServer = assignAddress() // Bind to the same address as the listen address @@ -115,7 +115,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Bind.MediaAPI = cfg.Listen.MediaAPI cfg.Bind.RoomServer = cfg.Listen.RoomServer cfg.Bind.SyncAPI = cfg.Listen.SyncAPI - cfg.Bind.PublicRoomsAPI = cfg.Listen.PublicRoomsAPI + cfg.Bind.CurrentState = cfg.Listen.CurrentState cfg.Bind.EDUServer = cfg.Listen.EDUServer return &cfg, port, nil diff --git a/internal/test/server.go b/internal/test/server.go index c3348d53..2d4117f4 100644 --- a/internal/test/server.go +++ b/internal/test/server.go @@ -99,7 +99,6 @@ func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) { "--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI), "--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI), "--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI), - "--public-rooms-api-server-url", "http://" + string(cfg.Listen.PublicRoomsAPI), "--tls-cert", "server.crt", "--tls-key", "server.key", } diff --git a/publicroomsapi/README.md b/publicroomsapi/README.md deleted file mode 100644 index 594fe29c..00000000 --- a/publicroomsapi/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Public rooms API - -This server is responsible for serving requests hitting `/publicRooms` and `/directory/list/room/{roomID}` as per: - -https://matrix.org/docs/spec/client_server/r0.2.0.html#listing-rooms diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go deleted file mode 100644 index b9686d56..00000000 --- a/publicroomsapi/consumers/roomserver.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package consumers - -import ( - "context" - "encoding/json" - - "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" - log "github.com/sirupsen/logrus" -) - -// OutputRoomEventConsumer consumes events that originated in the room server. -type OutputRoomEventConsumer struct { - rsAPI api.RoomserverInternalAPI - rsConsumer *internal.ContinualConsumer - db storage.Database -} - -// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. -func NewOutputRoomEventConsumer( - cfg *config.Dendrite, - kafkaConsumer sarama.Consumer, - store storage.Database, - rsAPI api.RoomserverInternalAPI, -) *OutputRoomEventConsumer { - consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputRoomEvent), - Consumer: kafkaConsumer, - PartitionStore: store, - } - s := &OutputRoomEventConsumer{ - rsConsumer: &consumer, - db: store, - rsAPI: rsAPI, - } - consumer.ProcessMessage = s.onMessage - - return s -} - -// Start consuming from room servers -func (s *OutputRoomEventConsumer) Start() error { - return s.rsConsumer.Start() -} - -// onMessage is called when the sync server receives a new event from the room server output log. -func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { - // Parse out the event JSON - var output api.OutputEvent - if err := json.Unmarshal(msg.Value, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("roomserver output log: message parse failure") - return nil - } - - if output.Type != api.OutputTypeNewRoomEvent { - log.WithField("type", output.Type).Debug( - "roomserver output log: ignoring unknown output type", - ) - return nil - } - - var remQueryRes api.QueryEventsByIDResponse - if len(output.NewRoomEvent.RemovesStateEventIDs) > 0 { - remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} - if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { - log.Warn(err) - return err - } - } - - var addQueryEvents, remQueryEvents []gomatrixserverlib.Event - for _, headeredEvent := range output.NewRoomEvent.AddsState() { - addQueryEvents = append(addQueryEvents, headeredEvent.Event) - } - for _, headeredEvent := range remQueryRes.Events { - remQueryEvents = append(remQueryEvents, headeredEvent.Event) - } - - return s.db.UpdateRoomFromEvents(context.TODO(), addQueryEvents, remQueryEvents) -} diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go deleted file mode 100644 index 8b68279a..00000000 --- a/publicroomsapi/directory/directory.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package directory - -import ( - "net/http" - - "github.com/matrix-org/dendrite/roomserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" - - "github.com/matrix-org/dendrite/clientapi/httputil" - "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/gomatrixserverlib" - - "github.com/matrix-org/util" -) - -type roomVisibility struct { - Visibility string `json:"visibility"` -} - -// GetVisibility implements GET /directory/list/room/{roomID} -func GetVisibility( - req *http.Request, publicRoomsDatabase storage.Database, - roomID string, -) util.JSONResponse { - isPublic, err := publicRoomsDatabase.GetRoomVisibility(req.Context(), roomID) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("publicRoomsDatabase.GetRoomVisibility failed") - return jsonerror.InternalServerError() - } - - var v roomVisibility - if isPublic { - v.Visibility = gomatrixserverlib.Public - } else { - v.Visibility = "private" - } - - return util.JSONResponse{ - Code: http.StatusOK, - JSON: v, - } -} - -// SetVisibility implements PUT /directory/list/room/{roomID} -// TODO: Allow admin users to edit the room visibility -func SetVisibility( - req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *userapi.Device, - roomID string, -) util.JSONResponse { - queryMembershipReq := api.QueryMembershipForUserRequest{ - RoomID: roomID, - UserID: dev.UserID, - } - var queryMembershipRes api.QueryMembershipForUserResponse - err := rsAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user") - return jsonerror.InternalServerError() - } - // Check if user id is in room - if !queryMembershipRes.IsInRoom { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("user does not belong to room"), - } - } - queryEventsReq := api.QueryLatestEventsAndStateRequest{ - RoomID: roomID, - StateToFetch: []gomatrixserverlib.StateKeyTuple{{ - EventType: gomatrixserverlib.MRoomPowerLevels, - StateKey: "", - }}, - } - var queryEventsRes api.QueryLatestEventsAndStateResponse - err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) - if err != nil || len(queryEventsRes.StateEvents) == 0 { - util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") - return jsonerror.InternalServerError() - } - - // NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event - power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event) - if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"), - } - } - - var v roomVisibility - if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil { - return *reqErr - } - - isPublic := v.Visibility == gomatrixserverlib.Public - if err := publicRoomsDatabase.SetRoomVisibility(req.Context(), isPublic, roomID); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("publicRoomsDatabase.SetRoomVisibility failed") - return jsonerror.InternalServerError() - } - - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct{}{}, - } -} diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go deleted file mode 100644 index df9df8ff..00000000 --- a/publicroomsapi/directory/public_rooms.go +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package directory - -import ( - "context" - "math/rand" - "net/http" - "sort" - "strconv" - "sync" - "time" - - "github.com/matrix-org/dendrite/clientapi/httputil" - "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/publicroomsapi/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -type PublicRoomReq struct { - Since string `json:"since,omitempty"` - Limit int16 `json:"limit,omitempty"` - Filter filter `json:"filter,omitempty"` -} - -type filter struct { - SearchTerms string `json:"generic_search_term,omitempty"` -} - -// GetPostPublicRooms implements GET and POST /publicRooms -func GetPostPublicRooms( - req *http.Request, publicRoomDatabase storage.Database, -) util.JSONResponse { - var request PublicRoomReq - if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { - return *fillErr - } - response, err := publicRooms(req.Context(), request, publicRoomDatabase) - if err != nil { - return jsonerror.InternalServerError() - } - return util.JSONResponse{ - Code: http.StatusOK, - JSON: response, - } -} - -// GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied. -func GetPostPublicRoomsWithExternal( - req *http.Request, publicRoomDatabase storage.Database, fedClient *gomatrixserverlib.FederationClient, - extRoomsProvider types.ExternalPublicRoomsProvider, -) util.JSONResponse { - var request PublicRoomReq - if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { - return *fillErr - } - response, err := publicRooms(req.Context(), request, publicRoomDatabase) - if err != nil { - return jsonerror.InternalServerError() - } - - if request.Since != "" { - // TODO: handle pagination tokens sensibly rather than ignoring them. - // ignore paginated requests since we don't handle them yet over federation. - // Only the initial request will contain federated rooms. - return util.JSONResponse{ - Code: http.StatusOK, - JSON: response, - } - } - - // If we have already hit the limit on the number of rooms, bail. - var limit int - if request.Limit > 0 { - limit = int(request.Limit) - len(response.Chunk) - if limit <= 0 { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: response, - } - } - } - - // downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16 - fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit)) - response.Chunk = append(response.Chunk, fedRooms...) - - // de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers - // are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit) - var publicRooms []gomatrixserverlib.PublicRoom - haveRoomIDs := make(map[string]bool) - rand.Shuffle(len(response.Chunk), func(i, j int) { - response.Chunk[i], response.Chunk[j] = response.Chunk[j], response.Chunk[i] - }) - for _, r := range response.Chunk { - if haveRoomIDs[r.RoomID] { - continue - } - haveRoomIDs[r.RoomID] = true - publicRooms = append(publicRooms, r) - } - // sort by member count - sort.SliceStable(publicRooms, func(i, j int) bool { - return publicRooms[i].JoinedMembersCount > publicRooms[j].JoinedMembersCount - }) - - response.Chunk = publicRooms - - return util.JSONResponse{ - Code: http.StatusOK, - JSON: response, - } -} - -// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers. -// Returns a list of public rooms up to the limit specified. -func bulkFetchPublicRoomsFromServers( - ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string, limit int16, -) (publicRooms []gomatrixserverlib.PublicRoom) { - // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. - // goroutines send rooms to this channel - roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit)) - // signalling channel to tell goroutines to stop sending rooms and quit - done := make(chan bool) - // signalling to say when we can close the room channel - var wg sync.WaitGroup - wg.Add(len(homeservers)) - // concurrently query for public rooms - for _, hs := range homeservers { - go func(homeserverDomain string) { - defer wg.Done() - util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms") - fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "") - if err != nil { - util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn( - "bulkFetchPublicRoomsFromServers: failed to query hs", - ) - return - } - for _, room := range fres.Chunk { - // atomically send a room or stop - select { - case roomCh <- room: - case <-done: - util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms") - return - } - } - }(hs) - } - - // Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request. - // This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be - // closed. - go func() { - wg.Wait() - util.GetLogger(ctx).Info("Cleaning up resources") - close(roomCh) - }() - - // fan-in results with timeout. We stop when we reach the limit. -FanIn: - for len(publicRooms) < int(limit) || limit == 0 { - // add a room or timeout - select { - case room, ok := <-roomCh: - if !ok { - util.GetLogger(ctx).Info("All homeservers have been queried, returning results.") - break FanIn - } - publicRooms = append(publicRooms, room) - case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got. - util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early") - break FanIn - case <-ctx.Done(): // the client hung up on us, let's stop. - util.GetLogger(ctx).Info("Client hung up, returning early") - break FanIn - } - } - // tell goroutines to stop - close(done) - - return publicRooms -} - -func publicRooms(ctx context.Context, request PublicRoomReq, publicRoomDatabase storage.Database) (*gomatrixserverlib.RespPublicRooms, error) { - var response gomatrixserverlib.RespPublicRooms - var limit int16 - var offset int64 - limit = request.Limit - offset, err := strconv.ParseInt(request.Since, 10, 64) - // ParseInt returns 0 and an error when trying to parse an empty string - // In that case, we want to assign 0 so we ignore the error - if err != nil && len(request.Since) > 0 { - util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed") - return nil, err - } - - est, err := publicRoomDatabase.CountPublicRooms(ctx) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed") - return nil, err - } - response.TotalRoomCountEstimate = int(est) - - if offset > 0 { - response.PrevBatch = strconv.Itoa(int(offset) - 1) - } - nextIndex := int(offset) + int(limit) - if response.TotalRoomCountEstimate > nextIndex { - response.NextBatch = strconv.Itoa(nextIndex) - } - - if response.Chunk, err = publicRoomDatabase.GetPublicRooms( - ctx, offset, limit, request.Filter.SearchTerms, - ); err != nil { - util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed") - return nil, err - } - - return &response, nil -} - -// fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request -// on /publicRooms by parsing the incoming HTTP request -// Filter is only filled for POST requests -func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse { - if httpReq.Method == http.MethodGet { - limit, err := strconv.Atoi(httpReq.FormValue("limit")) - // Atoi returns 0 and an error when trying to parse an empty string - // In that case, we want to assign 0 so we ignore the error - if err != nil && len(httpReq.FormValue("limit")) > 0 { - util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed") - reqErr := jsonerror.InternalServerError() - return &reqErr - } - request.Limit = int16(limit) - request.Since = httpReq.FormValue("since") - return nil - } else if httpReq.Method == http.MethodPost { - return httputil.UnmarshalJSONRequest(httpReq, request) - } - - return &util.JSONResponse{ - Code: http.StatusMethodNotAllowed, - JSON: jsonerror.NotFound("Bad method"), - } -} diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go deleted file mode 100644 index b9baa105..00000000 --- a/publicroomsapi/publicroomsapi.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package publicroomsapi - -import ( - "github.com/Shopify/sarama" - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/publicroomsapi/consumers" - "github.com/matrix-org/dendrite/publicroomsapi/routing" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/publicroomsapi/types" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" -) - -// AddPublicRoutes sets up and registers HTTP handlers for the PublicRoomsAPI -// component. -func AddPublicRoutes( - router *mux.Router, - cfg *config.Dendrite, - consumer sarama.Consumer, - userAPI userapi.UserInternalAPI, - publicRoomsDB storage.Database, - rsAPI roomserverAPI.RoomserverInternalAPI, - fedClient *gomatrixserverlib.FederationClient, - extRoomsProvider types.ExternalPublicRoomsProvider, -) { - rsConsumer := consumers.NewOutputRoomEventConsumer( - cfg, consumer, publicRoomsDB, rsAPI, - ) - if err := rsConsumer.Start(); err != nil { - logrus.WithError(err).Panic("failed to start public rooms server consumer") - } - - routing.Setup(router, userAPI, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) -} diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go deleted file mode 100644 index 9c82d350..00000000 --- a/publicroomsapi/routing/routing.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package routing - -import ( - "net/http" - - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/roomserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" - - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/publicroomsapi/directory" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/publicroomsapi/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -const pathPrefixR0 = "/client/r0" - -// Setup configures the given mux with publicroomsapi server listeners -// -// Due to Setup being used to call many other functions, a gocyclo nolint is -// applied: -// nolint: gocyclo -func Setup( - publicAPIMux *mux.Router, userAPI userapi.UserInternalAPI, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, - fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, -) { - r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() - - r0mux.Handle("/directory/list/room/{roomID}", - httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return directory.GetVisibility(req, publicRoomsDB, vars["roomID"]) - }), - ).Methods(http.MethodGet, http.MethodOptions) - // TODO: Add AS support - r0mux.Handle("/directory/list/room/{roomID}", - httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return directory.SetVisibility(req, publicRoomsDB, rsAPI, device, vars["roomID"]) - }), - ).Methods(http.MethodPut, http.MethodOptions) - r0mux.Handle("/publicRooms", - httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { - if extRoomsProvider != nil { - return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider) - } - return directory.GetPostPublicRooms(req, publicRoomsDB) - }), - ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) - - // Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is. - publicAPIMux.Handle("/federation/v1/publicRooms", - httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { - return directory.GetPostPublicRooms(req, publicRoomsDB) - }), - ).Methods(http.MethodGet) -} diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go deleted file mode 100644 index 0ca6f455..00000000 --- a/publicroomsapi/storage/interface.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "context" - - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/gomatrixserverlib" -) - -type Database interface { - internal.PartitionStorer - GetRoomVisibility(ctx context.Context, roomID string) (bool, error) - SetRoomVisibility(ctx context.Context, visible bool, roomID string) error - CountPublicRooms(ctx context.Context) (int64, error) - GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) - UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error - UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error -} diff --git a/publicroomsapi/storage/postgres/prepare.go b/publicroomsapi/storage/postgres/prepare.go deleted file mode 100644 index 70b6e516..00000000 --- a/publicroomsapi/storage/postgres/prepare.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "database/sql" -) - -// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. -type statementList []struct { - statement **sql.Stmt - sql string -} - -// prepare the SQL for each statement in the list and assign the result to the prepared statement. -func (s statementList) prepare(db *sql.DB) (err error) { - for _, statement := range s { - if *statement.statement, err = db.Prepare(statement.sql); err != nil { - return - } - } - return -} diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go deleted file mode 100644 index 39e35536..00000000 --- a/publicroomsapi/storage/postgres/public_rooms_table.go +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - "errors" - "fmt" - - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/gomatrixserverlib" - - "github.com/lib/pq" -) - -var editableAttributes = []string{ - "aliases", - "canonical_alias", - "name", - "topic", - "world_readable", - "guest_can_join", - "avatar_url", - "visibility", -} - -const publicRoomsSchema = ` --- Stores all of the rooms with data needed to create the server's room directory -CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms( - -- The room's ID - room_id TEXT NOT NULL PRIMARY KEY, - -- Number of joined members in the room - joined_members INTEGER NOT NULL DEFAULT 0, - -- Aliases of the room (empty array if none) - aliases TEXT[] NOT NULL DEFAULT '{}'::TEXT[], - -- Canonical alias of the room (empty string if none) - canonical_alias TEXT NOT NULL DEFAULT '', - -- Name of the room (empty string if none) - name TEXT NOT NULL DEFAULT '', - -- Topic of the room (empty string if none) - topic TEXT NOT NULL DEFAULT '', - -- Is the room world readable? - world_readable BOOLEAN NOT NULL DEFAULT false, - -- Can guest join the room? - guest_can_join BOOLEAN NOT NULL DEFAULT false, - -- URL of the room avatar (empty string if none) - avatar_url TEXT NOT NULL DEFAULT '', - -- Visibility of the room: true means the room is publicly visible, false - -- means the room is private - visibility BOOLEAN NOT NULL DEFAULT false -); -` - -const countPublicRoomsSQL = "" + - "SELECT COUNT(*) FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" - -const selectPublicRoomsSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms WHERE visibility = true" + - " ORDER BY joined_members DESC" + - " OFFSET $1" - -const selectPublicRoomsWithLimitSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms WHERE visibility = true" + - " ORDER BY joined_members DESC" + - " OFFSET $1 LIMIT $2" - -const selectPublicRoomsWithFilterSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" + - " AND (LOWER(name) LIKE LOWER($1)" + - " OR LOWER(topic) LIKE LOWER($1)" + - " OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" + - " ORDER BY joined_members DESC" + - " OFFSET $2" - -const selectPublicRoomsWithLimitAndFilterSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" + - " AND (LOWER(name) LIKE LOWER($1)" + - " OR LOWER(topic) LIKE LOWER($1)" + - " OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" + - " ORDER BY joined_members DESC" + - " OFFSET $2 LIMIT $3" - -const selectRoomVisibilitySQL = "" + - "SELECT visibility FROM publicroomsapi_public_rooms" + - " WHERE room_id = $1" - -const insertNewRoomSQL = "" + - "INSERT INTO publicroomsapi_public_rooms(room_id)" + - " VALUES ($1)" - -const incrementJoinedMembersInRoomSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET joined_members = joined_members + 1" + - " WHERE room_id = $1" - -const decrementJoinedMembersInRoomSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET joined_members = joined_members - 1" + - " WHERE room_id = $1" - -const updateRoomAttributeSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET %s = $1" + - " WHERE room_id = $2" - -type publicRoomsStatements struct { - countPublicRoomsStmt *sql.Stmt - selectPublicRoomsStmt *sql.Stmt - selectPublicRoomsWithLimitStmt *sql.Stmt - selectPublicRoomsWithFilterStmt *sql.Stmt - selectPublicRoomsWithLimitAndFilterStmt *sql.Stmt - selectRoomVisibilityStmt *sql.Stmt - insertNewRoomStmt *sql.Stmt - incrementJoinedMembersInRoomStmt *sql.Stmt - decrementJoinedMembersInRoomStmt *sql.Stmt - updateRoomAttributeStmts map[string]*sql.Stmt -} - -func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(publicRoomsSchema) - if err != nil { - return - } - - stmts := statementList{ - {&s.countPublicRoomsStmt, countPublicRoomsSQL}, - {&s.selectPublicRoomsStmt, selectPublicRoomsSQL}, - {&s.selectPublicRoomsWithLimitStmt, selectPublicRoomsWithLimitSQL}, - {&s.selectPublicRoomsWithFilterStmt, selectPublicRoomsWithFilterSQL}, - {&s.selectPublicRoomsWithLimitAndFilterStmt, selectPublicRoomsWithLimitAndFilterSQL}, - {&s.selectRoomVisibilityStmt, selectRoomVisibilitySQL}, - {&s.insertNewRoomStmt, insertNewRoomSQL}, - {&s.incrementJoinedMembersInRoomStmt, incrementJoinedMembersInRoomSQL}, - {&s.decrementJoinedMembersInRoomStmt, decrementJoinedMembersInRoomSQL}, - } - - if err = stmts.prepare(db); err != nil { - return - } - - s.updateRoomAttributeStmts = make(map[string]*sql.Stmt) - for _, editable := range editableAttributes { - stmt := fmt.Sprintf(updateRoomAttributeSQL, editable) - if s.updateRoomAttributeStmts[editable], err = db.Prepare(stmt); err != nil { - return - } - } - - return -} - -func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, err error) { - err = s.countPublicRoomsStmt.QueryRowContext(ctx).Scan(&nb) - return -} - -func (s *publicRoomsStatements) selectPublicRooms( - ctx context.Context, offset int64, limit int16, filter string, -) ([]gomatrixserverlib.PublicRoom, error) { - var rows *sql.Rows - var err error - - if len(filter) > 0 { - pattern := "%" + filter + "%" - if limit == 0 { - rows, err = s.selectPublicRoomsWithFilterStmt.QueryContext( - ctx, pattern, offset, - ) - } else { - rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext( - ctx, pattern, offset, limit, - ) - } - } else { - if limit == 0 { - rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset) - } else { - rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext( - ctx, offset, limit, - ) - } - } - - if err != nil { - return []gomatrixserverlib.PublicRoom{}, nil - } - defer internal.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed") - - rooms := []gomatrixserverlib.PublicRoom{} - for rows.Next() { - var r gomatrixserverlib.PublicRoom - var aliases pq.StringArray - - err = rows.Scan( - &r.RoomID, &r.JoinedMembersCount, &aliases, &r.CanonicalAlias, - &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, - ) - if err != nil { - return rooms, err - } - - r.Aliases = aliases - - rooms = append(rooms, r) - } - - return rooms, rows.Err() -} - -func (s *publicRoomsStatements) selectRoomVisibility( - ctx context.Context, roomID string, -) (v bool, err error) { - err = s.selectRoomVisibilityStmt.QueryRowContext(ctx, roomID).Scan(&v) - return -} - -func (s *publicRoomsStatements) insertNewRoom( - ctx context.Context, roomID string, -) error { - _, err := s.insertNewRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) incrementJoinedMembersInRoom( - ctx context.Context, roomID string, -) error { - _, err := s.incrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) decrementJoinedMembersInRoom( - ctx context.Context, roomID string, -) error { - _, err := s.decrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) updateRoomAttribute( - ctx context.Context, attrName string, attrValue attributeValue, roomID string, -) error { - stmt, isEditable := s.updateRoomAttributeStmts[attrName] - - if !isEditable { - return errors.New("Cannot edit " + attrName) - } - - var value interface{} - switch v := attrValue.(type) { - case []string: - value = pq.StringArray(v) - case bool, string: - value = attrValue - default: - return errors.New("Unsupported attribute type, must be bool, string or []string") - } - - _, err := stmt.ExecContext(ctx, value, roomID) - return err -} diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go deleted file mode 100644 index 36c6aec6..00000000 --- a/publicroomsapi/storage/postgres/storage.go +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - "encoding/json" - - "github.com/matrix-org/dendrite/internal/eventutil" - "github.com/matrix-org/dendrite/internal/sqlutil" - - "github.com/matrix-org/gomatrixserverlib" -) - -// PublicRoomsServerDatabase represents a public rooms server database. -type PublicRoomsServerDatabase struct { - db *sql.DB - sqlutil.PartitionOffsetStatements - statements publicRoomsStatements - localServerName gomatrixserverlib.ServerName -} - -type attributeValue interface{} - -// NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { - var db *sql.DB - var err error - if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { - return nil, err - } - storage := PublicRoomsServerDatabase{ - db: db, - localServerName: localServerName, - } - if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil { - return nil, err - } - if err = storage.statements.prepare(db); err != nil { - return nil, err - } - return &storage, nil -} - -// GetRoomVisibility returns the room visibility as a boolean: true if the room -// is publicly visible, false if not. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) GetRoomVisibility( - ctx context.Context, roomID string, -) (bool, error) { - return d.statements.selectRoomVisibility(ctx, roomID) -} - -// SetRoomVisibility updates the visibility attribute of a room. This attribute -// must be set to true if the room is publicly visible, false if not. -// Returns an error if the update failed. -func (d *PublicRoomsServerDatabase) SetRoomVisibility( - ctx context.Context, visible bool, roomID string, -) error { - return d.statements.updateRoomAttribute(ctx, "visibility", visible, roomID) -} - -// CountPublicRooms returns the number of room set as publicly visible on the server. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { - return d.statements.countPublicRooms(ctx) -} - -// GetPublicRooms returns an array containing the local rooms set as publicly visible, ordered by their number -// of joined members. This array can be limited by a given number of elements, and offset by a given value. -// If the limit is 0, doesn't limit the number of results. If the offset is 0 too, the array contains all -// the rooms set as publicly visible on the server. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) GetPublicRooms( - ctx context.Context, offset int64, limit int16, filter string, -) ([]gomatrixserverlib.PublicRoom, error) { - return d.statements.selectPublicRooms(ctx, offset, limit, filter) -} - -// UpdateRoomFromEvents iterate over a slice of state events and call -// UpdateRoomFromEvent on each of them to update the database representation of -// the rooms updated by each event. -// The slice of events to remove is used to update the number of joined members -// for the room in the database. -// If the update triggered by one of the events failed, aborts the process and -// returns an error. -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents( - ctx context.Context, - eventsToAdd []gomatrixserverlib.Event, - eventsToRemove []gomatrixserverlib.Event, -) error { - for _, event := range eventsToAdd { - if err := d.UpdateRoomFromEvent(ctx, event); err != nil { - return err - } - } - - for _, event := range eventsToRemove { - if event.Type() == "m.room.member" { - if err := d.updateNumJoinedUsers(ctx, event, true); err != nil { - return err - } - } - } - - return nil -} - -// UpdateRoomFromEvent updates the database representation of a room from a Matrix event, by -// checking the event's type to know which attribute to change and using the event's content -// to define the new value of the attribute. -// If the event doesn't match with any property used to compute the public room directory, -// does nothing. -// If something went wrong during the process, returns an error. -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( - ctx context.Context, event gomatrixserverlib.Event, -) error { - // Process the event according to its type - switch event.Type() { - case "m.room.create": - return d.statements.insertNewRoom(ctx, event.RoomID()) - case "m.room.member": - return d.updateNumJoinedUsers(ctx, event, false) - case "m.room.aliases": - return d.updateRoomAliases(ctx, event) - case "m.room.canonical_alias": - var content eventutil.CanonicalAliasContent - field := &(content.Alias) - attrName := "canonical_alias" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.name": - var content eventutil.NameContent - field := &(content.Name) - attrName := "name" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.topic": - var content eventutil.TopicContent - field := &(content.Topic) - attrName := "topic" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.avatar": - var content eventutil.AvatarContent - field := &(content.URL) - attrName := "avatar_url" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.history_visibility": - var content eventutil.HistoryVisibilityContent - field := &(content.HistoryVisibility) - attrName := "world_readable" - strForTrue := "world_readable" - return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) - case "m.room.guest_access": - var content eventutil.GuestAccessContent - field := &(content.GuestAccess) - attrName := "guest_can_join" - strForTrue := "can_join" - return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) - } - - // If the event type didn't match, return with no error - return nil -} - -// updateNumJoinedUsers updates the number of joined user in the database representation -// of a room using a given "m.room.member" Matrix event. -// If the membership property of the event isn't "join", ignores it and returs nil. -// If the remove parameter is set to false, increments the joined members counter in the -// database, if set to truem decrements it. -// Returns an error if the update failed. -func (d *PublicRoomsServerDatabase) updateNumJoinedUsers( - ctx context.Context, membershipEvent gomatrixserverlib.Event, remove bool, -) error { - membership, err := membershipEvent.Membership() - if err != nil { - return err - } - - if membership != gomatrixserverlib.Join { - return nil - } - - if remove { - return d.statements.decrementJoinedMembersInRoom(ctx, membershipEvent.RoomID()) - } - return d.statements.incrementJoinedMembersInRoom(ctx, membershipEvent.RoomID()) -} - -// updateStringAttribute updates a given string attribute in the database -// representation of a room using a given string data field from content of the -// Matrix event triggering the update. -// Returns an error if decoding the Matrix event's content or updating the attribute -// failed. -func (d *PublicRoomsServerDatabase) updateStringAttribute( - ctx context.Context, attrName string, event gomatrixserverlib.Event, - content interface{}, field *string, -) error { - if err := json.Unmarshal(event.Content(), content); err != nil { - return err - } - - return d.statements.updateRoomAttribute(ctx, attrName, *field, event.RoomID()) -} - -// updateBooleanAttribute updates a given boolean attribute in the database -// representation of a room using a given string data field from content of the -// Matrix event triggering the update. -// The attribute is set to true if the field matches a given string, false if not. -// Returns an error if decoding the Matrix event's content or updating the attribute -// failed. -func (d *PublicRoomsServerDatabase) updateBooleanAttribute( - ctx context.Context, attrName string, event gomatrixserverlib.Event, - content interface{}, field *string, strForTrue string, -) error { - if err := json.Unmarshal(event.Content(), content); err != nil { - return err - } - - var attrValue bool - if *field == strForTrue { - attrValue = true - } else { - attrValue = false - } - - return d.statements.updateRoomAttribute(ctx, attrName, attrValue, event.RoomID()) -} - -// updateRoomAliases decodes the content of a "m.room.aliases" Matrix event and update the list of aliases of -// a given room with it. -// Returns an error if decoding the Matrix event or updating the list failed. -func (d *PublicRoomsServerDatabase) updateRoomAliases( - ctx context.Context, aliasesEvent gomatrixserverlib.Event, -) error { - if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) { - return nil // only store our own aliases - } - var content eventutil.AliasesContent - if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { - return err - } - - return d.statements.updateRoomAttribute( - ctx, "aliases", content.Aliases, aliasesEvent.RoomID(), - ) -} diff --git a/publicroomsapi/storage/sqlite3/prepare.go b/publicroomsapi/storage/sqlite3/prepare.go deleted file mode 100644 index 482dfa2b..00000000 --- a/publicroomsapi/storage/sqlite3/prepare.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "database/sql" -) - -// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. -type statementList []struct { - statement **sql.Stmt - sql string -} - -// prepare the SQL for each statement in the list and assign the result to the prepared statement. -func (s statementList) prepare(db *sql.DB) (err error) { - for _, statement := range s { - if *statement.statement, err = db.Prepare(statement.sql); err != nil { - return - } - } - return -} diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go deleted file mode 100644 index 7b332e17..00000000 --- a/publicroomsapi/storage/sqlite3/public_rooms_table.go +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/gomatrixserverlib" -) - -var editableAttributes = []string{ - "aliases", - "canonical_alias", - "name", - "topic", - "world_readable", - "guest_can_join", - "avatar_url", - "visibility", -} - -const publicRoomsSchema = ` --- Stores all of the rooms with data needed to create the server's room directory -CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms( - room_id TEXT NOT NULL PRIMARY KEY, - joined_members INTEGER NOT NULL DEFAULT 0, - aliases TEXT NOT NULL DEFAULT '', - canonical_alias TEXT NOT NULL DEFAULT '', - name TEXT NOT NULL DEFAULT '', - topic TEXT NOT NULL DEFAULT '', - world_readable BOOLEAN NOT NULL DEFAULT false, - guest_can_join BOOLEAN NOT NULL DEFAULT false, - avatar_url TEXT NOT NULL DEFAULT '', - visibility BOOLEAN NOT NULL DEFAULT false -); -` - -const countPublicRoomsSQL = "" + - "SELECT COUNT(*) FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" - -const selectPublicRoomsSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms WHERE visibility = true" + - " ORDER BY joined_members DESC" + - " LIMIT 30 OFFSET $1" - -const selectPublicRoomsWithLimitSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms WHERE visibility = true" + - " ORDER BY joined_members DESC" + - " LIMIT $1 OFFSET $2" - -const selectPublicRoomsWithFilterSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" + - " AND (LOWER(name) LIKE LOWER($1)" + - " OR LOWER(topic) LIKE LOWER($1)" + - " OR LOWER(aliases) LIKE LOWER($1))" + // TODO: Is there a better way to search aliases? - " ORDER BY joined_members DESC" + - " LIMIT 30 OFFSET $2" - -const selectPublicRoomsWithLimitAndFilterSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" + - " AND (LOWER(name) LIKE LOWER($1)" + - " OR LOWER(topic) LIKE LOWER($1)" + - " OR LOWER(aliases) LIKE LOWER($1))" + // TODO: Is there a better way to search aliases? - " ORDER BY joined_members DESC" + - " LIMIT $3 OFFSET $2" - -const selectRoomVisibilitySQL = "" + - "SELECT visibility FROM publicroomsapi_public_rooms" + - " WHERE room_id = $1" - -const insertNewRoomSQL = "" + - "INSERT INTO publicroomsapi_public_rooms(room_id)" + - " VALUES ($1)" - -const incrementJoinedMembersInRoomSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET joined_members = joined_members + 1" + - " WHERE room_id = $1" - -const decrementJoinedMembersInRoomSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET joined_members = joined_members - 1" + - " WHERE room_id = $1" - -const updateRoomAttributeSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET %s = $1" + - " WHERE room_id = $2" - -type publicRoomsStatements struct { - countPublicRoomsStmt *sql.Stmt - selectPublicRoomsStmt *sql.Stmt - selectPublicRoomsWithLimitStmt *sql.Stmt - selectPublicRoomsWithFilterStmt *sql.Stmt - selectPublicRoomsWithLimitAndFilterStmt *sql.Stmt - selectRoomVisibilityStmt *sql.Stmt - insertNewRoomStmt *sql.Stmt - incrementJoinedMembersInRoomStmt *sql.Stmt - decrementJoinedMembersInRoomStmt *sql.Stmt - updateRoomAttributeStmts map[string]*sql.Stmt -} - -func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(publicRoomsSchema) - if err != nil { - return - } - - stmts := statementList{ - {&s.countPublicRoomsStmt, countPublicRoomsSQL}, - {&s.selectPublicRoomsStmt, selectPublicRoomsSQL}, - {&s.selectPublicRoomsWithLimitStmt, selectPublicRoomsWithLimitSQL}, - {&s.selectPublicRoomsWithFilterStmt, selectPublicRoomsWithFilterSQL}, - {&s.selectPublicRoomsWithLimitAndFilterStmt, selectPublicRoomsWithLimitAndFilterSQL}, - {&s.selectRoomVisibilityStmt, selectRoomVisibilitySQL}, - {&s.insertNewRoomStmt, insertNewRoomSQL}, - {&s.incrementJoinedMembersInRoomStmt, incrementJoinedMembersInRoomSQL}, - {&s.decrementJoinedMembersInRoomStmt, decrementJoinedMembersInRoomSQL}, - } - - if err = stmts.prepare(db); err != nil { - return - } - - s.updateRoomAttributeStmts = make(map[string]*sql.Stmt) - for _, editable := range editableAttributes { - stmt := fmt.Sprintf(updateRoomAttributeSQL, editable) - if s.updateRoomAttributeStmts[editable], err = db.Prepare(stmt); err != nil { - return - } - } - - return -} - -func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, err error) { - err = s.countPublicRoomsStmt.QueryRowContext(ctx).Scan(&nb) - return -} - -func (s *publicRoomsStatements) selectPublicRooms( - ctx context.Context, offset int64, limit int16, filter string, -) ([]gomatrixserverlib.PublicRoom, error) { - var rows *sql.Rows - var err error - - if len(filter) > 0 { - pattern := "%" + filter + "%" - if limit == 0 { - rows, err = s.selectPublicRoomsWithFilterStmt.QueryContext( - ctx, pattern, offset, - ) - } else { - rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext( - ctx, pattern, limit, offset, - ) - } - } else { - if limit == 0 { - rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset) - } else { - rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext( - ctx, limit, offset, - ) - } - } - - if err != nil { - return []gomatrixserverlib.PublicRoom{}, nil - } - defer internal.CloseAndLogIfError(ctx, rows, "selectPublicRooms failed to close rows") - - rooms := []gomatrixserverlib.PublicRoom{} - for rows.Next() { - var r gomatrixserverlib.PublicRoom - var aliasesJSON string - - err = rows.Scan( - &r.RoomID, &r.JoinedMembersCount, &aliasesJSON, &r.CanonicalAlias, - &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, - ) - if err != nil { - return rooms, err - } - - if len(aliasesJSON) > 0 { - if err := json.Unmarshal([]byte(aliasesJSON), &r.Aliases); err != nil { - return rooms, err - } - } - - rooms = append(rooms, r) - } - - return rooms, nil -} - -func (s *publicRoomsStatements) selectRoomVisibility( - ctx context.Context, roomID string, -) (v bool, err error) { - err = s.selectRoomVisibilityStmt.QueryRowContext(ctx, roomID).Scan(&v) - return -} - -func (s *publicRoomsStatements) insertNewRoom( - ctx context.Context, roomID string, -) error { - _, err := s.insertNewRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) incrementJoinedMembersInRoom( - ctx context.Context, roomID string, -) error { - _, err := s.incrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) decrementJoinedMembersInRoom( - ctx context.Context, roomID string, -) error { - _, err := s.decrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) updateRoomAttribute( - ctx context.Context, attrName string, attrValue attributeValue, roomID string, -) error { - stmt, isEditable := s.updateRoomAttributeStmts[attrName] - - if !isEditable { - return errors.New("Cannot edit " + attrName) - } - - var value interface{} - switch v := attrValue.(type) { - case []string: - b, _ := json.Marshal(v) - value = string(b) - case bool, string: - value = attrValue - default: - return errors.New("Unsupported attribute type, must be bool, string or []string") - } - - _, err := stmt.ExecContext(ctx, value, roomID) - return err -} diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go deleted file mode 100644 index 5c685d13..00000000 --- a/publicroomsapi/storage/sqlite3/storage.go +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - "encoding/json" - - _ "github.com/mattn/go-sqlite3" - - "github.com/matrix-org/dendrite/internal/eventutil" - "github.com/matrix-org/dendrite/internal/sqlutil" - - "github.com/matrix-org/gomatrixserverlib" -) - -// PublicRoomsServerDatabase represents a public rooms server database. -type PublicRoomsServerDatabase struct { - db *sql.DB - sqlutil.PartitionOffsetStatements - statements publicRoomsStatements - localServerName gomatrixserverlib.ServerName -} - -type attributeValue interface{} - -// NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { - var db *sql.DB - var err error - cs, err := sqlutil.ParseFileURI(dataSourceName) - if err != nil { - return nil, err - } - if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { - return nil, err - } - storage := PublicRoomsServerDatabase{ - db: db, - localServerName: localServerName, - } - if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil { - return nil, err - } - if err = storage.statements.prepare(db); err != nil { - return nil, err - } - return &storage, nil -} - -// GetRoomVisibility returns the room visibility as a boolean: true if the room -// is publicly visible, false if not. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) GetRoomVisibility( - ctx context.Context, roomID string, -) (bool, error) { - return d.statements.selectRoomVisibility(ctx, roomID) -} - -// SetRoomVisibility updates the visibility attribute of a room. This attribute -// must be set to true if the room is publicly visible, false if not. -// Returns an error if the update failed. -func (d *PublicRoomsServerDatabase) SetRoomVisibility( - ctx context.Context, visible bool, roomID string, -) error { - return d.statements.updateRoomAttribute(ctx, "visibility", visible, roomID) -} - -// CountPublicRooms returns the number of room set as publicly visible on the server. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { - return d.statements.countPublicRooms(ctx) -} - -// GetPublicRooms returns an array containing the local rooms set as publicly visible, ordered by their number -// of joined members. This array can be limited by a given number of elements, and offset by a given value. -// If the limit is 0, doesn't limit the number of results. If the offset is 0 too, the array contains all -// the rooms set as publicly visible on the server. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) GetPublicRooms( - ctx context.Context, offset int64, limit int16, filter string, -) ([]gomatrixserverlib.PublicRoom, error) { - return d.statements.selectPublicRooms(ctx, offset, limit, filter) -} - -// UpdateRoomFromEvents iterate over a slice of state events and call -// UpdateRoomFromEvent on each of them to update the database representation of -// the rooms updated by each event. -// The slice of events to remove is used to update the number of joined members -// for the room in the database. -// If the update triggered by one of the events failed, aborts the process and -// returns an error. -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents( - ctx context.Context, - eventsToAdd []gomatrixserverlib.Event, - eventsToRemove []gomatrixserverlib.Event, -) error { - for _, event := range eventsToAdd { - if err := d.UpdateRoomFromEvent(ctx, event); err != nil { - return err - } - } - - for _, event := range eventsToRemove { - if event.Type() == "m.room.member" { - if err := d.updateNumJoinedUsers(ctx, event, true); err != nil { - return err - } - } - } - - return nil -} - -// UpdateRoomFromEvent updates the database representation of a room from a Matrix event, by -// checking the event's type to know which attribute to change and using the event's content -// to define the new value of the attribute. -// If the event doesn't match with any property used to compute the public room directory, -// does nothing. -// If something went wrong during the process, returns an error. -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( - ctx context.Context, event gomatrixserverlib.Event, -) error { - // Process the event according to its type - switch event.Type() { - case "m.room.create": - return d.statements.insertNewRoom(ctx, event.RoomID()) - case "m.room.member": - return d.updateNumJoinedUsers(ctx, event, false) - case "m.room.aliases": - return d.updateRoomAliases(ctx, event) - case "m.room.canonical_alias": - var content eventutil.CanonicalAliasContent - field := &(content.Alias) - attrName := "canonical_alias" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.name": - var content eventutil.NameContent - field := &(content.Name) - attrName := "name" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.topic": - var content eventutil.TopicContent - field := &(content.Topic) - attrName := "topic" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.avatar": - var content eventutil.AvatarContent - field := &(content.URL) - attrName := "avatar_url" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.history_visibility": - var content eventutil.HistoryVisibilityContent - field := &(content.HistoryVisibility) - attrName := "world_readable" - strForTrue := "world_readable" - return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) - case "m.room.guest_access": - var content eventutil.GuestAccessContent - field := &(content.GuestAccess) - attrName := "guest_can_join" - strForTrue := "can_join" - return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) - } - - // If the event type didn't match, return with no error - return nil -} - -// updateNumJoinedUsers updates the number of joined user in the database representation -// of a room using a given "m.room.member" Matrix event. -// If the membership property of the event isn't "join", ignores it and returs nil. -// If the remove parameter is set to false, increments the joined members counter in the -// database, if set to truem decrements it. -// Returns an error if the update failed. -func (d *PublicRoomsServerDatabase) updateNumJoinedUsers( - ctx context.Context, membershipEvent gomatrixserverlib.Event, remove bool, -) error { - membership, err := membershipEvent.Membership() - if err != nil { - return err - } - - if membership != gomatrixserverlib.Join { - return nil - } - - if remove { - return d.statements.decrementJoinedMembersInRoom(ctx, membershipEvent.RoomID()) - } - return d.statements.incrementJoinedMembersInRoom(ctx, membershipEvent.RoomID()) -} - -// updateStringAttribute updates a given string attribute in the database -// representation of a room using a given string data field from content of the -// Matrix event triggering the update. -// Returns an error if decoding the Matrix event's content or updating the attribute -// failed. -func (d *PublicRoomsServerDatabase) updateStringAttribute( - ctx context.Context, attrName string, event gomatrixserverlib.Event, - content interface{}, field *string, -) error { - if err := json.Unmarshal(event.Content(), content); err != nil { - return err - } - - return d.statements.updateRoomAttribute(ctx, attrName, *field, event.RoomID()) -} - -// updateBooleanAttribute updates a given boolean attribute in the database -// representation of a room using a given string data field from content of the -// Matrix event triggering the update. -// The attribute is set to true if the field matches a given string, false if not. -// Returns an error if decoding the Matrix event's content or updating the attribute -// failed. -func (d *PublicRoomsServerDatabase) updateBooleanAttribute( - ctx context.Context, attrName string, event gomatrixserverlib.Event, - content interface{}, field *string, strForTrue string, -) error { - if err := json.Unmarshal(event.Content(), content); err != nil { - return err - } - - var attrValue bool - if *field == strForTrue { - attrValue = true - } else { - attrValue = false - } - - return d.statements.updateRoomAttribute(ctx, attrName, attrValue, event.RoomID()) -} - -// updateRoomAliases decodes the content of a "m.room.aliases" Matrix event and update the list of aliases of -// a given room with it. -// Returns an error if decoding the Matrix event or updating the list failed. -func (d *PublicRoomsServerDatabase) updateRoomAliases( - ctx context.Context, aliasesEvent gomatrixserverlib.Event, -) error { - if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) { - return nil // only store our own aliases - } - var content eventutil.AliasesContent - if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { - return err - } - - return d.statements.updateRoomAttribute( - ctx, "aliases", content.Aliases, aliasesEvent.RoomID(), - ) -} diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go deleted file mode 100644 index f6618804..00000000 --- a/publicroomsapi/storage/storage.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !wasm - -package storage - -import ( - "net/url" - - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" - "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" -) - -const schemePostgres = "postgres" -const schemeFile = "file" - -// NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) - } - switch uri.Scheme { - case schemePostgres: - return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) - case schemeFile: - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) - default: - return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) - } -} diff --git a/publicroomsapi/storage/storage_wasm.go b/publicroomsapi/storage/storage_wasm.go deleted file mode 100644 index 70ceeaf8..00000000 --- a/publicroomsapi/storage/storage_wasm.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "fmt" - "net/url" - - "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" -) - -// NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, err - } - switch uri.Scheme { - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") - case "file": - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) - default: - return nil, fmt.Errorf("Cannot use postgres implementation") - } -}