Limit database connections (#980, #564) (#998)

* Limit database connections (#564)

- Add new options to the config file database:
      max_open_conns: 100
      max_idle_conns: 2
      conn_max_lifetime: -1
- Implement connection parameter setup on the *DB (database/sql) in internal/sqlutil/trace.go:Open()
- Propagate the values in the form of DbProperties interface via all the
  Open() and NewDatabase() functions

Signed-off-by: Tomas Jirka <tomas.jirka@email.cz>

* Fix wasm builds

* Remove file accidentally added from working tree

Co-authored-by: Tomas Jirka <tomas.jirka@email.cz>

appservice/appservice.go

@@ -48,7 +48,7 @@ func SetupAppServiceAPIComponent(
 	transactionsCache *transactions.Cache,
 ) appserviceAPI.AppServiceQueryAPI {
 	// Create a connection to the appservice postgres DB
-	appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService))
+	appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to appservice db")
 	}

appservice/storage/postgres/storage.go

@@ -21,6 +21,7 @@ import (
 
 	// Import postgres database driver
 	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -33,10 +34,10 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

appservice/storage/sqlite3/storage.go

@@ -37,7 +37,7 @@ type Database struct {
 func NewDatabase(dataSourceName string) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

appservice/storage/storage.go

@@ -21,19 +21,22 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/storage/postgres"
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 )
 
-func NewDatabase(dataSourceName string) (Database, error) {
+// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
+// and sets DB connection parameters
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName)
 	default:
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	}
 }

appservice/storage/storage_wasm.go

@@ -19,9 +19,13 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 )
 
-func NewDatabase(dataSourceName string) (Database, error) {
+func NewDatabase(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

clientapi/auth/storage/accounts/postgres/storage.go

@@ -44,10 +44,10 @@ type Database struct {
 }
 
 // NewDatabase creates a new accounts and profiles database
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	partitions := common.PartitionOffsetStatements{}

clientapi/auth/storage/accounts/sqlite3/storage.go

@@ -50,7 +50,7 @@ type Database struct {
 func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	partitions := common.PartitionOffsetStatements{}

clientapi/auth/storage/accounts/storage.go

@@ -21,20 +21,23 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
+// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
+// and sets postgres connection parameters
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName, serverName)
 	default:
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	}
 }

clientapi/auth/storage/accounts/storage_wasm.go

@@ -19,10 +19,15 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
+func NewDatabase(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+	serverName gomatrixserverlib.ServerName,
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

clientapi/auth/storage/devices/postgres/storage.go

@@ -36,10 +36,10 @@ type Database struct {
 }
 
 // NewDatabase creates a new device database
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	d := devicesStatements{}

clientapi/auth/storage/devices/sqlite3/storage.go

@@ -41,7 +41,7 @@ type Database struct {
 func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	d := devicesStatements{}

clientapi/auth/storage/devices/storage.go

@@ -21,20 +21,23 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
+// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
+// and sets postgres connection parameters
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName, serverName)
 	default:
-		return postgres.NewDatabase(dataSourceName, serverName)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
 	}
 }

clientapi/auth/storage/devices/storage_wasm.go

@@ -19,10 +19,15 @@ import (
 	"net/url"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
+func NewDatabase(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+	serverName gomatrixserverlib.ServerName,
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

cmd/create-account/main.go

@@ -63,7 +63,7 @@ func main() {
 
 	serverName := gomatrixserverlib.ServerName(*serverNameStr)
 
-	accountDB, err := accounts.NewDatabase(*database, serverName)
+	accountDB, err := accounts.NewDatabase(*database, nil, serverName)
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)
@@ -78,7 +78,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	deviceDB, err := devices.NewDatabase(*database, serverName)
+	deviceDB, err := devices.NewDatabase(*database, nil, serverName)
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)

cmd/dendrite-demo-libp2p/main.go

@@ -56,6 +56,7 @@ func createKeyDB(
 ) keydb.Database {
 	db, err := keydb.NewDatabase(
 		string(base.Base.Cfg.Database.ServerKey),
+		base.Base.Cfg.DbProperties(),
 		base.Base.Cfg.Matrix.ServerName,
 		base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
 		base.Base.Cfg.Matrix.KeyID,

cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go

@@ -45,7 +45,7 @@ type PublicRoomsServerDatabase struct {
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
 func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) {
-	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName)
+	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil)
 	if err != nil {
 		return nil, err
 	}

cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go

@@ -48,7 +48,7 @@ type PublicRoomsServerDatabase struct {
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
 func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) {
-	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName)
+	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil)
 	if err != nil {
 		return nil, err
 	}

cmd/dendrite-monolith-server/main.go

@@ -79,7 +79,7 @@ func main() {
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
 	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}

cmd/dendrite-public-rooms-api-server/main.go

@@ -32,7 +32,7 @@ func main() {
 	rsAPI := base.CreateHTTPRoomserverAPIs()
 	rsAPI.SetFederationSenderAPI(fsAPI)
 
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to public rooms db")
 	}

common/basecomponent/base.go

@@ -150,7 +150,7 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede
 // CreateDeviceDB creates a new instance of the device database. Should only be
 // called once per component.
 func (b *BaseDendrite) CreateDeviceDB() devices.Database {
-	db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.Matrix.ServerName)
+	db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to devices db")
 	}
@@ -161,7 +161,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database {
 // CreateAccountsDB creates a new instance of the accounts database. Should only
 // be called once per component.
 func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
-	db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.Matrix.ServerName)
+	db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to accounts db")
 	}
@@ -174,6 +174,7 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
 func (b *BaseDendrite) CreateKeyDB() keydb.Database {
 	db, err := keydb.NewDatabase(
 		string(b.Cfg.Database.ServerKey),
+		b.Cfg.DbProperties(),
 		b.Cfg.Matrix.ServerName,
 		b.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
 		b.Cfg.Matrix.KeyID,
@@ -244,7 +245,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
 
 	uri, err := url.Parse(string(cfg.Database.Naffka))
 	if err != nil || uri.Scheme == "file" {
-		db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka))
+		db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka), nil)
 		if err != nil {
 			logrus.WithError(err).Panic("Failed to open naffka database")
 		}
@@ -254,7 +255,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
 			logrus.WithError(err).Panic("Failed to setup naffka database")
 		}
 	} else {
-		db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka))
+		db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil)
 		if err != nil {
 			logrus.WithError(err).Panic("Failed to open naffka database")
 		}

common/config/config.go

@@ -188,6 +188,12 @@ type Dendrite struct {
 		PublicRoomsAPI DataSource `yaml:"public_rooms_api"`
 		// The Naffka database is used internally by the naffka library, if used.
 		Naffka DataSource `yaml:"naffka,omitempty"`
+		// Maximum open connections to the DB (0 = use default, negative means unlimited)
+		MaxOpenConns int `yaml:"max_open_conns"`
+		// Maximum idle connections to the DB (0 = use default, negative means unlimited)
+		MaxIdleConns int `yaml:"max_idle_conns"`
+		// maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited)
+		ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"`
 	} `yaml:"database"`
 
 	// TURN Server Config
@@ -484,6 +490,15 @@ func (config *Dendrite) SetDefaults() {
 		defaultMaxFileSizeBytes := FileSizeBytes(10485760)
 		config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes
 	}
+
+	if config.Database.MaxIdleConns == 0 {
+		config.Database.MaxIdleConns = 2
+	}
+
+	if config.Database.MaxOpenConns == 0 {
+		config.Database.MaxOpenConns = 100
+	}
+
 }
 
 // Error returns a string detailing how many errors were contained within a
@@ -746,6 +761,33 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err
 	)
 }
 
+// MaxIdleConns returns maximum idle connections to the DB
+func (config Dendrite) MaxIdleConns() int {
+	return config.Database.MaxIdleConns
+}
+
+// MaxOpenConns returns maximum open connections to the DB
+func (config Dendrite) MaxOpenConns() int {
+	return config.Database.MaxOpenConns
+}
+
+// ConnMaxLifetime returns maximum amount of time a connection may be reused
+func (config Dendrite) ConnMaxLifetime() time.Duration {
+	return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second
+}
+
+// DbProperties functions return properties used by database/sql/DB
+type DbProperties interface {
+	MaxIdleConns() int
+	MaxOpenConns() int
+	ConnMaxLifetime() time.Duration
+}
+
+// DbProperties returns cfg as a DbProperties interface
+func (config Dendrite) DbProperties() DbProperties {
+	return config
+}
+
 // logrusLogger is a small wrapper that implements jaeger.Logger using logrus.
 type logrusLogger struct {
 	l *logrus.Logger

common/keydb/keydb.go

@@ -21,6 +21,7 @@ import (
 
 	"golang.org/x/crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/keydb/postgres"
 	"github.com/matrix-org/dendrite/common/keydb/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -29,20 +30,21 @@ import (
 // NewDatabase opens a database connection.
 func NewDatabase(
 	dataSourceName string,
+	dbProperties common.DbProperties,
 	serverName gomatrixserverlib.ServerName,
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
 	default:
-		return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
+		return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
 	}
 }

common/keydb/keydb_wasm.go

@@ -20,6 +20,7 @@ import (
 
 	"golang.org/x/crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/keydb/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -27,6 +28,7 @@ import (
 // NewDatabase opens a database connection.
 func NewDatabase(
 	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
 	serverName gomatrixserverlib.ServerName,
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,

common/keydb/postgres/keydb.go

@@ -21,6 +21,7 @@ import (
 
 	"golang.org/x/crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -37,11 +38,12 @@ type Database struct {
 // Returns an error if there was a problem talking to the database.
 func NewDatabase(
 	dataSourceName string,
+	dbProperties common.DbProperties,
 	serverName gomatrixserverlib.ServerName,
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (*Database, error) {
-	db, err := sqlutil.Open("postgres", dataSourceName)
+	db, err := sqlutil.Open("postgres", dataSourceName, dbProperties)
 	if err != nil {
 		return nil, err
 	}

common/keydb/sqlite3/keydb.go

@@ -44,7 +44,7 @@ func NewDatabase(
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (*Database, error) {
-	db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName)
+	db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil)
 	if err != nil {
 		return nil, err
 	}

common/sql.go

@@ -18,6 +18,7 @@ import (
 	"database/sql"
 	"fmt"
 	"runtime"
+	"time"
 )
 
 // A Transaction is something that can be committed or rolledback.
@@ -99,3 +100,10 @@ func SQLiteDriverName() string {
 	}
 	return "sqlite3"
 }
+
+// DbProperties functions return properties used by database/sql/DB
+type DbProperties interface {
+	MaxIdleConns() int
+	MaxOpenConns() int
+	ConnMaxLifetime() time.Duration
+}

dendrite-config.yaml

@@ -116,6 +116,9 @@ database:
     federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable"
     appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable"
     public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable"
+    max_open_conns: 100
+    max_idle_conns: 2
+    conn_max_lifetime: -1
     # If using naffka you need to specify a naffka database
     # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable"
 

federationsender/federationsender.go

@@ -37,7 +37,7 @@ func SetupFederationSenderComponent(
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	keyRing *gomatrixserverlib.KeyRing,
 ) api.FederationSenderInternalAPI {
-	federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender))
+	federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panic("failed to connect to federation sender db")
 	}

federationsender/storage/postgres/storage.go

@@ -33,10 +33,10 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (*Database, error) {
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

federationsender/storage/sqlite3/storage.go

@@ -38,7 +38,7 @@ type Database struct {
 func NewDatabase(dataSourceName string) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

federationsender/storage/storage.go

@@ -19,22 +19,23 @@ package storage
 import (
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/federationsender/storage/postgres"
 	"github.com/matrix-org/dendrite/federationsender/storage/sqlite3"
 )
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (Database, error) {
+func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	}
 	switch uri.Scheme {
 	case "file":
 		return sqlite3.NewDatabase(dataSourceName)
 	case "postgres":
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	default:
-		return postgres.NewDatabase(dataSourceName)
+		return postgres.NewDatabase(dataSourceName, dbProperties)
 	}
 }

federationsender/storage/storage_wasm.go

@@ -18,11 +18,15 @@ import (
 	"fmt"
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/federationsender/storage/sqlite3"
 )
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (Database, error) {
+func NewDatabase(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

internal/sqlutil/trace.go

@@ -21,9 +21,11 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/ngrok/sqlmw"
 	"github.com/sirupsen/logrus"
 )
@@ -76,12 +78,27 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [
 // Open opens a database specified by its database driver name and a driver-specific data source name,
 // usually consisting of at least a database name and connection information. Includes tracing driver
 // if DENDRITE_TRACE_SQL=1
-func Open(driverName, dsn string) (*sql.DB, error) {
+func Open(driverName, dsn string, dbProperties common.DbProperties) (*sql.DB, error) {
 	if tracingEnabled {
 		// install the wrapped driver
 		driverName += "-trace"
 	}
-	return sql.Open(driverName, dsn)
+	db, err := sql.Open(driverName, dsn)
+	if err != nil {
+		return nil, err
+	}
+	if driverName != common.SQLiteDriverName() && dbProperties != nil {
+		logrus.WithFields(logrus.Fields{
+			"MaxOpenConns":    dbProperties.MaxOpenConns(),
+			"MaxIdleConns":    dbProperties.MaxIdleConns(),
+			"ConnMaxLifetime": dbProperties.ConnMaxLifetime(),
+			"dataSourceName":  regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"),
+		}).Debug("Setting DB connection limits")
+		db.SetMaxOpenConns(dbProperties.MaxOpenConns())
+		db.SetMaxIdleConns(dbProperties.MaxIdleConns())
+		db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime())
+	}
+	return db, nil
 }
 
 func init() {

mediaapi/mediaapi.go

@@ -29,7 +29,7 @@ func SetupMediaAPIComponent(
 	base *basecomponent.BaseDendrite,
 	deviceDB devices.Database,
 ) {
-	mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI))
+	mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to media db")
 	}

mediaapi/storage/postgres/storage.go

@@ -21,6 +21,7 @@ import (
 
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -33,10 +34,10 @@ type Database struct {
 }
 
 // Open opens a postgres database.
-func Open(dataSourceName string) (*Database, error) {
+func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	if err = d.statements.prepare(d.db); err != nil {

mediaapi/storage/sqlite3/storage.go

@@ -37,7 +37,7 @@ type Database struct {
 func Open(dataSourceName string) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	if err = d.statements.prepare(d.db); err != nil {

mediaapi/storage/storage.go

@@ -19,22 +19,23 @@ package storage
 import (
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/mediaapi/storage/postgres"
 	"github.com/matrix-org/dendrite/mediaapi/storage/sqlite3"
 )
 
 // Open opens a postgres database.
-func Open(dataSourceName string) (Database, error) {
+func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.Open(dataSourceName)
+		return postgres.Open(dataSourceName, dbProperties)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.Open(dataSourceName)
+		return postgres.Open(dataSourceName, dbProperties)
 	case "file":
 		return sqlite3.Open(dataSourceName)
 	default:
-		return postgres.Open(dataSourceName)
+		return postgres.Open(dataSourceName, dbProperties)
 	}
 }

mediaapi/storage/storage_wasm.go

@@ -18,11 +18,15 @@ import (
 	"fmt"
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/mediaapi/storage/sqlite3"
 )
 
 // Open opens a postgres database.
-func Open(dataSourceName string) (Database, error) {
+func Open(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

publicroomsapi/storage/postgres/storage.go

@@ -36,10 +36,10 @@ type PublicRoomsServerDatabase struct {
 type attributeValue interface{}
 
 // NewPublicRoomsServerDatabase creates a new public rooms server database.
-func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) {
+func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (*PublicRoomsServerDatabase, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	storage := PublicRoomsServerDatabase{

publicroomsapi/storage/sqlite3/storage.go

@@ -41,7 +41,7 @@ type attributeValue interface{}
 func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
 		return nil, err
 	}
 	storage := PublicRoomsServerDatabase{

publicroomsapi/storage/storage.go

@@ -19,6 +19,7 @@ package storage
 import (
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage/postgres"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
 )
@@ -27,17 +28,17 @@ const schemePostgres = "postgres"
 const schemeFile = "file"
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) {
+func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewPublicRoomsServerDatabase(dataSourceName)
+		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties)
 	}
 	switch uri.Scheme {
 	case schemePostgres:
-		return postgres.NewPublicRoomsServerDatabase(dataSourceName)
+		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties)
 	case schemeFile:
 		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
 	default:
-		return postgres.NewPublicRoomsServerDatabase(dataSourceName)
+		return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties)
 	}
 }

roomserver/roomserver.go

@@ -35,7 +35,7 @@ func SetupRoomServerComponent(
 	keyRing gomatrixserverlib.JSONVerifier,
 	fedClient *gomatrixserverlib.FederationClient,
 ) api.RoomserverInternalAPI {
-	roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer))
+	roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to room server db")
 	}

roomserver/storage/postgres/storage.go

@@ -20,6 +20,7 @@ import (
 	"database/sql"
 	"encoding/json"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 
 	// Import the postgres database driver.
@@ -36,10 +37,10 @@ type Database struct {
 }
 
 // Open a postgres database.
-func Open(dataSourceName string) (*Database, error) {
+func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
+	if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	if err = d.statements.prepare(d.db); err != nil {

roomserver/storage/sqlite3/storage.go

@@ -37,7 +37,7 @@ type Database struct {
 	db         *sql.DB
 }
 
-// Open a postgres database.
+// Open a sqlite database.
 func Open(dataSourceName string) (*Database, error) {
 	var d Database
 	uri, err := url.Parse(dataSourceName)
@@ -52,7 +52,7 @@ func Open(dataSourceName string) (*Database, error) {
 	} else {
 		return nil, errors.New("no filename or path in connect string")
 	}
-	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil {
+	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil {
 		return nil, err
 	}
 	//d.db.Exec("PRAGMA journal_mode=WAL;")

roomserver/storage/storage.go

@@ -19,22 +19,23 @@ package storage
 import (
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
 	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
 )
 
-// NewPublicRoomsServerDatabase opens a database connection.
-func Open(dataSourceName string) (Database, error) {
+// Open opens a database connection.
+func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.Open(dataSourceName)
+		return postgres.Open(dataSourceName, dbProperties)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.Open(dataSourceName)
+		return postgres.Open(dataSourceName, dbProperties)
 	case "file":
 		return sqlite3.Open(dataSourceName)
 	default:
-		return postgres.Open(dataSourceName)
+		return postgres.Open(dataSourceName, dbProperties)
 	}
 }

roomserver/storage/storage_wasm.go

@@ -18,11 +18,15 @@ import (
 	"fmt"
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
 )
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func Open(dataSourceName string) (Database, error) {
+func Open(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

syncapi/storage/postgres/syncserver.go

@@ -61,10 +61,10 @@ type SyncServerDatasource struct {
 }
 
 // NewSyncServerDatasource creates a new sync server database
-func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) {
+func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProperties) (*SyncServerDatasource, error) {
 	var d SyncServerDatasource
 	var err error
-	if d.db, err = sqlutil.Open("postgres", dbDataSourceName); err != nil {
+	if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil {
 		return nil, err
 	}
 	if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil {

syncapi/storage/sqlite3/syncserver.go

@@ -80,7 +80,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro
 	} else {
 		return nil, errors.New("no filename or path in connect string")
 	}
-	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil {
+	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil {
 		return nil, err
 	}
 	if err = d.prepare(); err != nil {

syncapi/storage/storage.go

@@ -19,22 +19,23 @@ package storage
 import (
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/syncapi/storage/postgres"
 	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
 )
 
-// NewPublicRoomsServerDatabase opens a database connection.
-func NewSyncServerDatasource(dataSourceName string) (Database, error) {
+// NewSyncServerDatasource opens a database connection.
+func NewSyncServerDatasource(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
-		return postgres.NewSyncServerDatasource(dataSourceName)
+		return postgres.NewSyncServerDatasource(dataSourceName, dbProperties)
 	}
 	switch uri.Scheme {
 	case "postgres":
-		return postgres.NewSyncServerDatasource(dataSourceName)
+		return postgres.NewSyncServerDatasource(dataSourceName, dbProperties)
 	case "file":
 		return sqlite3.NewSyncServerDatasource(dataSourceName)
 	default:
-		return postgres.NewSyncServerDatasource(dataSourceName)
+		return postgres.NewSyncServerDatasource(dataSourceName, dbProperties)
 	}
 }

syncapi/storage/storage_wasm.go

@@ -18,11 +18,15 @@ import (
 	"fmt"
 	"net/url"
 
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
 )
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func NewSyncServerDatasource(dataSourceName string) (Database, error) {
+func NewSyncServerDatasource(
+	dataSourceName string,
+	dbProperties common.DbProperties, // nolint:unparam
+) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("Cannot use postgres implementation")

syncapi/syncapi.go

@@ -42,7 +42,7 @@ func SetupSyncAPIComponent(
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.Dendrite,
 ) {
-	syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI))
+	syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI), base.Cfg.DbProperties())
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to sync db")
 	}