char/dendrite
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add a single config file format for dendrite. (#141)

Browse source 
* Add a single config file format for dendrite.

This config file combines all the configuration needed by the
dendrite components into a single file.

* Resolve relative paths against the current working directory

* Review comments

* more review comments!

* Document what the purpose of the version field is
This commit is contained in:
Mark Haines 2017-06-16 16:52:03 +01:00 committed by GitHub
parent 5e490e773f
commit c6e59ca967
2 changed files with 494 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

360
src/github.com/matrix-org/dendrite/common/config/config.go Normal file
View file

@ -0,0 +1,360 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"bytes"
"crypto/sha256"
"encoding/pem"
"fmt"
"github.com/matrix-org/gomatrixserverlib"
"golang.org/x/crypto/ed25519"
"gopkg.in/yaml.v2"
"io/ioutil"
"path/filepath"
"strings"
"time"
)
// Version is the current version of the config format.
// This will change whenever we make breaking changes to the config format.
const Version = "v0"
// Dendrite contains all the config used by a dendrite process.
// Relative paths are resolved relative to the current working directory
type Dendrite struct {
// The version of the configuration file.
// If the version in a file doesn't match the current dendrite config
// version then we can give a clear error message telling the user
// to update their config file to the current version.
// The version of the file should only be different if there has
// been a breaking change to the config file format.
Version string `yaml:"version"`
// The configuration required for a matrix server.
Matrix struct {
// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
// Path to the private key which will be used to sign requests and events.
PrivateKeyPath Path `yaml:"private_key"`
// The private key which will be used to sign requests and events.
PrivateKey ed25519.PrivateKey `yaml:"-"`
// An arbitrary string used to uniquely identify the PrivateKey. Must start with the
// prefix "ed25519:".
KeyID gomatrixserverlib.KeyID `yaml:"-"`
// List of paths to X509 certificates used by the external federation listeners.
// These are used to calculate the TLS fingerprints to publish for this server.
// Other matrix servers talking to this server will expect the x509 certificate
// to match one of these certificates.
// The certificates should be in PEM format.
FederationCertificatePaths []Path `yaml:"federation_certificates"`
// A list of SHA256 TLS fingerprints for the X509 certificates used by the
// federation listener for this server.
TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"`
// How long a remote server can cache our server key for before requesting it again.
// Increasing this number will reduce the number of requests made by remote servers
// for our key, but increases the period a compromised key will be considered valid
// by remote servers.
// Defaults to 24 hours.
KeyValidityPeriod time.Duration `yaml:"key_validity_period"`
} `yaml:"matrix"`
// The configuration specific to the media repostitory.
Media struct {
// The base path to where the media files will be stored. May be relative or absolute.
BasePath Path `yaml:"base_path"`
// The absolute base path to where media files will be stored.
AbsBasePath Path `yaml:"-"`
// The maximum file size in bytes that is allowed to be stored on this server.
// Note: if max_file_size_bytes is set to 0, the size is unlimited.
// Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"`
// Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
DynamicThumbnails bool `yaml:"dynamic_thumbnails"`
// The maximum number of simultaneous thumbnail generators. default: 10
MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"`
// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"`
} `yaml:"media"`
// The configuration for talking to kafka.
Kafka struct {
// A list of kafka addresses to connect to.
Addresses []string `yaml:"addresses"`
// The names of the topics to use when reading and writing from kafka.
Topics struct {
// Topic for roomserver/api.InputRoomEvent events.
InputRoomEvent Topic `yaml:"input_room_event"`
// Topic for roomserver/api.OutputRoomEvent events.
OutputRoomEvent Topic `yaml:"output_room_event"`
}
} `yaml:"kafka"`
// Postgres Config
Database struct {
// The Account database stores the login details and account information
// for local users. It is accessed by the ClientAPI.
Account DataSource `yaml:"account"`
// The Device database stores session information for the devices of logged
// in local users. It is accessed by the ClientAPI, the MediaAPI and the SyncAPI.
Device DataSource `yaml:"device"`
// The MediaAPI database stores information about files uploaded and downloaded
// by local users. It is only accessed by the MediaAPI.
MediaAPI DataSource `yaml:"media_api"`
// The ServerKey database caches the public keys of remote servers.
// It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI.
ServerKey DataSource `yaml:"server_key"`
// The SyncAPI stores information used by the SyncAPI server.
// It is only accessed by the SyncAPI server.
SyncAPI DataSource `yaml:"sync_api"`
// The RoomServer database stores information about matrix rooms.
// It is only accessed by the RoomServer.
RoomServer DataSource `yaml:"room_server"`
} `yaml:"database"`
// The internal addresses the components will listen on.
// These should not be exposed externally as they expose metrics and debugging APIs.
Listen struct {
MediaAPI Address `yaml:"media_api"`
ClientAPI Address `yaml:"client_api"`
FederationAPI Address `yaml:"federation_api"`
SyncAPI Address `yaml:"sync_api"`
RoomServer Address `yaml:"room_server"`
} `yaml:"listen"`
}
// A Path on the filesystem.
type Path string
// A DataSource for opening a postgresql database using lib/pq.
type DataSource string
// A Topic in kafka.
type Topic string
// An Address to listen on.
type Address string
// FileSizeBytes is a file size in bytes
type FileSizeBytes int64
// ThumbnailSize contains a single thumbnail size configuration
type ThumbnailSize struct {
// Maximum width of the thumbnail image
Width int `yaml:"width"`
// Maximum height of the thumbnail image
Height int `yaml:"height"`
// ResizeMethod is one of crop or scale.
// crop scales to fill the requested dimensions and crops the excess.
// scale scales to fit the requested dimensions and one dimension may be smaller than requested.
ResizeMethod string `yaml:"method,omitempty"`
}
// Load a yaml config file
func Load(configPath string) (*Dendrite, error) {
configData, err := ioutil.ReadFile(configPath)
if err != nil {
return nil, err
}
basePath, err := filepath.Abs(".")
if err != nil {
return nil, err
}
// Pass the current working directory and ioutil.ReadFile so that they can
// be mocked in the tests
return loadConfig(basePath, configData, ioutil.ReadFile)
}
// An Error indicates a problem parsing the config.
type Error struct {
// List of problems encountered parsing the config.
Problems []string
}
func loadConfig(
basePath string,
configData []byte,
readFile func(string) ([]byte, error),
) (*Dendrite, error) {
var config Dendrite
var err error
if err = yaml.Unmarshal(configData, &config); err != nil {
return nil, err
}
config.setDefaults()
if err = config.check(); err != nil {
return nil, err
}
privateKeyPath := absPath(basePath, config.Matrix.PrivateKeyPath)
privateKeyData, err := readFile(privateKeyPath)
if err != nil {
return nil, err
}
if config.Matrix.KeyID, config.Matrix.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil {
return nil, err
}
for _, certPath := range config.Matrix.FederationCertificatePaths {
absCertPath := absPath(basePath, certPath)
pemData, err := readFile(absCertPath)
if err != nil {
return nil, err
}
fingerprint := fingerprintPEM(pemData)
if fingerprint == nil {
return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath)
}
config.Matrix.TLSFingerPrints = append(config.Matrix.TLSFingerPrints, *fingerprint)
}
config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath))
return &config, nil
}
func (config *Dendrite) setDefaults() {
if config.Matrix.KeyValidityPeriod == 0 {
config.Matrix.KeyValidityPeriod = 24 * time.Hour
}
if config.Media.MaxThumbnailGenerators == 0 {
config.Media.MaxThumbnailGenerators = 10
}
if config.Media.MaxFileSizeBytes == nil {
defaultMaxFileSizeBytes := FileSizeBytes(10485760)
config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes
}
}
func (e Error) Error() string {
if len(e.Problems) == 1 {
return e.Problems[0]
}
return fmt.Sprintf(
"%s (and %d other problems)", e.Problems[0], len(e.Problems)-1,
)
}
func (config *Dendrite) check() error {
var problems []string
if config.Version != Version {
return Error{[]string{fmt.Sprintf(
"unknown config version %q, expected %q", config.Version, Version,
)}}
}
checkNotEmpty := func(key string, value string) {
if value == "" {
problems = append(problems, fmt.Sprintf("missing config key %q", key))
}
}
checkNotZero := func(key string, value int64) {
if value == 0 {
problems = append(problems, fmt.Sprintf("missing config key %q", key))
}
}
checkPositive := func(key string, value int64) {
if value < 0 {
problems = append(problems, fmt.Sprintf("invalid value for config key %q: %d", key, value))
}
}
checkNotEmpty("matrix.server_name", string(config.Matrix.ServerName))
checkNotEmpty("matrix.private_key", string(config.Matrix.PrivateKeyPath))
checkNotZero("matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths)))
checkNotEmpty("media.base_path", string(config.Media.BasePath))
checkPositive("media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes))
checkPositive("media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators))
for i, size := range config.Media.ThumbnailSizes {
checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width))
checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height))
}
checkNotZero("kafka.addresses", int64(len(config.Kafka.Addresses)))
checkNotEmpty("kafka.topics.input_room_event", string(config.Kafka.Topics.InputRoomEvent))
checkNotEmpty("kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent))
checkNotEmpty("database.account", string(config.Database.Account))
checkNotEmpty("database.device", string(config.Database.Device))
checkNotEmpty("database.server_key", string(config.Database.ServerKey))
checkNotEmpty("database.media_api", string(config.Database.MediaAPI))
checkNotEmpty("database.sync_api", string(config.Database.SyncAPI))
checkNotEmpty("database.room_server", string(config.Database.RoomServer))
checkNotEmpty("listen.media_api", string(config.Listen.MediaAPI))
checkNotEmpty("listen.client_api", string(config.Listen.ClientAPI))
checkNotEmpty("listen.federation_api", string(config.Listen.FederationAPI))
checkNotEmpty("listen.sync_api", string(config.Listen.SyncAPI))
checkNotEmpty("listen.room_server", string(config.Listen.RoomServer))
if problems != nil {
return Error{problems}
}
return nil
}
func absPath(dir string, path Path) string {
if filepath.IsAbs(string(path)) {
// filepath.Join cleans the path so we should clean the absolute paths as well for consistency.
return filepath.Clean(string(path))
}
return filepath.Join(dir, string(path))
}
func readKeyPEM(path string, data []byte) (gomatrixserverlib.KeyID, ed25519.PrivateKey, error) {
for {
var keyBlock *pem.Block
keyBlock, data = pem.Decode(data)
if data == nil {
return "", nil, fmt.Errorf("no matrix private key PEM data in %q", path)
}
if keyBlock.Type == "MATRIX PRIVATE KEY" {
keyID := keyBlock.Headers["Key-ID"]
if keyID == "" {
return "", nil, fmt.Errorf("missing key ID in PEM data in %q", path)
}
if !strings.HasPrefix(keyID, "ed25519:") {
return "", nil, fmt.Errorf("key ID %q doesn't start with \"ed25519:\" in %q", keyID, path)
}
_, privKey, err := ed25519.GenerateKey(bytes.NewReader(keyBlock.Bytes))
if err != nil {
return "", nil, err
}
return gomatrixserverlib.KeyID(keyID), privKey, nil
}
}
}
func fingerprintPEM(data []byte) *gomatrixserverlib.TLSFingerprint {
for {
var certDERBlock *pem.Block
certDERBlock, data = pem.Decode(data)
if data == nil {
return nil
}
if certDERBlock.Type == "CERTIFICATE" {
digest := sha256.Sum256(certDERBlock.Bytes)
return &gomatrixserverlib.TLSFingerprint{digest[:]}
}
}
}

134
src/github.com/matrix-org/dendrite/common/config/config_test.go Normal file
View file

@ -0,0 +1,134 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"fmt"
"testing"
)
func TestLoadConfigRelative(t *testing.T) {
_, err := loadConfig("/my/config/dir", []byte(testConfig),
mockReadFile{
"/my/config/dir/matrix_key.pem": testKey,
"/my/config/dir/tls_cert.pem": testCert,
}.readFile,
)
if err != nil {
t.Error("failed to load config:", err)
}
}
const testConfig = `
version: v0
matrix:
server_name: localhost
private_key: matrix_key.pem
federation_certificates: [tls_cert.pem]
media:
base_path: media_store
kafka:
addresses: ["localhost:9092"]
topics:
input_room_event: input.room
output_room_event: output.room
database:
media_api: "postgresql:///media_api"
account: "postgresql:///account"
device: "postgresql:///device"
server_key: "postgresql:///server_keys"
sync_api: "postgresql:///syn_api"
room_server: "postgresql:///room_server"
listen:
room_server: "localhost:7770"
client_api: "localhost:7771"
federation_api: "localhost:7772"
sync_api: "localhost:7773"
media_api: "localhost:7774"
`
type mockReadFile map[string]string
func (m mockReadFile) readFile(path string) ([]byte, error) {
data, ok := m[path]
if !ok {
return nil, fmt.Errorf("no such file %q", path)
}
return []byte(data), nil
}
func TestReadKey(t *testing.T) {
keyID, _, err := readKeyPEM("path/to/key", []byte(testKey))
if err != nil {
t.Error("failed to load private key:", err)
}
wantKeyID := testKeyID
if wantKeyID != string(keyID) {
t.Errorf("wanted key ID to be %q, got %q", wantKeyID, keyID)
}
}
const testKeyID = "ed25519:c8NsuQ"
const testKey = `
-----BEGIN MATRIX PRIVATE KEY-----
Key-ID: ` + testKeyID + `
7KRZiZ2sTyRR8uqqUjRwczuwRXXkUMYIUHq4Mc3t4bE=
-----END MATRIX PRIVATE KEY-----
`
func TestFingerprintPEM(t *testing.T) {
got := fingerprintPEM([]byte(testCert))
if got == nil {
t.Error("failed to calculate fingerprint")
}
if string(got.SHA256) != testCertFingerprint {
t.Errorf("bad fingerprint: wanted %q got %q", got, testCertFingerprint)
}
}
const testCertFingerprint = "56.\\SPQxE\xd4\x95\xfb\xf6\xd5\x04\x91\xcb/\x07\xb1^\x88\x08\xe3\xc1p\xdfY\x04\x19w\xcb"
const testCert = `
-----BEGIN CERTIFICATE-----
MIIE0zCCArugAwIBAgIJAPype3u24LJeMA0GCSqGSIb3DQEBCwUAMAAwHhcNMTcw
NjEzMTQyODU4WhcNMTgwNjEzMTQyODU4WjAAMIICIjANBgkqhkiG9w0BAQEFAAOC
Ag8AMIICCgKCAgEA3vNSr7lCh/alxPFqairp/PYohwdsqPvOD7zf7dJCNhy0gbdC
9/APwIbPAPL9nU+o9ud1ACNCKBCQin/9LnI5vd5pa/Ne+mmRADDLB/BBBoywSJWG
NSfKJ9n3XY1bjgtqi53uUh+RDdQ7sXudDqCUxiiJZmS7oqK/mp88XXAgCbuXUY29
GmzbbDz37vntuSxDgUOnJ8uPSvRp5YPKogA3JwW1SyrlLt4Z30CQ6nH3Y2Q5SVfJ
NIQyMrnwyjA9bCdXezv1cLXoTYn7U9BRyzXTZeXs3y3ldnRfISXN35CU04Az1F8j
lfj7nXMEqI/qAj/qhxZ8nVBB+rpNOZy9RJko3O+G5Qa/EvzkQYV1rW4TM2Yme88A
QyJspoV/0bXk6gG987PonK2Uk5djxSULhnGVIqswydyH0Nzb+slRp2bSoWbaNlee
+6TIeiyTQYc055pCHOp22gtLrC5LQGchksi02St2ZzRHdnlfqCJ8S9sS7x3trzds
cYueg1sGI+O8szpQ3eUM7OhJOBrx6OlR7+QYnQg1wr/V+JAz1qcyTC1URcwfeqtg
QjxFdBD9LfCtfK+AO51H9ugtsPJqOh33PmvfvUBEM05OHCA0lNaWJHROGpm4T4cc
YQI9JQk/0lB7itF1qK5RG74qgKdjkBkfZxi0OqkUgHk6YHtJlKfET8zfrtcCAwEA
AaNQME4wHQYDVR0OBBYEFGwb0NgH0Zr7Ga23njEJ85Ozf8M9MB8GA1UdIwQYMBaA
FGwb0NgH0Zr7Ga23njEJ85Ozf8M9MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL
BQADggIBAKU3RHXggbq/pLhGinU5q/9QT0TB/0bBnF1wNFkKQC0FrNJ+ZnBNmusy
oqOn7DEohBCCDxT0kgOC05gLEsGLkSXlVyqCsPFfycCFhtu1QzSRtQNRxB3pW3Wq
4/RFVYv0PGBjVBKxImQlEmXJWEDwemGKqDQZPtqR/FTHTbJcaT0xQr5+1oG6lawt
I/2cW6GQ0kYW/Szps8FgNdSNgVqCjjNIzBYbWhRWMx/63qD1ReUbY7/Yw9KKT8nK
zXERpbTM9k+Pnm0g9Gep+9HJ1dBFJeuTPugKeSeyqg2OJbENw1hxGs/HjBXw7580
ioiMn/kMj6Tg/f3HCfKrdHHBFQw0/fJW6o17QImYIpPOPzc5RjXBrCJWb34kxqEd
NQdKgejWiV/LlVsguIF8hVZH2kRzvoyypkVUtSUYGmjvA5UXoORQZfJ+b41llq1B
GcSF6iaVbAFKnsUyyr1i9uHz/6Muqflphv/SfZxGheIn5u3PnhXrzDagvItjw0NS
n0Xq64k7fc42HXJpF8CGBkSaIhtlzcruO+vqR80B9r62+D0V7VmHOnP135MT6noU
8F0JQfEtP+I8NII5jHSF/khzSgP5g80LS9tEc2ILnIHK1StkInAoRQQ+/HsQsgbz
ANAf5kxmMsM0zlN2hkxl0H6o7wKlBSw3RI3cjfilXiMWRPJrzlc4
-----END CERTIFICATE-----
`