Application Service Registration (#390)

* Add ability for App Services to register users

AS Tokens are pulled from their respective configs, which are then
checked against when an AS tries to register using
m.login.application_service. If the token exists and the new username is
within their specified namespace, then the user is created as a
password-less user.

Signed-off-by: Andrew Morgan (https://amorgan.xyz) <andrew@amorgan.xyz>

* Validate loaded Application Services

* Ensure no two app services have the same token or ID
* Check namespaces are valid regex
* Ensure users can't register inside an exclusive app service namespace
* Ensure exclusive app service namespaces are exclusive with each other
* Precompile application service namespace regexes so we don't need to
do so every time a user is registered

Signed-off-by: Andrew Morgan (https://amorgan.xyz) <andrew@amorgan.xyz>

src/github.com/matrix-org/dendrite/clientapi/auth/authtypes/account.go

@@ -24,6 +24,7 @@ type Account struct {
 	Localpart    string
 	ServerName   gomatrixserverlib.ServerName
 	Profile      *Profile
+	AppServiceID string
 	// TODO: Other flags like IsAdmin, IsGuest
 	// TODO: Devices
 	// TODO: Associations (e.g. with application services)

src/github.com/matrix-org/dendrite/clientapi/auth/authtypes/logintypes.go

@@ -8,4 +8,5 @@ const (
 	LoginTypeDummy              = "m.login.dummy"
 	LoginTypeSharedSecret       = "org.matrix.login.shared_secret"
 	LoginTypeRecaptcha          = "m.login.recaptcha"
+	LoginTypeApplicationService = "m.login.application_service"
 )

src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/accounts_table.go

@@ -32,14 +32,16 @@ CREATE TABLE IF NOT EXISTS account_accounts (
     -- When this account was first created, as a unix timestamp (ms resolution).
     created_ts BIGINT NOT NULL,
     -- The password hash for this account. Can be NULL if this is a passwordless account.
-    password_hash TEXT
+    password_hash TEXT,
+    -- Identifies which Application Service this account belongs to, if any.
+    appservice_id TEXT
     -- TODO:
-    -- is_guest, is_admin, appservice_id, upgraded_ts, devices, any email reset stuff?
+    -- is_guest, is_admin, upgraded_ts, devices, any email reset stuff?
 );
 `
 
 const insertAccountSQL = "" +
-	"INSERT INTO account_accounts(localpart, created_ts, password_hash) VALUES ($1, $2, $3)"
+	"INSERT INTO account_accounts(localpart, created_ts, password_hash, appservice_id) VALUES ($1, $2, $3, $4)"
 
 const selectAccountByLocalpartSQL = "" +
 	"SELECT localpart FROM account_accounts WHERE localpart = $1"
@@ -78,17 +80,26 @@ func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.Server
 // this account will be passwordless. Returns an error if this account already exists. Returns the account
 // on success.
 func (s *accountsStatements) insertAccount(
-	ctx context.Context, localpart, hash string,
+	ctx context.Context, localpart, hash, appserviceID string,
 ) (*authtypes.Account, error) {
 	createdTimeMS := time.Now().UnixNano() / 1000000
 	stmt := s.insertAccountStmt
-	if _, err := stmt.ExecContext(ctx, localpart, createdTimeMS, hash); err != nil {
+
+	var err error
+	if appserviceID == "" {
+		_, err = stmt.ExecContext(ctx, localpart, createdTimeMS, hash, nil)
+	} else {
+		_, err = stmt.ExecContext(ctx, localpart, createdTimeMS, hash, appserviceID)
+	}
+	if err != nil {
 		return nil, err
 	}
+
 	return &authtypes.Account{
 		Localpart:    localpart,
 		UserID:       makeUserID(localpart, s.serverName),
 		ServerName:   s.serverName,
+		AppServiceID: appserviceID,
 	}, nil
 }
 

src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go

@@ -121,19 +121,25 @@ func (d *Database) SetDisplayName(
 // for this account. If no password is supplied, the account will be a passwordless account. If the
 // account already exists, it will return nil, nil.
 func (d *Database) CreateAccount(
-	ctx context.Context, localpart, plaintextPassword string,
+	ctx context.Context, localpart, plaintextPassword, appserviceID string,
 ) (*authtypes.Account, error) {
-	hash, err := hashPassword(plaintextPassword)
+	var err error
+
+	// Generate a password hash if this is not a password-less user
+	hash := ""
+	if plaintextPassword != "" {
+		hash, err = hashPassword(plaintextPassword)
 		if err != nil {
 			return nil, err
 		}
+	}
 	if err := d.profiles.insertProfile(ctx, localpart); err != nil {
 		if common.IsUniqueConstraintViolationErr(err) {
 			return nil, nil
 		}
 		return nil, err
 	}
-	return d.accounts.insertAccount(ctx, localpart, hash)
+	return d.accounts.insertAccount(ctx, localpart, hash, appserviceID)
 }
 
 // SaveMembership saves the user matching a given localpart as a member of a given

src/github.com/matrix-org/dendrite/clientapi/jsonerror/jsonerror.go

@@ -86,7 +86,7 @@ func MissingToken(msg string) *MatrixError {
 }
 
 // UnknownToken is an error when the client tries to access a resource which
-// requires authentication and supplies a valid, but out-of-date token.
+// requires authentication and supplies an unrecognized token
 func UnknownToken(msg string) *MatrixError {
 	return &MatrixError{"M_UNKNOWN_TOKEN", msg}
 }
@@ -109,6 +109,13 @@ func UserInUse(msg string) *MatrixError {
 	return &MatrixError{"M_USER_IN_USE", msg}
 }
 
+// ASExclusive is an error returned when an application service tries to
+// register an username that is outside of its registered namespace, or if a
+// user attempts to register a username within an exclusive namespace
+func ASExclusive(msg string) *MatrixError {
+	return &MatrixError{"M_EXCLUSIVE", msg}
+}
+
 // GuestAccessForbidden is an error which is returned when the client is
 // forbidden from accessing a resource as a guest.
 func GuestAccessForbidden(msg string) *MatrixError {

src/github.com/matrix-org/dendrite/clientapi/routing/register.go

@@ -63,7 +63,7 @@ var (
 // remembered. If ANY parameters are supplied, the server should REPLACE all knowledge of
 // previous parameters with the ones supplied. This mean you cannot "build up" request params.
 type registerRequest struct {
-	// registration parameters.
+	// registration parameters
 	Password string `json:"password"`
 	Username string `json:"username"`
 	Admin    bool   `json:"admin"`
@@ -71,6 +71,10 @@ type registerRequest struct {
 	Auth authDict `json:"auth"`
 
 	InitialDisplayName *string `json:"initial_device_display_name"`
+
+	// Application Services place Type in the root of their registration
+	// request, whereas clients place it in the authDict struct.
+	Type authtypes.LoginType `json:"type"`
 }
 
 type authDict struct {
@@ -233,7 +237,110 @@ func validateRecaptcha(
 	return nil
 }
 
-// Register processes a /register request. http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+// UsernameIsWithinApplicationServiceNamespace checks to see if a username falls
+// within any of the namespaces of a given Application Service. If no
+// Application Service is given, it will check to see if it matches any
+// Application Service's namespace.
+func UsernameIsWithinApplicationServiceNamespace(
+	cfg *config.Dendrite,
+	username string,
+	appservice *config.ApplicationService,
+) bool {
+	if appservice != nil {
+		// Loop through given Application Service's namespaces and see if any match
+		for _, namespace := range appservice.NamespaceMap["users"] {
+			// AS namespaces are checked for validity in config
+			if namespace.RegexpObject.MatchString(username) {
+				return true
+			}
+		}
+		return false
+	}
+
+	// Loop through all known Application Service's namespaces and see if any match
+	for _, knownAppservice := range cfg.Derived.ApplicationServices {
+		for _, namespace := range knownAppservice.NamespaceMap["users"] {
+			// AS namespaces are checked for validity in config
+			if namespace.RegexpObject.MatchString(username) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// UsernameMatchesMultipleExclusiveNamespaces will check if a given username matches
+// more than one exclusive namespace. More than one is not allowed
+func UsernameMatchesMultipleExclusiveNamespaces(
+	cfg *config.Dendrite,
+	username string,
+) bool {
+	// Check namespaces and see if more than one match
+	matchCount := 0
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		for _, namespaceSlice := range appservice.NamespaceMap {
+			for _, namespace := range namespaceSlice {
+				// Check if we have a match on this username
+				if namespace.RegexpObject.MatchString(username) {
+					matchCount++
+				}
+			}
+		}
+	}
+	return matchCount > 1
+}
+
+// validateApplicationService checks if a provided application service token
+// corresponds to one that is registered. If so, then it checks if the desired
+// username is within that application service's namespace. As long as these
+// two requirements are met, no error will be returned.
+func validateApplicationService(
+	cfg *config.Dendrite,
+	req *http.Request,
+	username string,
+) (string, *util.JSONResponse) {
+	// Check if the token if the application service is valid with one we have
+	// registered in the config.
+	accessToken := req.URL.Query().Get("access_token")
+	var matchedApplicationService *config.ApplicationService
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		if appservice.ASToken == accessToken {
+			matchedApplicationService = &appservice
+			break
+		}
+	}
+	if matchedApplicationService != nil {
+		return "", &util.JSONResponse{
+			Code: 401,
+			JSON: jsonerror.UnknownToken("Supplied access_token does not match any known application service"),
+		}
+	}
+
+	// Ensure the desired username is within at least one of the application service's namespaces.
+	if !UsernameIsWithinApplicationServiceNamespace(cfg, username, matchedApplicationService) {
+		// If we didn't find any matches, return M_EXCLUSIVE
+		return "", &util.JSONResponse{
+			Code: 401,
+			JSON: jsonerror.ASExclusive(fmt.Sprintf(
+				"Supplied username %s did not match any namespaces for application service ID: %s", username, matchedApplicationService.ID)),
+		}
+	}
+
+	// Check this user does not fit multiple application service namespaces
+	if UsernameMatchesMultipleExclusiveNamespaces(cfg, username) {
+		return "", &util.JSONResponse{
+			Code: 401,
+			JSON: jsonerror.ASExclusive(fmt.Sprintf(
+				"Supplied username %s matches multiple exclusive application service namespaces. Only 1 match allowed", username)),
+		}
+	}
+
+	// No errors, registration valid
+	return matchedApplicationService.ID, nil
+}
+
+// Register processes a /register request.
+// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
 func Register(
 	req *http.Request,
 	accountDB *accounts.Database,
@@ -273,6 +380,16 @@ func Register(
 		return *resErr
 	}
 
+	// Make sure normal user isn't registering under an exclusive application
+	// service namespace
+	if r.Auth.Type != "m.login.application_service" &&
+		cfg.Derived.ExclusiveApplicationServicesUsernameRegexp.MatchString(r.Username) {
+		return util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.ASExclusive("This username is reserved by an application service."),
+		}
+	}
+
 	logger := util.GetLogger(req.Context())
 	logger.WithFields(log.Fields{
 		"username":   r.Username,
@@ -294,7 +411,6 @@ func handleRegistrationFlow(
 	deviceDB *devices.Database,
 ) util.JSONResponse {
 	// TODO: Shared secret registration (create new user scripts)
-	// TODO: AS API registration
 	// TODO: Enable registration config flag
 	// TODO: Guest account upgrading
 
@@ -331,6 +447,21 @@ func handleRegistrationFlow(
 		// Add SharedSecret to the list of completed registration stages
 		sessions[sessionID] = append(sessions[sessionID], authtypes.LoginTypeSharedSecret)
 
+	case authtypes.LoginTypeApplicationService:
+		// Check Application Service register user request is valid.
+		// The application service's ID is returned if so.
+		appserviceID, err := validateApplicationService(cfg, req, r.Username)
+
+		if err != nil {
+			return *err
+		}
+
+		// If no error, application service was successfully validated.
+		// Don't need to worry about appending to registration stages as
+		// application service registration is entirely separate.
+		return completeRegistration(req.Context(), accountDB, deviceDB,
+			r.Username, "", appserviceID, r.InitialDisplayName)
+
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
 		// Add Dummy to the list of completed registration stages
@@ -344,7 +475,29 @@ func handleRegistrationFlow(
 	}
 
 	// Check if the user's registration flow has been completed successfully
-	if !checkFlowCompleted(sessions[sessionID], cfg.Derived.Registration.Flows) {
+	// A response with current registration flow and remaining available methods
+	// will be returned if a flow has not been successfully completed yet
+	return checkAndCompleteFlow(sessions[sessionID], req, r, sessionID, cfg, accountDB, deviceDB)
+}
+
+// checkAndCompleteFlow checks if a given registration flow is completed given
+// a set of allowed flows. If so, registration is completed, otherwise a
+// response with
+func checkAndCompleteFlow(
+	flow []authtypes.LoginType,
+	req *http.Request,
+	r registerRequest,
+	sessionID string,
+	cfg *config.Dendrite,
+	accountDB *accounts.Database,
+	deviceDB *devices.Database,
+) util.JSONResponse {
+	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
+		// This flow was completed, registration can continue
+		return completeRegistration(req.Context(), accountDB, deviceDB,
+			r.Username, r.Password, "", r.InitialDisplayName)
+	}
+
 	// There are still more stages to complete.
 	// Return the flows and those that have been completed.
 	return util.JSONResponse{
@@ -352,10 +505,6 @@ func handleRegistrationFlow(
 		JSON: newUserInteractiveResponse(sessionID,
 			cfg.Derived.Registration.Flows, cfg.Derived.Registration.Params),
 	}
-	}
-
-	return completeRegistration(req.Context(), accountDB, deviceDB,
-		r.Username, r.Password, r.InitialDisplayName)
 }
 
 // LegacyRegister process register requests from the legacy v1 API
@@ -396,10 +545,10 @@ func LegacyRegister(
 			return util.MessageResponse(403, "HMAC incorrect")
 		}
 
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, nil)
+		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", nil)
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, nil)
+		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", nil)
 	default:
 		return util.JSONResponse{
 			Code: 501,
@@ -441,7 +590,7 @@ func completeRegistration(
 	ctx context.Context,
 	accountDB *accounts.Database,
 	deviceDB *devices.Database,
-	username, password string,
+	username, password, appserviceID string,
 	displayName *string,
 ) util.JSONResponse {
 	if username == "" {
@@ -450,14 +599,15 @@ func completeRegistration(
 			JSON: jsonerror.BadJSON("missing username"),
 		}
 	}
-	if password == "" {
+	// Blank passwords are only allowed by registered application services
+	if password == "" && appserviceID == "" {
 		return util.JSONResponse{
 			Code: 400,
 			JSON: jsonerror.BadJSON("missing password"),
 		}
 	}
 
-	acc, err := accountDB.CreateAccount(ctx, username, password)
+	acc, err := accountDB.CreateAccount(ctx, username, password, appserviceID)
 	if err != nil {
 		return util.JSONResponse{
 			Code: 500,
@@ -580,7 +730,10 @@ func checkFlows(
 // checkFlowCompleted checks if a registration flow complies with any allowed flow
 // dictated by the server. Order of stages does not matter. A user may complete
 // extra stages as long as the required stages of at least one flow is met.
-func checkFlowCompleted(flow []authtypes.LoginType, allowedFlows []authtypes.Flow) bool {
+func checkFlowCompleted(
+	flow []authtypes.LoginType,
+	allowedFlows []authtypes.Flow,
+) bool {
 	// Iterate through possible flows to check whether any have been fully completed.
 	for _, allowedFlow := range allowedFlows {
 		if checkFlows(flow, allowedFlow.Stages) {

src/github.com/matrix-org/dendrite/cmd/create-account/main.go

@@ -69,7 +69,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	account, err := accountDB.CreateAccount(context.Background(), *username, *password)
+	account, err := accountDB.CreateAccount(context.Background(), *username, *password, "")
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)

src/github.com/matrix-org/dendrite/common/config/appservice.go

@@ -15,8 +15,11 @@
 package config
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
+	"regexp"
+	"strings"
 
 	"gopkg.in/yaml.v2"
 )
@@ -28,6 +31,8 @@ type ApplicationServiceNamespace struct {
 	Exclusive bool `yaml:"exclusive"`
 	// A regex pattern that represents the namespace
 	Regex string `yaml:"regex"`
+	// Regex object representing our pattern. Saves having to recompile every time
+	RegexpObject *regexp.Regexp
 }
 
 // ApplicationService represents a Matrix application service.
@@ -44,11 +49,12 @@ type ApplicationService struct {
 	// Localpart of application service user
 	SenderLocalpart string `yaml:"sender_localpart"`
 	// Information about an application service's namespaces
-	Namespaces map[string][]ApplicationServiceNamespace `yaml:"namespaces"`
+	NamespaceMap map[string][]ApplicationServiceNamespace `yaml:"namespaces"`
 }
 
+// loadAppservices iterates through all application service config files
+// and loads their data into the config object for later access.
 func loadAppservices(config *Dendrite) error {
-	// Iterate through and return all the Application Services
 	for _, configPath := range config.ApplicationServices.ConfigFiles {
 		// Create a new application service
 		var appservice ApplicationService
@@ -75,5 +81,110 @@ func loadAppservices(config *Dendrite) error {
 			config.Derived.ApplicationServices, appservice)
 	}
 
+	// Check for any errors in the loaded application services
+	return checkErrors(config)
+}
+
+// setupRegexps will create regex objects for exclusive and non-exclusive
+// usernames, aliases and rooms of all application services, so that other
+// methods can quickly check if a particular string matches any of them.
+func setupRegexps(cfg *Dendrite) {
+	// Combine all exclusive namespaces for later string checking
+	var exclusiveUsernameStrings, exclusiveAliasStrings, exclusiveRoomStrings []string
+
+	// If an application service's regex is marked as exclusive, add
+	// it's contents to the overall exlusive regex string
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		for key, namespaceSlice := range appservice.NamespaceMap {
+			switch key {
+			case "users":
+				appendExclusiveNamespaceRegexs(&exclusiveUsernameStrings, namespaceSlice)
+			case "aliases":
+				appendExclusiveNamespaceRegexs(&exclusiveAliasStrings, namespaceSlice)
+			case "rooms":
+				appendExclusiveNamespaceRegexs(&exclusiveRoomStrings, namespaceSlice)
+			}
+		}
+	}
+
+	// Join the regexes together into one big regex.
+	// i.e. "app1.*", "app2.*" -> "(app1.*)|(app2.*)"
+	// Later we can check if a username or some other string matches any exclusive
+	// regex and deny access if it isn't from an application service
+	exclusiveUsernames := strings.Join(exclusiveUsernameStrings, "|")
+
+	// TODO: Aliases and rooms. Needed?
+	//exclusiveAliases := strings.Join(exclusiveAliasStrings, "|")
+	//exclusiveRooms := strings.Join(exclusiveRoomStrings, "|")
+
+	cfg.Derived.ExclusiveApplicationServicesUsernameRegexp, _ = regexp.Compile(exclusiveUsernames)
+}
+
+// concatenateExclusiveNamespaces takes a slice of strings and a slice of
+// namespaces and will append the regexes of only the exclusive namespaces
+// into the string slice
+func appendExclusiveNamespaceRegexs(
+	exclusiveStrings *[]string, namespaces []ApplicationServiceNamespace,
+) {
+	for _, namespace := range namespaces {
+		if namespace.Exclusive {
+			// We append parenthesis to later separate each regex when we compile
+			// i.e. "app1.*", "app2.*" -> "(app1.*)|(app2.*)"
+			*exclusiveStrings = append(*exclusiveStrings, "("+namespace.Regex+")")
+		}
+
+		// Compile this regex into a Regexp object for later use
+		namespace.RegexpObject, _ = regexp.Compile(namespace.Regex)
+	}
+}
+
+// checkErrors checks for any configuration errors amongst the loaded
+// application services according to the application service spec.
+func checkErrors(config *Dendrite) error {
+	var idMap = make(map[string]bool)
+	var tokenMap = make(map[string]bool)
+
+	// Check that no two application services have the same as_token or id
+	for _, appservice := range config.Derived.ApplicationServices {
+		// Check if we've already seen this ID
+		if idMap[appservice.ID] {
+			return Error{[]string{fmt.Sprintf(
+				"Application Service ID %s must be unique", appservice.ID,
+			)}}
+		}
+		if tokenMap[appservice.ASToken] {
+			return Error{[]string{fmt.Sprintf(
+				"Application Service Token %s must be unique", appservice.ASToken,
+			)}}
+		}
+
+		// Add the id/token to their respective maps if we haven't already
+		// seen them.
+		idMap[appservice.ID] = true
+		tokenMap[appservice.ID] = true
+	}
+
+	// Check that namespace(s) are valid regex
+	for _, appservice := range config.Derived.ApplicationServices {
+		for _, namespaceSlice := range appservice.NamespaceMap {
+			for _, namespace := range namespaceSlice {
+				if !IsValidRegex(namespace.Regex) {
+					return Error{[]string{fmt.Sprintf(
+						"Invalid regex string for Application Service %s", appservice.ID,
+					)}}
+				}
+			}
+		}
+	}
+	setupRegexps(config)
+
 	return nil
 }
+
+// IsValidRegex returns true or false based on whether the
+// given string is valid regex or not
+func IsValidRegex(regexString string) bool {
+	_, err := regexp.Compile(regexString)
+
+	return err == nil
+}

src/github.com/matrix-org/dendrite/common/config/config.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"io/ioutil"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -230,6 +231,13 @@ type Dendrite struct {
 		// Application Services parsed from their config files
 		// The paths of which were given above in the main config file
 		ApplicationServices []ApplicationService
+
+		// A meta-regex compiled from all exclusive Application Service
+		// Regexes. When a user registers, we check that their username
+		// does not match any exclusive Application Service namespaces
+		ExclusiveApplicationServicesUsernameRegexp *regexp.Regexp
+
+		// TODO: Exclusive alias, room regexp's
 	} `yaml:"-"`
 }