Alias key backup endpoints onto /unstable, fix key backup bugs (#1947)

* Default /unstable requests to stable endpoints if not overridden specifically with a custom route

* Rewrite URL

* Try something different

* Fix routing manually

* Fix selectLatestVersionSQL

* Don't return 0 if no backup version exists

* Log more useful error

* fix up replace keys check

* Don't enforce uniqueness on e2e_room_keys_versions_idx

Co-authored-by: kegsay <kegan@matrix.org>

clientapi/routing/routing.go

@@ -898,157 +898,171 @@ func Setup(
 
 	// Key Backup Versions (Metadata)
 
-	r0mux.Handle("/room_keys/version/{version}",
+	getBackupKeysVersion := httputil.MakeAuthAPI("get_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		httputil.MakeAuthAPI("get_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
-			if err != nil {
+			return util.ErrorResponse(err)
-				return util.ErrorResponse(err)
+		}
-			}
+		return KeyBackupVersion(req, userAPI, device, vars["version"])
-			return KeyBackupVersion(req, userAPI, device, vars["version"])
+	})
-		}),
+
-	).Methods(http.MethodGet, http.MethodOptions)
+	getLatestBackupKeysVersion := httputil.MakeAuthAPI("get_latest_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-	r0mux.Handle("/room_keys/version",
+		return KeyBackupVersion(req, userAPI, device, "")
-		httputil.MakeAuthAPI("get_latest_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+	})
-			return KeyBackupVersion(req, userAPI, device, "")
+
-		}),
+	putBackupKeysVersion := httputil.MakeAuthAPI("put_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-	).Methods(http.MethodGet, http.MethodOptions)
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-	r0mux.Handle("/room_keys/version/{version}",
+		if err != nil {
-		httputil.MakeAuthAPI("put_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return util.ErrorResponse(err)
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		}
-			if err != nil {
+		return ModifyKeyBackupVersionAuthData(req, userAPI, device, vars["version"])
-				return util.ErrorResponse(err)
+	})
-			}
+
-			return ModifyKeyBackupVersionAuthData(req, userAPI, device, vars["version"])
+	deleteBackupKeysVersion := httputil.MakeAuthAPI("delete_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		}),
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-	).Methods(http.MethodPut)
+		if err != nil {
-	r0mux.Handle("/room_keys/version/{version}",
+			return util.ErrorResponse(err)
-		httputil.MakeAuthAPI("delete_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		}
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		return DeleteKeyBackupVersion(req, userAPI, device, vars["version"])
-			if err != nil {
+	})
-				return util.ErrorResponse(err)
+
-			}
+	postNewBackupKeysVersion := httputil.MakeAuthAPI("post_new_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			return DeleteKeyBackupVersion(req, userAPI, device, vars["version"])
+		return CreateKeyBackupVersion(req, userAPI, device)
-		}),
+	})
-	).Methods(http.MethodDelete)
+
-	r0mux.Handle("/room_keys/version",
+	r0mux.Handle("/room_keys/version/{version}", getBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
-		httputil.MakeAuthAPI("post_new_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+	r0mux.Handle("/room_keys/version", getLatestBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
-			return CreateKeyBackupVersion(req, userAPI, device)
+	r0mux.Handle("/room_keys/version/{version}", putBackupKeysVersion).Methods(http.MethodPut)
-		}),
+	r0mux.Handle("/room_keys/version/{version}", deleteBackupKeysVersion).Methods(http.MethodDelete)
-	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/room_keys/version", postNewBackupKeysVersion).Methods(http.MethodPost, http.MethodOptions)
+
+	unstableMux.Handle("/room_keys/version/{version}", getBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
+	unstableMux.Handle("/room_keys/version", getLatestBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions)
+	unstableMux.Handle("/room_keys/version/{version}", putBackupKeysVersion).Methods(http.MethodPut)
+	unstableMux.Handle("/room_keys/version/{version}", deleteBackupKeysVersion).Methods(http.MethodDelete)
+	unstableMux.Handle("/room_keys/version", postNewBackupKeysVersion).Methods(http.MethodPost, http.MethodOptions)
 
 	// Inserting E2E Backup Keys
 
 	// Bulk room and session
-	r0mux.Handle("/room_keys/keys",
+	putBackupKeys := httputil.MakeAuthAPI("put_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		httputil.MakeAuthAPI("put_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		version := req.URL.Query().Get("version")
-			version := req.URL.Query().Get("version")
+		if version == "" {
-			if version == "" {
+			return util.JSONResponse{
-				return util.JSONResponse{
+				Code: 400,
-					Code: 400,
+				JSON: jsonerror.InvalidArgumentValue("version must be specified"),
-					JSON: jsonerror.InvalidArgumentValue("version must be specified"),
-				}
 			}
-			var reqBody keyBackupSessionRequest
+		}
-			resErr := clientutil.UnmarshalJSONRequest(req, &reqBody)
+		var reqBody keyBackupSessionRequest
-			if resErr != nil {
+		resErr := clientutil.UnmarshalJSONRequest(req, &reqBody)
-				return *resErr
+		if resErr != nil {
-			}
+			return *resErr
-			return UploadBackupKeys(req, userAPI, device, version, &reqBody)
+		}
-		}),
+		return UploadBackupKeys(req, userAPI, device, version, &reqBody)
-	).Methods(http.MethodPut)
+	})
+
 	// Single room bulk session
-	r0mux.Handle("/room_keys/keys/{roomID}",
+	putBackupKeysRoom := httputil.MakeAuthAPI("put_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		httputil.MakeAuthAPI("put_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
-			if err != nil {
+			return util.ErrorResponse(err)
-				return util.ErrorResponse(err)
+		}
+		version := req.URL.Query().Get("version")
+		if version == "" {
+			return util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.InvalidArgumentValue("version must be specified"),
 			}
-			version := req.URL.Query().Get("version")
+		}
-			if version == "" {
+		roomID := vars["roomID"]
-				return util.JSONResponse{
+		var reqBody keyBackupSessionRequest
-					Code: 400,
+		reqBody.Rooms = make(map[string]struct {
-					JSON: jsonerror.InvalidArgumentValue("version must be specified"),
+			Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
-				}
+		})
-			}
+		reqBody.Rooms[roomID] = struct {
-			roomID := vars["roomID"]
+			Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
-			var reqBody keyBackupSessionRequest
+		}{
-			reqBody.Rooms = make(map[string]struct {
+			Sessions: map[string]userapi.KeyBackupSession{},
-				Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+		}
-			})
+		body := reqBody.Rooms[roomID]
-			reqBody.Rooms[roomID] = struct {
+		resErr := clientutil.UnmarshalJSONRequest(req, &body)
-				Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+		if resErr != nil {
-			}{
+			return *resErr
-				Sessions: map[string]userapi.KeyBackupSession{},
+		}
-			}
+		reqBody.Rooms[roomID] = body
-			body := reqBody.Rooms[roomID]
+		return UploadBackupKeys(req, userAPI, device, version, &reqBody)
-			resErr := clientutil.UnmarshalJSONRequest(req, &body)
+	})
-			if resErr != nil {
+
-				return *resErr
-			}
-			reqBody.Rooms[roomID] = body
-			return UploadBackupKeys(req, userAPI, device, version, &reqBody)
-		}),
-	).Methods(http.MethodPut)
 	// Single room, single session
-	r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}",
+	putBackupKeysRoomSession := httputil.MakeAuthAPI("put_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		httputil.MakeAuthAPI("put_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
-			if err != nil {
+			return util.ErrorResponse(err)
-				return util.ErrorResponse(err)
+		}
+		version := req.URL.Query().Get("version")
+		if version == "" {
+			return util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.InvalidArgumentValue("version must be specified"),
 			}
-			version := req.URL.Query().Get("version")
+		}
-			if version == "" {
+		var reqBody userapi.KeyBackupSession
-				return util.JSONResponse{
+		resErr := clientutil.UnmarshalJSONRequest(req, &reqBody)
-					Code: 400,
+		if resErr != nil {
-					JSON: jsonerror.InvalidArgumentValue("version must be specified"),
+			return *resErr
-				}
+		}
-			}
+		roomID := vars["roomID"]
-			var reqBody userapi.KeyBackupSession
+		sessionID := vars["sessionID"]
-			resErr := clientutil.UnmarshalJSONRequest(req, &reqBody)
+		var keyReq keyBackupSessionRequest
-			if resErr != nil {
+		keyReq.Rooms = make(map[string]struct {
-				return *resErr
+			Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
-			}
+		})
-			roomID := vars["roomID"]
+		keyReq.Rooms[roomID] = struct {
-			sessionID := vars["sessionID"]
+			Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
-			var keyReq keyBackupSessionRequest
+		}{
-			keyReq.Rooms = make(map[string]struct {
+			Sessions: make(map[string]userapi.KeyBackupSession),
-				Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+		}
-			})
+		keyReq.Rooms[roomID].Sessions[sessionID] = reqBody
-			keyReq.Rooms[roomID] = struct {
+		return UploadBackupKeys(req, userAPI, device, version, &keyReq)
-				Sessions map[string]userapi.KeyBackupSession `json:"sessions"`
+	})
-			}{
+
-				Sessions: make(map[string]userapi.KeyBackupSession),
+	r0mux.Handle("/room_keys/keys", putBackupKeys).Methods(http.MethodPut)
-			}
+	r0mux.Handle("/room_keys/keys/{roomID}", putBackupKeysRoom).Methods(http.MethodPut)
-			keyReq.Rooms[roomID].Sessions[sessionID] = reqBody
+	r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", putBackupKeysRoomSession).Methods(http.MethodPut)
-			return UploadBackupKeys(req, userAPI, device, version, &keyReq)
+
-		}),
+	unstableMux.Handle("/room_keys/keys", putBackupKeys).Methods(http.MethodPut)
-	).Methods(http.MethodPut)
+	unstableMux.Handle("/room_keys/keys/{roomID}", putBackupKeysRoom).Methods(http.MethodPut)
+	unstableMux.Handle("/room_keys/keys/{roomID}/{sessionID}", putBackupKeysRoomSession).Methods(http.MethodPut)
 
 	// Querying E2E Backup Keys
 
-	r0mux.Handle("/room_keys/keys",
+	getBackupKeys := httputil.MakeAuthAPI("get_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		httputil.MakeAuthAPI("get_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), "", "")
-			return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), "", "")
+	})
-		}),
+
-	).Methods(http.MethodGet, http.MethodOptions)
+	getBackupKeysRoom := httputil.MakeAuthAPI("get_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-	r0mux.Handle("/room_keys/keys/{roomID}",
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-		httputil.MakeAuthAPI("get_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		if err != nil {
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			return util.ErrorResponse(err)
-			if err != nil {
+		}
-				return util.ErrorResponse(err)
+		return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], "")
-			}
+	})
-			return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], "")
+
-		}),
+	getBackupKeysRoomSession := httputil.MakeAuthAPI("get_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-	).Methods(http.MethodGet, http.MethodOptions)
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
-	r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}",
+		if err != nil {
-		httputil.MakeAuthAPI("get_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			return util.ErrorResponse(err)
-			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		}
-			if err != nil {
+		return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], vars["sessionID"])
-				return util.ErrorResponse(err)
+	})
-			}
+
-			return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], vars["sessionID"])
+	r0mux.Handle("/room_keys/keys", getBackupKeys).Methods(http.MethodGet, http.MethodOptions)
-		}),
+	r0mux.Handle("/room_keys/keys/{roomID}", getBackupKeysRoom).Methods(http.MethodGet, http.MethodOptions)
-	).Methods(http.MethodGet, http.MethodOptions)
+	r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", getBackupKeysRoomSession).Methods(http.MethodGet, http.MethodOptions)
+
+	unstableMux.Handle("/room_keys/keys", getBackupKeys).Methods(http.MethodGet, http.MethodOptions)
+	unstableMux.Handle("/room_keys/keys/{roomID}", getBackupKeysRoom).Methods(http.MethodGet, http.MethodOptions)
+	unstableMux.Handle("/room_keys/keys/{roomID}/{sessionID}", getBackupKeysRoomSession).Methods(http.MethodGet, http.MethodOptions)
 
 	// Deleting E2E Backup Keys
 

userapi/api/api.go

@@ -72,13 +72,11 @@ func (a *KeyBackupSession) ShouldReplaceRoomKey(newKey *KeyBackupSession) bool {
 	// "if the keys have different values for is_verified, then it will keep the key that has is_verified set to true"
 	if newKey.IsVerified && !a.IsVerified {
 		return true
-	}
+	} else if newKey.FirstMessageIndex < a.FirstMessageIndex {
-	// "if they have the same values for is_verified, then it will keep the key with a lower first_message_index"
+		// "if they have the same values for is_verified, then it will keep the key with a lower first_message_index"
-	if newKey.FirstMessageIndex < a.FirstMessageIndex {
 		return true
-	}
+	} else if newKey.ForwardedCount < a.ForwardedCount {
-	// "and finally, is is_verified and first_message_index are equal, then it will keep the key with a lower forwarded_count"
+		// "and finally, is is_verified and first_message_index are equal, then it will keep the key with a lower forwarded_count"
-	if newKey.ForwardedCount < a.ForwardedCount {
 		return true
 	}
 	return false

userapi/storage/accounts/postgres/key_backup_table.go

@@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys (
     session_data TEXT NOT NULL
 );
 CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id, version);
-CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version);
+CREATE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version);
 `
 
 const insertBackupKeySQL = "" +

userapi/storage/accounts/postgres/key_backup_version_table.go

@@ -146,12 +146,19 @@ func (s *keyBackupVersionStatements) selectKeyBackup(
 ) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) {
 	var versionInt int64
 	if version == "" {
-		err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&versionInt)
+		var v *int64 // allows nulls
+		if err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&v); err != nil {
+			return
+		}
+		if v == nil {
+			err = sql.ErrNoRows
+			return
+		}
+		versionInt = *v
 	} else {
-		versionInt, err = strconv.ParseInt(version, 10, 64)
+		if versionInt, err = strconv.ParseInt(version, 10, 64); err != nil {
-	}
+			return
-	if err != nil {
+		}
-		return
 	}
 	versionResult = strconv.FormatInt(versionInt, 10)
 	var deletedInt int

userapi/storage/accounts/postgres/storage.go

@@ -479,7 +479,7 @@ func (d *Database) UpsertBackupKeys(
 						err = d.keyBackups.updateBackupKey(ctx, txn, userID, version, newKey)
 						changed = true
 						if err != nil {
-							return err
+							return fmt.Errorf("d.keyBackups.updateBackupKey: %w", err)
 						}
 					}
 					// if we shouldn't replace the key we do nothing with it
@@ -490,7 +490,7 @@ func (d *Database) UpsertBackupKeys(
 			err = d.keyBackups.insertBackupKey(ctx, txn, userID, version, newKey)
 			changed = true
 			if err != nil {
-				return err
+				return fmt.Errorf("d.keyBackups.insertBackupKey: %w", err)
 			}
 		}
 

userapi/storage/accounts/sqlite3/key_backup_table.go

@@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys (
     session_data TEXT NOT NULL
 );
 CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id, version);
-CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version);
+CREATE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version);
 `
 
 const insertBackupKeySQL = "" +

userapi/storage/accounts/sqlite3/key_backup_version_table.go

@@ -144,12 +144,19 @@ func (s *keyBackupVersionStatements) selectKeyBackup(
 ) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) {
 	var versionInt int64
 	if version == "" {
-		err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&versionInt)
+		var v *int64 // allows nulls
+		if err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&v); err != nil {
+			return
+		}
+		if v == nil {
+			err = sql.ErrNoRows
+			return
+		}
+		versionInt = *v
 	} else {
-		versionInt, err = strconv.ParseInt(version, 10, 64)
+		if versionInt, err = strconv.ParseInt(version, 10, 64); err != nil {
-	}
+			return
-	if err != nil {
+		}
-		return
 	}
 	versionResult = strconv.FormatInt(versionInt, 10)
 	var deletedInt int

userapi/storage/accounts/sqlite3/storage.go

@@ -520,7 +520,7 @@ func (d *Database) UpsertBackupKeys(
 						err = d.keyBackups.updateBackupKey(ctx, txn, userID, version, newKey)
 						changed = true
 						if err != nil {
-							return err
+							return fmt.Errorf("d.keyBackups.updateBackupKey: %w", err)
 						}
 					}
 					// if we shouldn't replace the key we do nothing with it
@@ -531,7 +531,7 @@ func (d *Database) UpsertBackupKeys(
 			err = d.keyBackups.insertBackupKey(ctx, txn, userID, version, newKey)
 			changed = true
 			if err != nil {
-				return err
+				return fmt.Errorf("d.keyBackups.insertBackupKey: %w", err)
 			}
 		}