From c4f5a0a6316200f797bf81d77c3cd2815a73706d Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Thu, 6 Aug 2020 08:29:59 -0400 Subject: [PATCH 01/48] Keep track of State at event for state resolution feat: first steps towards joining rooms over federation Add state-res as a dependency of conduit Add reverse_topological_power_sort before append_pdu Implement statehashstatid_pduid tree for keeping track of state Clean up implementation of state_hash as key for tracking state --- Cargo.lock | 206 +++++++++++- Cargo.toml | 7 +- src/client_server/account.rs | 6 +- src/client_server/alias.rs | 16 +- src/client_server/context.rs | 30 +- src/client_server/device.rs | 8 +- src/client_server/directory.rs | 13 +- src/client_server/filter.rs | 21 +- src/client_server/keys.rs | 2 +- src/client_server/media.rs | 4 +- src/client_server/membership.rs | 125 ++++++-- src/client_server/message.rs | 43 +-- src/client_server/room.rs | 2 +- src/client_server/session.rs | 20 +- src/client_server/state.rs | 48 ++- src/client_server/sync.rs | 2 +- src/client_server/unversioned.rs | 13 +- src/database.rs | 5 +- src/database/rooms.rs | 525 +++++++++++++++++++++---------- src/database/uiaa.rs | 6 +- src/pdu.rs | 29 ++ src/ruma_wrapper.rs | 10 +- src/server_server.rs | 27 +- src/utils.rs | 6 + 24 files changed, 818 insertions(+), 356 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a7334c..ffee8ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,6 +75,15 @@ dependencies = [ "opaque-debug 0.2.3", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "arc-swap" version = "0.4.7" @@ -248,6 +257,17 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "chrono" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942f72db697d8767c22d46a598e01f2d3b475501ea43d0db4f16d90259182d0b" +dependencies = [ + "num-integer", + "num-traits", + "time 0.1.43", +] + [[package]] name = "cloudabi" version = "0.1.0" @@ -281,6 +301,7 @@ dependencies = [ "serde", "serde_json", "sled", + "state-res", "thiserror", "tokio", ] @@ -456,6 +477,12 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" +[[package]] +name = "either" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" + [[package]] name = "encoding_rs" version = "0.8.23" @@ -872,6 +899,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -951,6 +987,21 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.8" @@ -1439,6 +1490,31 @@ dependencies = [ "syn", ] +[[package]] +name = "regex" +version = "1.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -1560,21 +1636,23 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-api", + "ruma-appservice-api", "ruma-client-api", "ruma-common", "ruma-events", "ruma-federation-api", "ruma-identifiers", + "ruma-serde", "ruma-signatures", ] [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "http", "percent-encoding", @@ -1589,7 +1667,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1597,14 +1675,28 @@ dependencies = [ "syn", ] +[[package]] +name = "ruma-appservice-api" +version = "0.2.0-alpha.1" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" +dependencies = [ + "ruma-api", + "ruma-common", + "ruma-events", + "ruma-identifiers", + "serde", + "serde_json", +] + [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "assign", "http", "js_int", + "percent-encoding", "ruma-api", "ruma-common", "ruma-events", @@ -1618,7 +1710,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-identifiers", @@ -1631,7 +1723,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-common", @@ -1646,7 +1738,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1657,7 +1749,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-api", @@ -1672,7 +1764,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1684,7 +1776,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro2", "quote", @@ -1695,7 +1787,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-serde", "serde", @@ -1706,7 +1798,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "form_urlencoded", "itoa", @@ -1718,7 +1810,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "base64 0.12.3", "ring", @@ -1910,6 +2002,15 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "sharded-slab" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06d5a3f5166fb5b42a5439f2eee8b9de149e235961e3eb21c5808fc3ea17ff3e" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.2.1" @@ -1983,6 +2084,22 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" +[[package]] +name = "state-res" +version = "0.1.0" +source = "git+https://github.com/ruma/state-res#789c8140890e076d38b23fa1147c4ff0500c0d38" +dependencies = [ + "itertools", + "js_int", + "maplit", + "ruma", + "serde", + "serde_json", + "thiserror", + "tracing", + "tracing-subscriber", +] + [[package]] name = "stdweb" version = "0.4.20" @@ -2104,6 +2221,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + [[package]] name = "time" version = "0.1.43" @@ -2251,9 +2377,21 @@ checksum = "6d79ca061b032d6ce30c660fded31189ca0b9922bf483cd70759f13a2d86786c" dependencies = [ "cfg-if", "log", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe233f4227389ab7df5b32649239da7ebe0b281824b4e84b342d04d3fd8c25e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.14" @@ -2263,6 +2401,48 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-log" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ccba2f8f16e0ed268fc765d9b7ff22e965e7185d32f8f1ec8294fe17d86e79" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd165311cc4d7a555ad11cc77a37756df836182db0d81aac908c8184c584f40" +dependencies = [ + "ansi_term", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", + "tracing-serde", +] + [[package]] name = "try-lock" version = "0.2.3" diff --git a/Cargo.toml b/Cargo.toml index 4945e3c..4c14d71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,9 +16,7 @@ edition = "2018" #rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } -#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "987d48666cf166cf12100b5dbc61b5e3385c4014" } # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers -#ruma = { path = "../ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } +ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently log = "0.4.8" # Used for emitting log entries @@ -33,6 +31,9 @@ reqwest = "0.10.6" # Used to send requests thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key +# state-res = { path = "../../state-res" } +state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0" } + [features] default = ["conduit_bin"] diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 9837d1b..9fa1a9c 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -75,7 +75,7 @@ pub fn get_register_available_route( )] pub fn register_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { if db.globals.registration_disabled() { return Err(Error::BadRequest( @@ -223,7 +223,7 @@ pub fn register_route( )] pub fn change_password_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -305,7 +305,7 @@ pub fn whoami_route(body: Ruma) -> ConduitResult, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index a029388..7dc9078 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -26,7 +26,7 @@ pub fn create_alias_route( db.rooms .set_alias(&body.room_alias, Some(&body.room_id), &db.globals)?; - Ok(create_alias::Response.into()) + Ok(create_alias::Response::new().into()) } #[cfg_attr( @@ -39,7 +39,7 @@ pub fn delete_alias_route( ) -> ConduitResult { db.rooms.set_alias(&body.room_alias, None, &db.globals)?; - Ok(delete_alias::Response.into()) + Ok(delete_alias::Response::new().into()) } #[cfg_attr( @@ -60,11 +60,7 @@ pub async fn get_alias_route( ) .await?; - return Ok(get_alias::Response { - room_id: response.room_id, - servers: response.servers, - } - .into()); + return Ok(get_alias::Response::new(response.room_id, response.servers).into()); } let room_id = db @@ -75,9 +71,5 @@ pub async fn get_alias_route( "Room with alias not found.", ))?; - Ok(get_alias::Response { - room_id, - servers: vec![db.globals.server_name().to_string()], - } - .into()) + Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_string()]).into()) } diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 7a6cbce..7b1fad9 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -12,7 +12,7 @@ use rocket::get; )] pub fn get_context_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -75,18 +75,18 @@ pub fn get_context_route( .map(|(_, pdu)| pdu.to_room_event()) .collect::>(); - Ok(get_context::Response { - start: start_token, - end: end_token, - events_before, - event: Some(base_event), - events_after, - state: db // TODO: State at event - .rooms - .room_state_full(&body.room_id)? - .values() - .map(|pdu| pdu.to_state_event()) - .collect(), - } - .into()) + let mut resp = get_context::Response::new(); + resp.start = start_token; + resp.end = end_token; + resp.events_before = events_before; + resp.event = Some(base_event); + resp.events_after = events_after; + resp.state = db // TODO: State at event + .rooms + .room_state_full(&body.room_id)? + .values() + .map(|pdu| pdu.to_state_event()) + .collect(); + + Ok(resp.into()) } diff --git a/src/client_server/device.rs b/src/client_server/device.rs index 379f827..89033f0 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -37,7 +37,7 @@ pub fn get_devices_route( )] pub fn get_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,7 @@ pub fn get_device_route( )] pub fn update_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -80,7 +80,7 @@ pub fn update_device_route( )] pub fn delete_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -127,7 +127,7 @@ pub fn delete_device_route( )] pub fn delete_devices_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 26188f7..0aace15 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -6,7 +6,7 @@ use ruma::{ error::ErrorKind, r0::{ directory::{ - self, get_public_rooms, get_public_rooms_filtered, get_room_visibility, + get_public_rooms, get_public_rooms_filtered, get_room_visibility, set_room_visibility, }, room, @@ -14,6 +14,7 @@ use ruma::{ }, federation, }, + directory::PublicRoomsChunk, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, @@ -35,15 +36,15 @@ pub async fn get_public_rooms_filtered_route( if let Some(other_server) = body .server .clone() - .filter(|server| server != &db.globals.server_name().as_str()) + .filter(|server| server != db.globals.server_name().as_str()) { let response = server_server::send_request( &db, other_server, federation::directory::get_public_rooms::v1::Request { limit: body.limit, - since: body.since.clone(), - room_network: federation::directory::get_public_rooms::v1::RoomNetwork::Matrix, + since: body.since.as_deref(), + room_network: ruma::directory::RoomNetwork::Matrix, }, ) .await?; @@ -107,7 +108,7 @@ pub async fn get_public_rooms_filtered_route( // TODO: Do not load full state? let state = db.rooms.room_state_full(&room_id)?; - let chunk = directory::PublicRoomsChunk { + let chunk = PublicRoomsChunk { aliases: Vec::new(), canonical_alias: state .get(&(EventType::RoomCanonicalAlias, "".to_owned())) @@ -272,7 +273,7 @@ pub async fn get_public_rooms_route( body: get_public_rooms_filtered::IncomingRequest { filter: None, limit, - room_network: get_public_rooms_filtered::RoomNetwork::Matrix, + room_network: ruma::directory::RoomNetwork::Matrix, server, since, }, diff --git a/src/client_server/filter.rs b/src/client_server/filter.rs index 165419a..4322de3 100644 --- a/src/client_server/filter.rs +++ b/src/client_server/filter.rs @@ -7,23 +7,18 @@ use rocket::{get, post}; #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))] pub fn get_filter_route() -> ConduitResult { // TODO - Ok(get_filter::Response { - filter: filter::FilterDefinition { - event_fields: None, - event_format: None, - account_data: None, - room: None, - presence: None, - }, - } + Ok(get_filter::Response::new(filter::FilterDefinition { + event_fields: None, + event_format: None, + account_data: None, + room: None, + presence: None, + }) .into()) } #[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/user/<_>/filter"))] pub fn create_filter_route() -> ConduitResult { // TODO - Ok(create_filter::Response { - filter_id: utils::random_string(10), - } - .into()) + Ok(create_filter::Response::new(utils::random_string(10)).into()) } diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index f88878c..3311529 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -167,7 +167,7 @@ pub fn claim_keys_route( )] pub fn upload_signing_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/media.rs b/src/client_server/media.rs index efcb3a6..79c1f08 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -53,7 +53,7 @@ pub fn create_content_route( )] pub fn get_content_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _server_name: String, _media_id: String, ) -> ConduitResult { @@ -85,7 +85,7 @@ pub fn get_content_route( )] pub fn get_content_thumbnail_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _server_name: String, _media_id: String, ) -> ConduitResult { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 84c0ebd..c04cf7f 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -20,6 +20,8 @@ use ruma::{ events::{room::member, EventType}, EventId, Raw, RoomId, RoomVersionId, }; +use state_res::StateEvent; + use std::{collections::BTreeMap, convert::TryFrom}; #[cfg(feature = "conduit_bin")] @@ -92,17 +94,73 @@ pub async fn join_room_by_id_route( let send_join_response = server_server::send_request( &db, body.room_id.server_name().to_string(), - federation::membership::create_join_event::v2::Request { + federation::membership::create_join_event::v1::Request { room_id: body.room_id.clone(), event_id, - pdu_stub: serde_json::from_value::>(join_event_stub_value) + pdu_stub: serde_json::from_value(join_event_stub_value) .expect("Raw::from_value always works"), }, ) .await?; - dbg!(send_join_response); - todo!("Take send_join_response and 'create' the room using that data"); + dbg!(&send_join_response); + // todo!("Take send_join_response and 'create' the room using that data"); + + let mut event_map = send_join_response + .room_state + .state + .iter() + .map(|pdu| pdu.deserialize().map(StateEvent::Full)) + .map(|ev| { + let ev = ev?; + Ok::<_, serde_json::Error>((ev.event_id(), ev)) + }) + .collect::, _>>() + .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; + + let _auth_chain = send_join_response + .room_state + .auth_chain + .iter() + .flat_map(|pdu| pdu.deserialize().ok()) + .map(StateEvent::Full) + .collect::>(); + + // TODO make StateResolution's methods free functions ? or no self param ? + let sorted_events_ids = state_res::StateResolution::default() + .reverse_topological_power_sort( + &body.room_id, + &event_map.keys().cloned().collect::>(), + &mut event_map, + &db.rooms, + &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + ); + + for ev_id in &sorted_events_ids { + // this is a `state_res::StateEvent` that holds a `ruma::Pdu` + let pdu = event_map.get(ev_id).ok_or_else(|| { + Error::Conflict("Found event_id in sorted events that is not in resolved state") + })?; + + db.rooms.append_pdu( + PduBuilder { + room_id: pdu.room_id().unwrap_or(&body.room_id).clone(), + sender: pdu.sender().clone(), + event_type: pdu.kind(), + content: pdu.content().clone(), + unsigned: Some( + pdu.unsigned() + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ), + state_key: pdu.state_key(), + redacts: pdu.redacts().cloned(), + }, + &db.globals, + &db.account_data, + )?; + } } let event = member::MemberEventContent { @@ -127,10 +185,7 @@ pub async fn join_room_by_id_route( &db.account_data, )?; - Ok(join_room_by_id::Response { - room_id: body.room_id.clone(), - } - .into()) + Ok(join_room_by_id::Response::new(body.room_id.clone()).into()) } #[cfg_attr( @@ -140,7 +195,7 @@ pub async fn join_room_by_id_route( pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, db2: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, @@ -148,7 +203,13 @@ pub async fn join_room_by_id_or_alias_route( client_server::get_alias_route( db, Ruma { - body: alias::get_alias::IncomingRequest { room_alias }, + body: alias::get_alias::IncomingRequest::try_from(http::Request::new( + serde_json::json!({ "room_alias": room_alias }) + .to_string() + .as_bytes() + .to_vec(), + )) + .unwrap(), sender_id: body.sender_id.clone(), device_id: body.device_id.clone(), json_body: None, @@ -160,14 +221,32 @@ pub async fn join_room_by_id_or_alias_route( } }; + // TODO ruma needs to implement the same constructors for the Incoming variants + let tps = if let Some(in_tps) = &body.third_party_signed { + Some(ruma::api::client::r0::membership::ThirdPartySigned { + token: &in_tps.token, + sender: &in_tps.sender, + signatures: in_tps.signatures.clone(), + mxid: &in_tps.mxid, + }) + } else { + None + }; + let body = Ruma { sender_id: body.sender_id.clone(), device_id: body.device_id.clone(), json_body: None, - body: join_room_by_id::IncomingRequest { - room_id, - third_party_signed: body.third_party_signed.clone(), - }, + body: join_room_by_id::IncomingRequest::try_from(http::Request::new( + serde_json::json!({ + "room_id": room_id, + "third_party_signed": tps, + }) + .to_string() + .as_bytes() + .to_vec(), + )) + .unwrap(), }; Ok(join_room_by_id_or_alias::Response { @@ -219,7 +298,7 @@ pub fn leave_room_route( &db.account_data, )?; - Ok(leave_room::Response.into()) + Ok(leave_room::Response::new().into()) } #[cfg_attr( @@ -266,7 +345,7 @@ pub fn invite_user_route( )] pub fn kick_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -304,7 +383,7 @@ pub fn kick_user_route( &db.account_data, )?; - Ok(kick_user::Response.into()) + Ok(kick_user::Response::new().into()) } #[cfg_attr( @@ -313,7 +392,7 @@ pub fn kick_user_route( )] pub fn ban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -359,7 +438,7 @@ pub fn ban_user_route( &db.account_data, )?; - Ok(ban_user::Response.into()) + Ok(ban_user::Response::new().into()) } #[cfg_attr( @@ -368,7 +447,7 @@ pub fn ban_user_route( )] pub fn unban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -405,7 +484,7 @@ pub fn unban_user_route( &db.account_data, )?; - Ok(unban_user::Response.into()) + Ok(unban_user::Response::new().into()) } #[cfg_attr( @@ -414,13 +493,13 @@ pub fn unban_user_route( )] pub fn forget_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); db.rooms.forget(&body.room_id, &sender_id)?; - Ok(forget_room::Response.into()) + Ok(forget_room::Response::new().into()) } #[cfg_attr( diff --git a/src/client_server/message.rs b/src/client_server/message.rs index d851214..1b461d2 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -1,8 +1,11 @@ use super::State; use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Ruma}; -use ruma::api::client::{ - error::ErrorKind, - r0::message::{get_message_events, send_message_event}, +use ruma::{ + api::client::{ + error::ErrorKind, + r0::message::{get_message_events, send_message_event}, + }, + events::EventContent, }; use std::convert::TryInto; @@ -26,7 +29,7 @@ pub fn send_message_event_route( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), - event_type: body.event_type.clone(), + event_type: body.content.event_type().into(), content: serde_json::from_str( body.json_body .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? @@ -41,7 +44,7 @@ pub fn send_message_event_route( &db.account_data, )?; - Ok(send_message_event::Response { event_id }.into()) + Ok(send_message_event::Response::new(event_id).into()) } #[cfg_attr( @@ -50,7 +53,7 @@ pub fn send_message_event_route( )] pub fn get_message_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -92,13 +95,13 @@ pub fn get_message_events_route( .map(|(_, pdu)| pdu.to_room_event()) .collect::>(); - Ok(get_message_events::Response { - start: Some(body.from.clone()), - end: end_token, - chunk: events_after, - state: Vec::new(), - } - .into()) + let mut resp = get_message_events::Response::new(); + resp.start = Some(body.from.clone()); + resp.end = end_token; + resp.chunk = events_after; + resp.state = Vec::new(); + + Ok(resp.into()) } get_message_events::Direction::Backward => { let events_before = db @@ -116,13 +119,13 @@ pub fn get_message_events_route( .map(|(_, pdu)| pdu.to_room_event()) .collect::>(); - Ok(get_message_events::Response { - start: Some(body.from.clone()), - end: start_token, - chunk: events_before, - state: Vec::new(), - } - .into()) + let mut resp = get_message_events::Response::new(); + resp.start = Some(body.from.clone()); + resp.end = start_token; + resp.chunk = events_before; + resp.state = Vec::new(); + + Ok(resp.into()) } } } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index b5f1529..589a2dc 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -315,7 +315,7 @@ pub fn create_room_route( db.rooms.set_public(&room_id, true)?; } - Ok(create_room::Response { room_id }.into()) + Ok(create_room::Response::new(room_id).into()) } #[cfg_attr( diff --git a/src/client_server/session.rs b/src/client_server/session.rs index 4011058..948b455 100644 --- a/src/client_server/session.rs +++ b/src/client_server/session.rs @@ -1,5 +1,4 @@ -use super::State; -use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH}; +use super::{State, DEVICE_ID_LENGTH, TOKEN_LENGTH}; use crate::{utils, ConduitResult, Database, Error, Ruma}; use ruma::{ api::client::{ @@ -18,10 +17,7 @@ use rocket::{get, post}; /// when logging in. #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/login"))] pub fn get_login_types_route() -> ConduitResult { - Ok(get_login_types::Response { - flows: vec![get_login_types::LoginType::Password], - } - .into()) + Ok(get_login_types::Response::new(vec![get_login_types::LoginType::Password]).into()) } /// # `POST /_matrix/client/r0/login` @@ -40,15 +36,15 @@ pub fn get_login_types_route() -> ConduitResult { )] pub fn login_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { // Validate login method let user_id = // TODO: Other login methods - if let (login::UserInfo::MatrixId(username), login::LoginInfo::Password { password }) = - (body.user.clone(), body.login_info.clone()) + if let (login::IncomingUserInfo::MatrixId(username), login::IncomingLoginInfo::Password { password }) = + (&body.user, &body.login_info) { - let user_id = UserId::parse_with_server_name(username, db.globals.server_name()) + let user_id = UserId::parse_with_server_name(username.to_string(), db.globals.server_name()) .map_err(|_| Error::BadRequest( ErrorKind::InvalidUsername, "Username is invalid." @@ -126,7 +122,7 @@ pub fn logout_route( db.users.remove_device(&sender_id, device_id)?; - Ok(logout::Response.into()) + Ok(logout::Response::new().into()) } /// # `POST /_matrix/client/r0/logout/all` @@ -154,5 +150,5 @@ pub fn logout_all_route( } } - Ok(logout_all::Response.into()) + Ok(logout_all::Response::new().into()) } diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 60b3e9f..14cc497 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -8,9 +8,9 @@ use ruma::{ send_state_event_for_empty_key, send_state_event_for_key, }, }, - events::{room::canonical_alias, EventType}, - Raw, + events::{AnyStateEventContent, EventContent}, }; +use std::convert::TryFrom; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; @@ -33,17 +33,10 @@ pub fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - if body.event_type == EventType::RoomCanonicalAlias { - let canonical_alias = serde_json::from_value::< - Raw, - >(content.clone()) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid canonical alias."))?; + if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = &body.content { + let mut aliases = canonical_alias.alt_aliases.clone(); - let mut aliases = canonical_alias.alt_aliases; - - if let Some(alias) = canonical_alias.alias { + if let Some(alias) = canonical_alias.alias.clone() { aliases.push(alias); } @@ -68,7 +61,7 @@ pub fn send_state_event_for_key_route( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), - event_type: body.event_type.clone(), + event_type: body.content.event_type().into(), content, unsigned: None, state_key: Some(body.state_key.clone()), @@ -78,7 +71,7 @@ pub fn send_state_event_for_key_route( &db.account_data, )?; - Ok(send_state_event_for_key::Response { event_id }.into()) + Ok(send_state_event_for_key::Response::new(event_id).into()) } #[cfg_attr( @@ -93,25 +86,28 @@ pub fn send_state_event_for_empty_key_route( let Ruma { body: send_state_event_for_empty_key::IncomingRequest { - room_id, - event_type, - data, + room_id, content, .. }, sender_id, device_id, json_body, } = body; - Ok(send_state_event_for_empty_key::Response { - event_id: send_state_event_for_key_route( + Ok(send_state_event_for_empty_key::Response::new( + send_state_event_for_key_route( db, Ruma { - body: send_state_event_for_key::IncomingRequest { - room_id, - event_type, - data, - state_key: "".to_owned(), - }, + body: send_state_event_for_key::IncomingRequest::try_from(http::Request::new( + serde_json::json!({ + "room_id": room_id, + "state_key": "", + "content": content, + }) + .to_string() + .as_bytes() + .to_vec(), + )) + .unwrap(), sender_id, device_id, json_body, @@ -119,7 +115,7 @@ pub fn send_state_event_for_empty_key_route( )? .0 .event_id, - } + ) .into()) } diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 2307f02..ae4c224 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -31,7 +31,7 @@ use std::{ )] pub async fn sync_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/unversioned.rs b/src/client_server/unversioned.rs index 3ff8bec..ea7f633 100644 --- a/src/client_server/unversioned.rs +++ b/src/client_server/unversioned.rs @@ -1,6 +1,5 @@ use crate::ConduitResult; use ruma::api::client::unversioned::get_supported_versions; -use std::collections::BTreeMap; #[cfg(feature = "conduit_bin")] use rocket::get; @@ -17,13 +16,11 @@ use rocket::get; /// unstable features in their stable releases #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/versions"))] pub fn get_supported_versions_route() -> ConduitResult { - let mut unstable_features = BTreeMap::new(); + let mut resp = + get_supported_versions::Response::new(vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()]); - unstable_features.insert("org.matrix.e2e_cross_signing".to_owned(), true); + resp.unstable_features + .insert("org.matrix.e2e_cross_signing".to_owned(), true); - Ok(get_supported_versions::Response { - versions: vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()], - unstable_features, - } - .into()) + Ok(resp.into()) } diff --git a/src/database.rs b/src/database.rs index 7bbb6dd..6cd65c3 100644 --- a/src/database.rs +++ b/src/database.rs @@ -97,8 +97,8 @@ impl Database { }, pduid_pdu: db.open_tree("pduid_pdu")?, eventid_pduid: db.open_tree("eventid_pduid")?, + roomstateid_pduid: db.open_tree("roomstateid_pduid")?, roomid_pduleaves: db.open_tree("roomid_pduleaves")?, - roomstateid_pdu: db.open_tree("roomstateid_pdu")?, alias_roomid: db.open_tree("alias_roomid")?, aliasid_alias: db.open_tree("alias_roomid")?, @@ -111,6 +111,9 @@ impl Database { userroomid_invited: db.open_tree("userroomid_invited")?, roomuserid_invited: db.open_tree("roomuserid_invited")?, userroomid_left: db.open_tree("userroomid_left")?, + + stateid_pduid: db.open_tree("stateid_pduid")?, + pduid_statehash: db.open_tree("pduid_statehash")?, }, account_data: account_data::AccountData { roomuserdataid_accountdata: db.open_tree("roomuserdataid_accountdata")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index d2cd5e9..0d36326 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -9,7 +9,7 @@ use ruma::{ events::{ ignored_user_list, room::{ - join_rules, member, + member, power_levels::{self, PowerLevelsEventContent}, redaction, }, @@ -18,19 +18,31 @@ use ruma::{ EventId, Raw, RoomAliasId, RoomId, UserId, }; use sled::IVec; +use state_res::{event_auth, Requester, StateEvent, StateMap, StateStore}; + use std::{ - collections::{BTreeMap, HashMap}, + collections::{hash_map::DefaultHasher, BTreeMap, HashMap}, convert::{TryFrom, TryInto}, + hash::{Hash, Hasher}, mem, + result::Result as StdResult, }; +/// The unique identifier of each state group. +/// +/// This is created when a state group is added to the database by +/// hashing the entire state. +pub type StateHashId = String; + +/// This identifier consists of roomId + count. It represents a +/// unique event, it will never be overwritten or removed. +pub type PduId = IVec; + pub struct Rooms { pub edus: edus::RoomEdus, pub(super) pduid_pdu: sled::Tree, // PduId = RoomId + Count pub(super) eventid_pduid: sled::Tree, pub(super) roomid_pduleaves: sled::Tree, - pub(super) roomstateid_pdu: sled::Tree, // RoomStateId = Room + StateType + StateKey - pub(super) alias_roomid: sled::Tree, pub(super) aliasid_alias: sled::Tree, // AliasId = RoomId + Count pub(super) publicroomids: sled::Tree, @@ -42,9 +54,263 @@ pub struct Rooms { pub(super) userroomid_invited: sled::Tree, pub(super) roomuserid_invited: sled::Tree, pub(super) userroomid_left: sled::Tree, + + // STATE TREES + /// This holds the full current state, including the latest event. + pub(super) roomstateid_pduid: sled::Tree, // RoomStateId = Room + StateType + StateKey + /// This holds the full room state minus the latest event. + pub(super) pduid_statehash: sled::Tree, // PDU id -> StateHash + /// Also holds the full room state minus the latest event. + pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + (EventType, StateKey) } +impl StateStore for Rooms { + fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> StdResult { + let pid = self + .eventid_pduid + .get(event_id.as_bytes()) + .map_err(|e| e.to_string())? + .ok_or_else(|| "PDU via room_id and event_id not found in the db.".to_owned())?; + + utils::deserialize( + &self + .pduid_pdu + .get(pid) + .map_err(|e| e.to_string())? + .ok_or_else(|| "PDU via pduid not found in db.".to_owned())?, + ) + .and_then(|pdu: StateEvent| { + // conduit's PDU's always contain a room_id but some + // of ruma's do not so this must be an Option + if pdu.room_id() == Some(room_id) { + Ok(pdu) + } else { + Err(Error::bad_database("Found PDU for incorrect room in db.")) + } + }) + .map_err(|e| e.to_string()) + } +} + +// These are the methods related to STATE resolution. impl Rooms { + /// Generates a new StateHash and associates it with the incoming event. + /// + /// This adds all current state events (not including the incoming event) + /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. + /// The incoming event is the `pdu_id` passed to this method. + pub fn append_state_pdu(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result { + let state_hash = self.new_state_hash_id(room_id)?; + let state = self.current_state_pduids(room_id)?; + + let mut key = state_hash.as_bytes().to_vec(); + key.push(0xff); + + // TODO eventually we could avoid writing to the DB so much on every event + // by keeping track of the delta and write that every so often + for ((ev_ty, state_key), pid) in state { + let mut state_id = key.to_vec(); + state_id.extend_from_slice(ev_ty.to_string().as_bytes()); + key.push(0xff); + state_id.extend_from_slice(state_key.expect("state event").as_bytes()); + key.push(0xff); + + self.stateid_pduid.insert(&state_id, &pid)?; + } + + // This event's state does not include the event itself. `current_state_pduids` + // uses `roomstateid_pduid` before the current event is inserted to the tree so the state + // will be everything up to but not including the incoming event. + self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + + Ok(state_hash) + } + + /// Builds a `StateMap` by iterating over all keys that start + /// with `state_hash`, this gives the full state at event "x". + pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { + self.stateid_pduid + .scan_prefix(state_hash.as_bytes()) + .values() + .map(|pduid| { + self.pduid_pdu.get(&pduid?)?.map_or_else( + || Err(Error::bad_database("Failed to find StateMap.")), + |b| { + serde_json::from_slice::(&b) + .map_err(|_| Error::bad_database("Invalid PDU in db.")) + }, + ) + }) + .map(|pdu| { + let pdu = pdu?; + Ok(((pdu.kind, pdu.state_key), pdu.event_id)) + }) + .collect::>>() + } + + // TODO make this return Result + /// Fetches the previous StateHash ID to `current`. + pub fn prev_state_hash(&self, current: StateHashId) -> Option { + let mut found = false; + for pair in self.pduid_statehash.iter().rev() { + let prev = utils::string_from_bytes(&pair.ok()?.1).ok()?; + if current == prev { + found = true; + } + if current != prev && found { + return Some(prev); + } + } + None + } + + /// Fetch the current State using the `roomstateid_pduid` tree. + pub fn current_state_pduids(&self, room_id: &RoomId) -> Result> { + // TODO this could also scan roomstateid_pduid if we passed in room_id ? + self.roomstateid_pduid + .scan_prefix(room_id.as_bytes()) + .values() + .map(|pduid| { + let pduid = &pduid?; + self.pduid_pdu.get(pduid)?.map_or_else( + || { + Err(Error::bad_database( + "Failed to find current state of pduid's.", + )) + }, + |b| { + Ok(( + serde_json::from_slice::(&b) + .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + pduid.clone(), + )) + }, + ) + }) + .map(|pair| { + let (pdu, id) = pair?; + Ok(((pdu.kind, pdu.state_key), id)) + }) + .collect::>>() + } + + /// Returns the last state hash key added to the db. + pub fn current_state_hash(&self, room_id: &RoomId) -> Result { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + + // We must check here because this method is called outside and before + // `append_state_pdu` so the DB can be empty + if self.pduid_statehash.scan_prefix(prefix).next().is_none() { + // TODO use ring crate to hash + return Ok(room_id.as_str().to_owned()); + } + + self.pduid_statehash + .iter() + .next_back() + .map(|pair| { + utils::string_from_bytes(&pair?.1) + .map_err(|_| Error::bad_database("Invalid state hash string in db.")) + }) + .ok_or_else(|| Error::bad_database("No PDU's found for this room."))? + } + + /// This fetches auth event_ids from the current state using the + /// full `roomstateid_pdu` tree. + pub fn get_auth_event_ids( + &self, + room_id: &RoomId, + kind: &EventType, + sender: &UserId, + state_key: Option<&str>, + content: serde_json::Value, + ) -> Result> { + let auth_events = state_res::auth_types_for_event( + kind.clone(), + sender, + state_key.map(|s| s.to_string()), + content, + ); + + let mut events = vec![]; + for (event_type, state_key) in auth_events { + if let Some(state_key) = state_key.as_ref() { + if let Some(id) = self.room_state_get(room_id, &event_type, state_key)? { + events.push(id.event_id); + } + } + } + Ok(events) + } + + // This fetches auth events from the current state using the + /// full `roomstateid_pdu` tree. + pub fn get_auth_events( + &self, + room_id: &RoomId, + kind: &EventType, + sender: &UserId, + state_key: Option<&str>, + content: serde_json::Value, + ) -> Result> { + let auth_events = state_res::auth_types_for_event( + kind.clone(), + sender, + state_key.map(|s| s.to_string()), + content, + ); + + let mut events = StateMap::new(); + for (event_type, state_key) in auth_events { + if let Some(s_key) = state_key.as_ref() { + if let Some(pdu) = self.room_state_get(room_id, &event_type, s_key)? { + events.insert((event_type, state_key), pdu); + } + } + } + Ok(events) + } + + /// Generate a new StateHash. + /// + /// A unique hash made from hashing the current states pduid's. + /// Because `append_state_pdu` handles the empty state db case it does not + /// have to be here. + fn new_state_hash_id(&self, room_id: &RoomId) -> Result { + // Use hashed roomId as the first StateHash key for first state event in room + if self + .pduid_statehash + .scan_prefix(room_id.as_bytes()) + .next() + .is_none() + { + // TODO use ring crate to hash + return Ok(room_id.as_str().to_owned()); + } + + let pdu_ids_to_hash = self + .pduid_statehash + .scan_prefix(room_id.as_bytes()) + .values() + .next_back() + .unwrap() // We just checked if the tree was empty + .map(|hash| { + self.stateid_pduid + .scan_prefix(hash) + .values() + // pduid is roomId + count so just hash the whole thing + .map(|pid| Ok(pid?.to_vec())) + .collect::>>>() + })??; + + let mut hasher = DefaultHasher::new(); + pdu_ids_to_hash.hash(&mut hasher); + let hash = hasher.finish().to_string(); + // TODO not sure how you want to hash this + Ok(hash) + } + /// Checks if a room exists. pub fn exists(&self, room_id: &RoomId) -> Result { let mut prefix = room_id.to_string().as_bytes().to_vec(); @@ -64,16 +330,20 @@ impl Rooms { room_id: &RoomId, ) -> Result> { let mut hashmap = HashMap::new(); - for pdu in self - .roomstateid_pdu - .scan_prefix(&room_id.to_string().as_bytes()) - .values() - .map(|value| { - Ok::<_, Error>( - serde_json::from_slice::(&value?) + for pdu in + self.roomstateid_pduid + .scan_prefix(&room_id.to_string().as_bytes()) + .values() + .map(|value| { + Ok::<_, Error>( + serde_json::from_slice::( + &self.pduid_pdu.get(value?)?.ok_or_else(|| { + Error::bad_database("PDU not found for ID in db.") + })?, + ) .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - ) - }) + ) + }) { let pdu = pdu?; let state_key = pdu.state_key.clone().ok_or_else(|| { @@ -95,16 +365,20 @@ impl Rooms { prefix.extend_from_slice(&event_type.to_string().as_bytes()); let mut hashmap = HashMap::new(); - for pdu in self - .roomstateid_pdu - .scan_prefix(&prefix) - .values() - .map(|value| { - Ok::<_, Error>( - serde_json::from_slice::(&value?) + for pdu in + self.roomstateid_pduid + .scan_prefix(&prefix) + .values() + .map(|value| { + Ok::<_, Error>( + serde_json::from_slice::( + &self.pduid_pdu.get(value?)?.ok_or_else(|| { + Error::bad_database("PDU not found for ID in db.") + })?, + ) .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - ) - }) + ) + }) { let pdu = pdu?; let state_key = pdu.state_key.clone().ok_or_else(|| { @@ -115,23 +389,28 @@ impl Rooms { Ok(hashmap) } - /// Returns the full room state. + /// Returns a single PDU in `room_id` with key (`event_type`, `state_key`). pub fn room_state_get( &self, room_id: &RoomId, event_type: &EventType, state_key: &str, ) -> Result> { - let mut key = room_id.to_string().as_bytes().to_vec(); + let mut key = room_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(&event_type.to_string().as_bytes()); key.push(0xff); key.extend_from_slice(&state_key.as_bytes()); - self.roomstateid_pdu.get(&key)?.map_or(Ok(None), |value| { + self.roomstateid_pduid.get(&key)?.map_or(Ok(None), |value| { Ok::<_, Error>(Some( - serde_json::from_slice::(&value) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + serde_json::from_slice::( + &self + .pduid_pdu + .get(value)? + .ok_or_else(|| Error::bad_database("PDU not found for ID in db."))?, + ) + .map_err(|_| Error::bad_database("Invalid PDU in db."))?, )) }) } @@ -139,7 +418,7 @@ impl Rooms { /// Returns the `count` of this pdu's id. pub fn get_pdu_count(&self, event_id: &EventId) -> Result> { self.eventid_pduid - .get(event_id.to_string().as_bytes())? + .get(event_id.as_bytes())? .map_or(Ok(None), |pdu_id| { Ok(Some( utils::u64_from_bytes( @@ -153,7 +432,7 @@ impl Rooms { /// Returns the json of a pdu. pub fn get_pdu_json(&self, event_id: &EventId) -> Result> { self.eventid_pduid - .get(event_id.to_string().as_bytes())? + .get(event_id.as_bytes())? .map_or(Ok(None), |pdu_id| { Ok(Some( serde_json::from_slice(&self.pduid_pdu.get(pdu_id)?.ok_or_else(|| { @@ -174,7 +453,7 @@ impl Rooms { /// Returns the pdu. pub fn get_pdu(&self, event_id: &EventId) -> Result> { self.eventid_pduid - .get(event_id.to_string().as_bytes())? + .get(event_id.as_bytes())? .map_or(Ok(None), |pdu_id| { Ok(Some( serde_json::from_slice(&self.pduid_pdu.get(pdu_id)?.ok_or_else(|| { @@ -238,16 +517,15 @@ impl Rooms { /// Replace the leaves of a room with a new event. pub fn replace_pdu_leaves(&self, room_id: &RoomId, event_id: &EventId) -> Result<()> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); for key in self.roomid_pduleaves.scan_prefix(&prefix).keys() { self.roomid_pduleaves.remove(key?)?; } - prefix.extend_from_slice(event_id.to_string().as_bytes()); - self.roomid_pduleaves - .insert(&prefix, &*event_id.to_string())?; + prefix.extend_from_slice(event_id.as_bytes()); + self.roomid_pduleaves.insert(&prefix, event_id.as_bytes())?; Ok(()) } @@ -272,6 +550,14 @@ impl Rooms { // TODO: Make sure this isn't called twice in parallel let prev_events = self.get_pdu_leaves(&room_id)?; + let auth_events = self.get_auth_events( + &room_id, + &event_type, + &sender, + state_key.as_deref(), + content.clone(), + )?; + // Is the event authorized? if let Some(state_key) = &state_key { let power_levels = self @@ -333,138 +619,24 @@ impl Rooms { // Don't allow encryption events when it's disabled !globals.encryption_disabled() } - EventType::RoomMember => { - let target_user_id = UserId::try_from(&**state_key).map_err(|_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "State key of member event does not contain user id.", - ) - })?; - - let current_membership = self - .room_state_get( - &room_id, - &EventType::RoomMember, - &target_user_id.to_string(), - )? - .map_or(Ok::<_, Error>(member::MembershipState::Leave), |pdu| { - Ok(serde_json::from_value::>( - pdu.content, - ) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| Error::bad_database("Invalid Member event in db."))? - .membership) - })?; - - let target_membership = - serde_json::from_value::>(content.clone()) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| Error::bad_database("Invalid Member event in db."))? - .membership; - - let target_power = power_levels.users.get(&target_user_id).map_or_else( - || { - if target_membership != member::MembershipState::Join { - None - } else { - Some(&power_levels.users_default) - } - }, - // If it's okay, wrap with Some(_) - Some, - ); - - let join_rules = - self.room_state_get(&room_id, &EventType::RoomJoinRules, "")? - .map_or(Ok::<_, Error>(join_rules::JoinRule::Public), |pdu| { - Ok(serde_json::from_value::< - Raw, - >(pdu.content) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| { - Error::bad_database("Database contains invalid JoinRules event") - })? - .join_rule) - })?; - - if target_membership == member::MembershipState::Join { - let mut prev_events = prev_events.iter(); - let prev_event = self - .get_pdu(prev_events.next().ok_or(Error::BadRequest( - ErrorKind::Unknown, - "Membership can't be the first event", - ))?)? - .ok_or_else(|| { - Error::bad_database("PDU leaf points to invalid event!") - })?; - if prev_event.kind == EventType::RoomCreate - && prev_event.prev_events.is_empty() - { - true - } else if sender != target_user_id { - false - } else if let member::MembershipState::Ban = current_membership { - false - } else { - join_rules == join_rules::JoinRule::Invite - && (current_membership == member::MembershipState::Join - || current_membership == member::MembershipState::Invite) - || join_rules == join_rules::JoinRule::Public - } - } else if target_membership == member::MembershipState::Invite { - if let Some(third_party_invite_json) = content.get("third_party_invite") { - if current_membership == member::MembershipState::Ban { - false - } else { - let _third_party_invite = - serde_json::from_value::( - third_party_invite_json.clone(), - ) - .map_err(|_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "ThirdPartyInvite is invalid", - ) - })?; - todo!("handle third party invites"); - } - } else if sender_membership != member::MembershipState::Join - || current_membership == member::MembershipState::Join - || current_membership == member::MembershipState::Ban - { - false - } else { - sender_power - .filter(|&p| p >= &power_levels.invite) - .is_some() - } - } else if target_membership == member::MembershipState::Leave { - if sender == target_user_id { - current_membership == member::MembershipState::Join - || current_membership == member::MembershipState::Invite - } else if sender_membership != member::MembershipState::Join - || current_membership == member::MembershipState::Ban - && sender_power.filter(|&p| p < &power_levels.ban).is_some() - { - false - } else { - sender_power.filter(|&p| p >= &power_levels.kick).is_some() - && target_power < sender_power - } - } else if target_membership == member::MembershipState::Ban { - if sender_membership != member::MembershipState::Join { - false - } else { - sender_power.filter(|&p| p >= &power_levels.ban).is_some() - && target_power < sender_power - } - } else { - false - } - } + EventType::RoomMember => event_auth::is_membership_change_allowed( + // TODO this is a bit of a hack but not sure how to have a type + // declared in `state_res` crate be + Requester { + prev_event_ids: prev_events.to_owned(), + room_id: &room_id, + content: &content, + state_key: Some(state_key.to_owned()), + sender: &sender, + }, + &auth_events + .iter() + .map(|((ty, key), pdu)| { + Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) + }) + .collect::>>()?, + ) + .ok_or(Error::Conflict("Found incoming PDU with invalid data."))?, EventType::RoomCreate => prev_events.is_empty(), // Not allow any of the following events if the sender is not joined. _ if sender_membership != member::MembershipState::Join => false, @@ -474,7 +646,7 @@ impl Rooms { >= &power_levels.state_default } } { - error!("Unauthorized"); + error!("Unauthorized {}", event_type); // Not authorized return Err(Error::BadRequest( ErrorKind::Forbidden, @@ -483,7 +655,7 @@ impl Rooms { } } else if !self.is_joined(&sender, &room_id)? { // TODO: auth rules apply to all events, not only those with a state key - error!("Unauthorized"); + error!("Unauthorized {}", event_type); return Err(Error::BadRequest( ErrorKind::Forbidden, "Event is not authorized", @@ -524,7 +696,10 @@ impl Rooms { depth: depth .try_into() .map_err(|_| Error::bad_database("Depth is invalid"))?, - auth_events: Vec::new(), + auth_events: auth_events + .into_iter() + .map(|(_, pdu)| pdu.event_id) + .collect(), redacts: redacts.clone(), unsigned, hashes: ruma::events::pdu::EventHash { @@ -564,15 +739,19 @@ impl Rooms { self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?; self.eventid_pduid - .insert(pdu.event_id.to_string(), pdu_id.clone())?; + .insert(pdu.event_id.to_string(), &*pdu_id)?; - if let Some(state_key) = pdu.state_key { - let mut key = room_id.to_string().as_bytes().to_vec(); + if let Some(state_key) = &pdu.state_key { + // We call this first because our StateHash relies on the + // state before the new event + self.append_state_pdu(&room_id, &pdu_id)?; + + let mut key = room_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(pdu.kind.to_string().as_bytes()); key.push(0xff); key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pdu.insert(key, &*pdu_json.to_string())?; + self.roomstateid_pduid.insert(key, pdu_id.as_slice())?; } match event_type { diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index cece8db..e318f43 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -2,7 +2,7 @@ use crate::{Error, Result}; use ruma::{ api::client::{ error::ErrorKind, - r0::uiaa::{AuthData, UiaaInfo}, + r0::uiaa::{IncomingAuthData, UiaaInfo}, }, DeviceId, UserId, }; @@ -26,12 +26,12 @@ impl Uiaa { &self, user_id: &UserId, device_id: &DeviceId, - auth: &AuthData, + auth: &IncomingAuthData, uiaainfo: &UiaaInfo, users: &super::users::Users, globals: &super::globals::Globals, ) -> Result<(bool, UiaaInfo)> { - if let AuthData::DirectRequest { + if let IncomingAuthData::DirectRequest { kind, session, auth_parameters, diff --git a/src/pdu.rs b/src/pdu.rs index 9936802..5485f23 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -177,6 +177,35 @@ impl PduEvent { } } +impl PduEvent { + pub fn convert_for_state_res(&self) -> Result { + serde_json::from_value(json!({ + "event_id": self.event_id, + "room_id": self.room_id, + "sender": self.sender, + "origin": self.origin, + "origin_server_ts": self.origin_server_ts, + "type": self.kind, + "content": self.content, + "state_key": self.state_key, + "prev_events": self.prev_events + .iter() + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "depth": self.depth, + "auth_events": self.auth_events + .iter() + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "redacts": self.redacts, + "unsigned": self.unsigned, + "hashes": self.hashes, + "signatures": self.signatures, + })) + .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) + } +} + /// Build the start of a PDU in order to add it to the `Database`. #[derive(Debug)] pub struct PduBuilder { diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 8d86204..80e6e58 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -1,5 +1,8 @@ use crate::Error; -use ruma::identifiers::{DeviceId, UserId}; +use ruma::{ + api::IncomingRequest, + identifiers::{DeviceId, UserId}, +}; use std::{convert::TryInto, ops::Deref}; #[cfg(feature = "conduit_bin")] @@ -16,13 +19,12 @@ use { tokio::io::AsyncReadExt, Request, State, }, - ruma::api::IncomingRequest, std::io::Cursor, }; /// This struct converts rocket requests into ruma structs by converting them into http requests /// first. -pub struct Ruma { +pub struct Ruma { pub body: T, pub sender_id: Option, pub device_id: Option>, @@ -110,7 +112,7 @@ impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { } } -impl Deref for Ruma { +impl Deref for Ruma { type Target = T; fn deref(&self) -> &Self::Target { diff --git a/src/server_server.rs b/src/server_server.rs index f48f502..e47b50a 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,14 +1,17 @@ use crate::{client_server, ConduitResult, Database, Error, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; -use ruma::api::federation::{ - directory::get_public_rooms, - discovery::{ - get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, +use ruma::api::{ + client, + federation::{ + directory::get_public_rooms, + discovery::{ + get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, + }, + transactions::send_transaction_message, }, - transactions::send_transaction_message, + OutgoingRequest, }; -use ruma::api::{client, OutgoingRequest}; use serde_json::json; use std::{ collections::BTreeMap, @@ -204,11 +207,11 @@ pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json { )] pub async fn get_public_rooms_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let Ruma { body: - get_public_rooms::v1::Request { + get_public_rooms::v1::IncomingRequest { room_network: _room_network, // TODO limit, since, @@ -229,7 +232,7 @@ pub async fn get_public_rooms_route( body: client::r0::directory::get_public_rooms_filtered::IncomingRequest { filter: None, limit, - room_network: client::r0::directory::get_public_rooms_filtered::RoomNetwork::Matrix, + room_network: ruma::directory::RoomNetwork::Matrix, server: None, since, }, @@ -268,9 +271,9 @@ pub async fn get_public_rooms_route( feature = "conduit_bin", put("/_matrix/federation/v1/send/<_>", data = "") )] -pub fn send_transaction_message_route( - db: State<'_, Database>, - body: Ruma, +pub fn send_transaction_message_route<'a>( + _db: State<'a, Database>, + body: Ruma, ) -> ConduitResult { dbg!(&*body); Ok(send_transaction_message::v1::Response { diff --git a/src/utils.rs b/src/utils.rs index 8cf1b2c..77a7d1f 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,3 +1,4 @@ +use crate::Error; use argon2::{Config, Variant}; use cmp::Ordering; use rand::prelude::*; @@ -90,3 +91,8 @@ pub fn common_elements( .all(|b| b) })) } + +pub fn deserialize<'de, T: serde::Deserialize<'de>>(val: &'de sled::IVec) -> Result { + serde_json::from_slice::(val.as_ref()) + .map_err(|_| Error::bad_database("PDU in db is invalid.")) +} From cb68bf9e0c073aa060b3fe6d9aa14bbfaebc5dae Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 18 Aug 2020 14:32:38 -0400 Subject: [PATCH 02/48] Use ring crate to generate StatHashes when saving stateid/statehash --- Cargo.lock | 1 + Cargo.toml | 2 +- src/database/rooms.rs | 20 ++++++++++++-------- src/utils.rs | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ffee8ea..faa9e89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -295,6 +295,7 @@ dependencies = [ "log", "rand", "reqwest", + "ring", "rocket", "ruma", "rust-argon2 0.8.2", diff --git a/Cargo.toml b/Cargo.toml index 4c14d71..78d8f76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ image = { version = "0.23.4", default-features = false, features = ["jpeg", "png base64 = "0.12.3" # Used to encode server public key # state-res = { path = "../../state-res" } state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0" } - +ring = "0.16.15" [features] default = ["conduit_bin"] diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0d36326..6366c8c 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -4,6 +4,7 @@ pub use edus::RoomEdus; use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; use log::error; +use ring::digest; use ruma::{ api::client::error::ErrorKind, events::{ @@ -21,9 +22,8 @@ use sled::IVec; use state_res::{event_auth, Requester, StateEvent, StateMap, StateStore}; use std::{ - collections::{hash_map::DefaultHasher, BTreeMap, HashMap}, + collections::{BTreeMap, HashMap}, convert::{TryFrom, TryInto}, - hash::{Hash, Hasher}, mem, result::Result as StdResult, }; @@ -285,8 +285,10 @@ impl Rooms { .next() .is_none() { - // TODO use ring crate to hash - return Ok(room_id.as_str().to_owned()); + return utils::string_from_bytes( + digest::digest(&digest::SHA256, room_id.as_bytes()).as_ref(), + ) + .map_err(|_| Error::bad_database("Empty state generated invalid string from hash.")); } let pdu_ids_to_hash = self @@ -304,11 +306,13 @@ impl Rooms { .collect::>>>() })??; - let mut hasher = DefaultHasher::new(); - pdu_ids_to_hash.hash(&mut hasher); - let hash = hasher.finish().to_string(); + let hash = digest::digest( + &digest::SHA256, + &pdu_ids_to_hash.into_iter().flatten().collect::>(), + ); // TODO not sure how you want to hash this - Ok(hash) + utils::string_from_bytes(hash.as_ref()) + .map_err(|_| Error::bad_database("State generated invalid string from hash.")) } /// Checks if a room exists. diff --git a/src/utils.rs b/src/utils.rs index 77a7d1f..b549153 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -94,5 +94,5 @@ pub fn common_elements( pub fn deserialize<'de, T: serde::Deserialize<'de>>(val: &'de sled::IVec) -> Result { serde_json::from_slice::(val.as_ref()) - .map_err(|_| Error::bad_database("PDU in db is invalid.")) + .map_err(|_| Error::bad_database("Found invalid bytes as PDU in db.")) } From 846a0098c182272a5669bacf0d27fa988eaa4c23 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 18 Aug 2020 16:26:03 -0400 Subject: [PATCH 03/48] Split append_pdu -> append_pdu and build_and_append Move all state event appending to append_state_pdu. --- Cargo.lock | 22 ++-- src/client_server/account.rs | 2 +- src/client_server/membership.rs | 37 ++---- src/client_server/message.rs | 2 +- src/client_server/profile.rs | 4 +- src/client_server/redact.rs | 2 +- src/client_server/room.rs | 20 +-- src/client_server/state.rs | 2 +- src/database/rooms.rs | 217 ++++++++++++++++---------------- src/pdu.rs | 28 ++++- 10 files changed, 175 insertions(+), 161 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index faa9e89..3c5d836 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,9 +116,9 @@ checksum = "4af5687fe33aec5e70ef14caac5e0d363e335e5e5d6385fb75978d0c241b1d67" [[package]] name = "async-trait" -version = "0.1.37" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caae68055714ff28740f310927e04f2eba76ff580b16fb18ed90073ee71646f7" +checksum = "6e1a4a2f97ce50c9d0282c1468816208588441492b40d813b2e0419c22c05e7f" dependencies = [ "proc-macro2", "quote", @@ -1194,9 +1194,9 @@ checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" [[package]] name = "once_cell" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" +checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" [[package]] name = "opaque-debug" @@ -1860,9 +1860,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac94b333ee2aac3284c5b8a1b7fb4dd11cba88c244e3fe33cdbd047af0eb693" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" dependencies = [ "base64 0.12.3", "log", @@ -2088,7 +2088,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res#789c8140890e076d38b23fa1147c4ff0500c0d38" +source = "git+https://github.com/ruma/state-res#4e9b428c0db50ac3a3421ced12a6fd202a1c36a3" dependencies = [ "itertools", "js_int", @@ -2281,9 +2281,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" +checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" [[package]] name = "tokio" @@ -2384,9 +2384,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe233f4227389ab7df5b32649239da7ebe0b281824b4e84b342d04d3fd8c25e" +checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" dependencies = [ "proc-macro2", "quote", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 9fa1a9c..9e52f6d 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -354,7 +354,7 @@ pub fn deactivate_route( third_party_invite: None, }; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index c04cf7f..824e871 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -1,6 +1,8 @@ use super::State; use crate::{ - client_server, pdu::PduBuilder, server_server, utils, ConduitResult, Database, Error, Ruma, + client_server, + pdu::{PduBuilder, PduEvent}, + server_server, utils, ConduitResult, Database, Error, Ruma, }; use ruma::{ api::{ @@ -142,24 +144,9 @@ pub async fn join_room_by_id_route( Error::Conflict("Found event_id in sorted events that is not in resolved state") })?; - db.rooms.append_pdu( - PduBuilder { - room_id: pdu.room_id().unwrap_or(&body.room_id).clone(), - sender: pdu.sender().clone(), - event_type: pdu.kind(), - content: pdu.content().clone(), - unsigned: Some( - pdu.unsigned() - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), - ), - state_key: pdu.state_key(), - redacts: pdu.redacts().cloned(), - }, - &db.globals, - &db.account_data, - )?; + // We do not rebuild the PDU in this case only insert to DB + db.rooms + .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; } } @@ -171,7 +158,7 @@ pub async fn join_room_by_id_route( third_party_invite: None, }; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -284,7 +271,7 @@ pub fn leave_room_route( event.membership = member::MembershipState::Leave; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -312,7 +299,7 @@ pub fn invite_user_route( let sender_id = body.sender_id.as_ref().expect("user is authenticated"); if let invite_user::InvitationRecipient::UserId { user_id } = &body.recipient { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -369,7 +356,7 @@ pub fn kick_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; // TODO: reason - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -424,7 +411,7 @@ pub fn ban_user_route( }, )?; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -470,7 +457,7 @@ pub fn unban_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 1b461d2..03832d8 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -25,7 +25,7 @@ pub fn send_message_event_route( let mut unsigned = serde_json::Map::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into()); - let event_id = db.rooms.append_pdu( + let event_id = db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 1313db7..0707b34 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -31,7 +31,7 @@ pub fn set_displayname_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -134,7 +134,7 @@ pub fn set_avatar_url_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index fc65c23..8708692 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -18,7 +18,7 @@ pub fn redact_event_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - let event_id = db.rooms.append_pdu( + let event_id = db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 589a2dc..3ee21b6 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -56,7 +56,7 @@ pub fn create_room_route( content.room_version = RoomVersionId::Version6; // 1. The room create event - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -71,7 +71,7 @@ pub fn create_room_route( )?; // 2. Let the room creator join - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -120,7 +120,7 @@ pub fn create_room_route( }) .expect("event is valid, we just created it") }; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -144,7 +144,7 @@ pub fn create_room_route( }); // 4.1 Join Rules - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -169,7 +169,7 @@ pub fn create_room_route( )?; // 4.2 History Visibility - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -187,7 +187,7 @@ pub fn create_room_route( )?; // 4.3 Guest Access - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -224,7 +224,7 @@ pub fn create_room_route( continue; } - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -243,7 +243,7 @@ pub fn create_room_route( // 6. Events implied by name and topic if let Some(name) = &body.name { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -264,7 +264,7 @@ pub fn create_room_route( } if let Some(topic) = &body.topic { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -284,7 +284,7 @@ pub fn create_room_route( // 7. Events implied by invite (and TODO: invite_3pid) for user in &body.invite { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 14cc497..2920de2 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -57,7 +57,7 @@ pub fn send_state_event_for_key_route( } } - let event_id = db.rooms.append_pdu( + let event_id = db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 6366c8c..0339b7f 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -4,6 +4,7 @@ pub use edus::RoomEdus; use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; use log::error; +// TODO if ruma-signatures re-exports `use ruma::signatures::digest;` use ring::digest; use ruma::{ api::client::error::ErrorKind, @@ -99,7 +100,13 @@ impl Rooms { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. /// The incoming event is the `pdu_id` passed to this method. - pub fn append_state_pdu(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result { + pub fn append_state_pdu( + &self, + room_id: &RoomId, + pdu_id: &[u8], + state_key: &str, + kind: &EventType, + ) -> Result { let state_hash = self.new_state_hash_id(room_id)?; let state = self.current_state_pduids(room_id)?; @@ -123,6 +130,13 @@ impl Rooms { // will be everything up to but not including the incoming event. self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + let mut key = room_id.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(kind.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(state_key.as_bytes()); + self.roomstateid_pduid.insert(key, pdu_id)?; + Ok(state_hash) } @@ -535,8 +549,92 @@ impl Rooms { } /// Creates a new persisted data unit and adds it to a room. - #[allow(clippy::blocks_in_if_conditions)] pub fn append_pdu( + &self, + pdu: PduEvent, + globals: &super::globals::Globals, + account_data: &super::account_data::AccountData, + ) -> Result { + let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); + ruma::signatures::hash_and_sign_event( + globals.server_name().as_str(), + globals.keypair(), + &mut pdu_json, + ) + .expect("event is valid, we just created it"); + + self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?; + + // Increment the last index and use that + // This is also the next_batch/since value + let index = globals.next_count()?; + + let mut pdu_id = pdu.room_id.as_bytes().to_vec(); + pdu_id.push(0xff); + pdu_id.extend_from_slice(&index.to_be_bytes()); + + self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?; + + self.eventid_pduid + .insert(pdu.event_id.as_bytes(), &*pdu_id)?; + + if let Some(state_key) = &pdu.state_key { + self.append_state_pdu(&pdu.room_id, &pdu_id, state_key, &pdu.kind)?; + } + + match pdu.kind { + EventType::RoomRedaction => { + if let Some(redact_id) = &pdu.redacts { + // TODO: Reason + let _reason = serde_json::from_value::>( + pdu.content, + ) + .expect("Raw::from_value always works.") + .deserialize() + .map_err(|_| { + Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid redaction event content.", + ) + })? + .reason; + + self.redact_pdu(&redact_id)?; + } + } + EventType::RoomMember => { + if let Some(state_key) = &pdu.state_key { + // if the state_key fails + let target_user_id = UserId::try_from(state_key.as_str()) + .expect("This state_key was previously validated"); + // Update our membership info, we do this here incase a user is invited + // and immediately leaves we need the DB to record the invite event for auth + self.update_membership( + &pdu.room_id, + &target_user_id, + serde_json::from_value::(pdu.content).map_err( + |_| { + Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid redaction event content.", + ) + }, + )?, + &pdu.sender, + account_data, + globals, + )?; + } + } + _ => {} + } + self.edus.room_read_set(&pdu.room_id, &pdu.sender, index)?; + + Ok(pdu.event_id) + } + + /// Creates a new persisted data unit and adds it to a room. + pub fn build_and_append_pdu( &self, pdu_builder: PduBuilder, globals: &super::globals::Globals, @@ -618,6 +716,7 @@ impl Rooms { ); // Is the event allowed? + #[allow(clippy::blocks_in_if_conditions)] if !match event_type { EventType::RoomEncryption => { // Don't allow encryption events when it's disabled @@ -687,15 +786,15 @@ impl Rooms { let mut pdu = PduEvent { event_id: EventId::try_from("$thiswillbefilledinlater").expect("we know this is valid"), - room_id: room_id.clone(), - sender: sender.clone(), + room_id, + sender, origin: globals.server_name().to_owned(), origin_server_ts: utils::millis_since_unix_epoch() .try_into() .expect("time is valid"), - kind: event_type.clone(), - content: content.clone(), - state_key: state_key.clone(), + kind: event_type, + content, + state_key, prev_events, depth: depth .try_into() @@ -704,7 +803,7 @@ impl Rooms { .into_iter() .map(|(_, pdu)| pdu.event_id) .collect(), - redacts: redacts.clone(), + redacts, unsigned, hashes: ruma::events::pdu::EventHash { sha256: "aaa".to_owned(), @@ -722,105 +821,7 @@ impl Rooms { )) .expect("ruma's reference hashes are valid event ids"); - let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); - ruma::signatures::hash_and_sign_event( - globals.server_name().as_str(), - globals.keypair(), - &mut pdu_json, - ) - .expect("event is valid, we just created it"); - - self.replace_pdu_leaves(&room_id, &pdu.event_id)?; - - // Increment the last index and use that - // This is also the next_batch/since value - let index = globals.next_count()?; - - let mut pdu_id = room_id.to_string().as_bytes().to_vec(); - pdu_id.push(0xff); - pdu_id.extend_from_slice(&index.to_be_bytes()); - - self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?; - - self.eventid_pduid - .insert(pdu.event_id.to_string(), &*pdu_id)?; - - if let Some(state_key) = &pdu.state_key { - // We call this first because our StateHash relies on the - // state before the new event - self.append_state_pdu(&room_id, &pdu_id)?; - - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(pdu.kind.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pduid.insert(key, pdu_id.as_slice())?; - } - - match event_type { - EventType::RoomRedaction => { - if let Some(redact_id) = &redacts { - // TODO: Reason - let _reason = - serde_json::from_value::>(content) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid redaction event content.", - ) - })? - .reason; - - self.redact_pdu(&redact_id)?; - } - } - EventType::RoomMember => { - if let Some(state_key) = state_key { - // if the state_key fails - let target_user_id = UserId::try_from(state_key) - .expect("This state_key was previously validated"); - // Update our membership info, we do this here incase a user is invited - // and immediately leaves we need the DB to record the invite event for auth - self.update_membership( - &room_id, - &target_user_id, - serde_json::from_value::(content).map_err( - |_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid redaction event content.", - ) - }, - )?, - &sender, - account_data, - globals, - )?; - } - } - EventType::RoomMessage => { - if let Some(body) = content.get("body").and_then(|b| b.as_str()) { - for word in body - .split_terminator(|c: char| !c.is_alphanumeric()) - .map(str::to_lowercase) - { - let mut key = room_id.to_string().as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(word.as_bytes()); - key.push(0xff); - key.extend_from_slice(&pdu_id); - self.tokenids.insert(key, &[])?; - } - } - } - _ => {} - } - self.edus.room_read_set(&room_id, &sender, index)?; - - Ok(pdu.event_id) + self.append_pdu(pdu, globals, account_data) } /// Returns an iterator over all PDUs in a room. @@ -999,7 +1000,7 @@ impl Rooms { if is_ignored { member_content.membership = member::MembershipState::Leave; - self.append_pdu( + self.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: user_id.clone(), diff --git a/src/pdu.rs b/src/pdu.rs index 5485f23..eec8e49 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -9,7 +9,7 @@ use ruma::{ }; use serde::{Deserialize, Serialize}; use serde_json::json; -use std::collections::HashMap; +use std::{collections::HashMap, convert::TryFrom}; #[derive(Deserialize, Serialize)] pub struct PduEvent { @@ -177,6 +177,30 @@ impl PduEvent { } } +impl TryFrom<&state_res::StateEvent> for PduEvent { + type Error = Error; + fn try_from(pdu: &state_res::StateEvent) -> Result { + serde_json::from_value(json!({ + "event_id": pdu.event_id(), + "room_id": pdu.room_id(), + "sender": pdu.sender(), + "origin": pdu.origin(), + "origin_server_ts": pdu.origin_server_ts(), + "event_type": pdu.kind(), + "content": pdu.content(), + "state_key": pdu.state_key(), + "prev_events": pdu.prev_event_ids(), + "depth": pdu.depth(), + "auth_events": pdu.auth_events(), + "redacts": pdu.redacts(), + "unsigned": pdu.unsigned(), + "hashes": pdu.hashes(), + "signatures": pdu.signatures(), + })) + .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) + } +} + impl PduEvent { pub fn convert_for_state_res(&self) -> Result { serde_json::from_value(json!({ @@ -190,11 +214,13 @@ impl PduEvent { "state_key": self.state_key, "prev_events": self.prev_events .iter() + // TODO How do we create one of these .map(|id| (id, EventHash { sha256: "hello".into() })) .collect::>(), "depth": self.depth, "auth_events": self.auth_events .iter() + // TODO How do we create one of these .map(|id| (id, EventHash { sha256: "hello".into() })) .collect::>(), "redacts": self.redacts, From d73c6aa8addcaac5f13cbaa009960659a4365bb2 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 19 Aug 2020 17:27:24 -0400 Subject: [PATCH 04/48] Add roomid_statehash tree, clean up review issues --- src/client_server/membership.rs | 10 +--- src/database.rs | 1 + src/database/rooms.rs | 95 +++++++++++++++++---------------- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 824e871..555291e 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -112,11 +112,7 @@ pub async fn join_room_by_id_route( .room_state .state .iter() - .map(|pdu| pdu.deserialize().map(StateEvent::Full)) - .map(|ev| { - let ev = ev?; - Ok::<_, serde_json::Error>((ev.event_id(), ev)) - }) + .map(|pdu| pdu.deserialize().map(StateEvent::Full).map(|ev| (ev.event_id(), ev))) .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; @@ -140,9 +136,7 @@ pub async fn join_room_by_id_route( for ev_id in &sorted_events_ids { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` - let pdu = event_map.get(ev_id).ok_or_else(|| { - Error::Conflict("Found event_id in sorted events that is not in resolved state") - })?; + let pdu = event_map.get(ev_id).expect("Found event_id in sorted events that is not in resolved state"); // We do not rebuild the PDU in this case only insert to DB db.rooms diff --git a/src/database.rs b/src/database.rs index 6cd65c3..a105058 100644 --- a/src/database.rs +++ b/src/database.rs @@ -114,6 +114,7 @@ impl Database { stateid_pduid: db.open_tree("stateid_pduid")?, pduid_statehash: db.open_tree("pduid_statehash")?, + roomid_statehash: db.open_tree("roomid_statehash")?, }, account_data: account_data::AccountData { roomuserdataid_accountdata: db.open_tree("roomuserdataid_accountdata")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0339b7f..6273005 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -63,6 +63,8 @@ pub struct Rooms { pub(super) pduid_statehash: sled::Tree, // PDU id -> StateHash /// Also holds the full room state minus the latest event. pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + (EventType, StateKey) + /// The room_id -> the latest StateHash + pub(super) roomid_statehash: sled::Tree, } impl StateStore for Rooms { @@ -93,53 +95,7 @@ impl StateStore for Rooms { } } -// These are the methods related to STATE resolution. impl Rooms { - /// Generates a new StateHash and associates it with the incoming event. - /// - /// This adds all current state events (not including the incoming event) - /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. - /// The incoming event is the `pdu_id` passed to this method. - pub fn append_state_pdu( - &self, - room_id: &RoomId, - pdu_id: &[u8], - state_key: &str, - kind: &EventType, - ) -> Result { - let state_hash = self.new_state_hash_id(room_id)?; - let state = self.current_state_pduids(room_id)?; - - let mut key = state_hash.as_bytes().to_vec(); - key.push(0xff); - - // TODO eventually we could avoid writing to the DB so much on every event - // by keeping track of the delta and write that every so often - for ((ev_ty, state_key), pid) in state { - let mut state_id = key.to_vec(); - state_id.extend_from_slice(ev_ty.to_string().as_bytes()); - key.push(0xff); - state_id.extend_from_slice(state_key.expect("state event").as_bytes()); - key.push(0xff); - - self.stateid_pduid.insert(&state_id, &pid)?; - } - - // This event's state does not include the event itself. `current_state_pduids` - // uses `roomstateid_pduid` before the current event is inserted to the tree so the state - // will be everything up to but not including the incoming event. - self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; - - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(kind.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pduid.insert(key, pdu_id)?; - - Ok(state_hash) - } - /// Builds a `StateMap` by iterating over all keys that start /// with `state_hash`, this gives the full state at event "x". pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { @@ -633,6 +589,53 @@ impl Rooms { Ok(pdu.event_id) } + /// Generates a new StateHash and associates it with the incoming event. + /// + /// This adds all current state events (not including the incoming event) + /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. + /// The incoming event is the `pdu_id` passed to this method. + fn append_state_pdu( + &self, + room_id: &RoomId, + pdu_id: &[u8], + state_key: &str, + kind: &EventType, + ) -> Result { + let state_hash = self.new_state_hash_id(room_id)?; + let state = self.current_state_pduids(room_id)?; + + let mut key = state_hash.as_bytes().to_vec(); + key.push(0xff); + + // TODO eventually we could avoid writing to the DB so much on every event + // by keeping track of the delta and write that every so often + for ((ev_ty, state_key), pid) in state { + let mut state_id = key.to_vec(); + state_id.extend_from_slice(ev_ty.to_string().as_bytes()); + key.push(0xff); + state_id.extend_from_slice(state_key.expect("state event").as_bytes()); + key.push(0xff); + + self.stateid_pduid.insert(&state_id, &pid)?; + } + + // This event's state does not include the event itself. `current_state_pduids` + // uses `roomstateid_pduid` before the current event is inserted to the tree so the state + // will be everything up to but not including the incoming event. + self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + + self.roomid_statehash.insert(room_id.as_bytes(), state_hash.as_bytes())?; + + let mut key = room_id.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(kind.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(state_key.as_bytes()); + self.roomstateid_pduid.insert(key, pdu_id)?; + + Ok(state_hash) + } + /// Creates a new persisted data unit and adds it to a room. pub fn build_and_append_pdu( &self, From 64fb0374b60461ad1b7600e92032dfc1e30b39d3 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 19 Aug 2020 19:29:39 -0400 Subject: [PATCH 05/48] Use Vec instead of string for digest bytes and add roomid_statehash --- src/database/rooms.rs | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 6273005..c8ff198 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -33,7 +33,7 @@ use std::{ /// /// This is created when a state group is added to the database by /// hashing the entire state. -pub type StateHashId = String; +pub type StateHashId = Vec; /// This identifier consists of roomId + count. It represents a /// unique event, it will never be overwritten or removed. @@ -100,7 +100,7 @@ impl Rooms { /// with `state_hash`, this gives the full state at event "x". pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { self.stateid_pduid - .scan_prefix(state_hash.as_bytes()) + .scan_prefix(&state_hash) .values() .map(|pduid| { self.pduid_pdu.get(&pduid?)?.map_or_else( @@ -123,12 +123,12 @@ impl Rooms { pub fn prev_state_hash(&self, current: StateHashId) -> Option { let mut found = false; for pair in self.pduid_statehash.iter().rev() { - let prev = utils::string_from_bytes(&pair.ok()?.1).ok()?; - if current == prev { + let prev = pair.ok()?.1; + if current == prev.as_ref() { found = true; } - if current != prev && found { - return Some(prev); + if current != prev.as_ref() && found { + return Some(prev.to_vec()); } } None @@ -172,17 +172,14 @@ impl Rooms { // We must check here because this method is called outside and before // `append_state_pdu` so the DB can be empty if self.pduid_statehash.scan_prefix(prefix).next().is_none() { - // TODO use ring crate to hash - return Ok(room_id.as_str().to_owned()); + // return the hash of the room_id, this represents a room with no state + return self.new_state_hash_id(room_id); } self.pduid_statehash .iter() .next_back() - .map(|pair| { - utils::string_from_bytes(&pair?.1) - .map_err(|_| Error::bad_database("Invalid state hash string in db.")) - }) + .map(|pair| Ok(pair?.1.to_vec())) .ok_or_else(|| Error::bad_database("No PDU's found for this room."))? } @@ -255,10 +252,9 @@ impl Rooms { .next() .is_none() { - return utils::string_from_bytes( - digest::digest(&digest::SHA256, room_id.as_bytes()).as_ref(), - ) - .map_err(|_| Error::bad_database("Empty state generated invalid string from hash.")); + return Ok(digest::digest(&digest::SHA256, room_id.as_bytes()) + .as_ref() + .to_vec()); } let pdu_ids_to_hash = self @@ -280,9 +276,7 @@ impl Rooms { &digest::SHA256, &pdu_ids_to_hash.into_iter().flatten().collect::>(), ); - // TODO not sure how you want to hash this - utils::string_from_bytes(hash.as_ref()) - .map_err(|_| Error::bad_database("State generated invalid string from hash.")) + Ok(hash.as_ref().to_vec()) } /// Checks if a room exists. @@ -604,7 +598,7 @@ impl Rooms { let state_hash = self.new_state_hash_id(room_id)?; let state = self.current_state_pduids(room_id)?; - let mut key = state_hash.as_bytes().to_vec(); + let mut key = state_hash.to_vec(); key.push(0xff); // TODO eventually we could avoid writing to the DB so much on every event @@ -622,9 +616,10 @@ impl Rooms { // This event's state does not include the event itself. `current_state_pduids` // uses `roomstateid_pduid` before the current event is inserted to the tree so the state // will be everything up to but not including the incoming event. - self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + self.pduid_statehash.insert(pdu_id, state_hash.as_slice())?; - self.roomid_statehash.insert(room_id.as_bytes(), state_hash.as_bytes())?; + self.roomid_statehash + .insert(room_id.as_bytes(), state_hash.as_slice())?; let mut key = room_id.as_bytes().to_vec(); key.push(0xff); From ea2003240400a5870f1f755d8a13bfb618101aed Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 19 Aug 2020 19:30:28 -0400 Subject: [PATCH 06/48] Helper for join_room_by_id route so routes aren't calling routes --- src/client_server/membership.rs | 325 ++++++++++++++++---------------- 1 file changed, 163 insertions(+), 162 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 555291e..0d9fa12 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -13,14 +13,14 @@ use ruma::{ membership::{ ban_user, forget_room, get_member_events, invite_user, join_room_by_id, join_room_by_id_or_alias, joined_members, joined_rooms, kick_user, leave_room, - unban_user, + unban_user, IncomingThirdPartySigned, }, }, }, federation, }, events::{room::member, EventType}, - EventId, Raw, RoomId, RoomVersionId, + EventId, Raw, RoomId, RoomVersionId, UserId, }; use state_res::StateEvent; @@ -37,136 +37,13 @@ pub async fn join_room_by_id_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - - // Ask a remote server if we don't have this room - if !db.rooms.exists(&body.room_id)? && body.room_id.server_name() != db.globals.server_name() { - let make_join_response = server_server::send_request( - &db, - body.room_id.server_name().to_string(), - federation::membership::create_join_event_template::v1::Request { - room_id: body.room_id.clone(), - user_id: sender_id.clone(), - ver: vec![RoomVersionId::Version5, RoomVersionId::Version6], - }, - ) - .await?; - - let mut join_event_stub_value = - serde_json::from_str::(make_join_response.event.json().get()) - .map_err(|_| { - Error::BadServerResponse("Invalid make_join event json received from server.") - })?; - - let join_event_stub = - join_event_stub_value - .as_object_mut() - .ok_or(Error::BadServerResponse( - "Invalid make join event object received from server.", - ))?; - - join_event_stub.insert( - "origin".to_owned(), - db.globals.server_name().to_owned().to_string().into(), - ); - join_event_stub.insert( - "origin_server_ts".to_owned(), - utils::millis_since_unix_epoch().into(), - ); - - // Generate event id - let event_id = EventId::try_from(&*format!( - "${}", - ruma::signatures::reference_hash(&join_event_stub_value) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - // We don't leave the event id into the pdu because that's only allowed in v1 or v2 rooms - let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); - join_event_stub.remove("event_id"); - - ruma::signatures::hash_and_sign_event( - db.globals.server_name().as_str(), - db.globals.keypair(), - &mut join_event_stub_value, - ) - .expect("event is valid, we just created it"); - - let send_join_response = server_server::send_request( - &db, - body.room_id.server_name().to_string(), - federation::membership::create_join_event::v1::Request { - room_id: body.room_id.clone(), - event_id, - pdu_stub: serde_json::from_value(join_event_stub_value) - .expect("Raw::from_value always works"), - }, - ) - .await?; - - dbg!(&send_join_response); - // todo!("Take send_join_response and 'create' the room using that data"); - - let mut event_map = send_join_response - .room_state - .state - .iter() - .map(|pdu| pdu.deserialize().map(StateEvent::Full).map(|ev| (ev.event_id(), ev))) - .collect::, _>>() - .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - - let _auth_chain = send_join_response - .room_state - .auth_chain - .iter() - .flat_map(|pdu| pdu.deserialize().ok()) - .map(StateEvent::Full) - .collect::>(); - - // TODO make StateResolution's methods free functions ? or no self param ? - let sorted_events_ids = state_res::StateResolution::default() - .reverse_topological_power_sort( - &body.room_id, - &event_map.keys().cloned().collect::>(), - &mut event_map, - &db.rooms, - &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort - ); - - for ev_id in &sorted_events_ids { - // this is a `state_res::StateEvent` that holds a `ruma::Pdu` - let pdu = event_map.get(ev_id).expect("Found event_id in sorted events that is not in resolved state"); - - // We do not rebuild the PDU in this case only insert to DB - db.rooms - .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; - } - } - - let event = member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }; - - db.rooms.build_and_append_pdu( - PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; - - Ok(join_room_by_id::Response::new(body.room_id.clone()).into()) + join_room_by_id_helper( + &db, + body.sender_id.as_ref(), + &body.room_id, + body.third_party_signed.as_ref(), + ) + .await } #[cfg_attr( @@ -185,7 +62,7 @@ pub async fn join_room_by_id_or_alias_route( db, Ruma { body: alias::get_alias::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ "room_alias": room_alias }) + serde_json::json!({ "room_alias": room_alias, }) .to_string() .as_bytes() .to_vec(), @@ -202,36 +79,16 @@ pub async fn join_room_by_id_or_alias_route( } }; - // TODO ruma needs to implement the same constructors for the Incoming variants - let tps = if let Some(in_tps) = &body.third_party_signed { - Some(ruma::api::client::r0::membership::ThirdPartySigned { - token: &in_tps.token, - sender: &in_tps.sender, - signatures: in_tps.signatures.clone(), - mxid: &in_tps.mxid, - }) - } else { - None - }; - - let body = Ruma { - sender_id: body.sender_id.clone(), - device_id: body.device_id.clone(), - json_body: None, - body: join_room_by_id::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ - "room_id": room_id, - "third_party_signed": tps, - }) - .to_string() - .as_bytes() - .to_vec(), - )) - .unwrap(), - }; - Ok(join_room_by_id_or_alias::Response { - room_id: join_room_by_id_route(db2, body).await?.0.room_id, + room_id: join_room_by_id_helper( + &db2, + body.sender_id.as_ref(), + &room_id, + body.third_party_signed.as_ref(), + ) + .await? + .0 + .room_id, } .into()) } @@ -568,3 +425,147 @@ pub fn joined_members_route( Ok(joined_members::Response { joined }.into()) } + +async fn join_room_by_id_helper( + db: &Database, + sender_id: Option<&UserId>, + room_id: &RoomId, + _third_party_signed: Option<&IncomingThirdPartySigned>, +) -> ConduitResult { + let sender_id = sender_id.expect("user is authenticated"); + + // Ask a remote server if we don't have this room + if !db.rooms.exists(&room_id)? && room_id.server_name() != db.globals.server_name() { + let make_join_response = server_server::send_request( + &db, + room_id.server_name().to_string(), + federation::membership::create_join_event_template::v1::Request { + room_id: room_id.clone(), + user_id: sender_id.clone(), + ver: vec![RoomVersionId::Version5, RoomVersionId::Version6], + }, + ) + .await?; + + let mut join_event_stub_value = + serde_json::from_str::(make_join_response.event.json().get()) + .map_err(|_| { + Error::BadServerResponse("Invalid make_join event json received from server.") + })?; + + let join_event_stub = + join_event_stub_value + .as_object_mut() + .ok_or(Error::BadServerResponse( + "Invalid make join event object received from server.", + ))?; + + join_event_stub.insert( + "origin".to_owned(), + db.globals.server_name().to_owned().to_string().into(), + ); + join_event_stub.insert( + "origin_server_ts".to_owned(), + utils::millis_since_unix_epoch().into(), + ); + + // Generate event id + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&join_event_stub_value) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + // We don't leave the event id into the pdu because that's only allowed in v1 or v2 rooms + let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); + join_event_stub.remove("event_id"); + + ruma::signatures::hash_and_sign_event( + db.globals.server_name().as_str(), + db.globals.keypair(), + &mut join_event_stub_value, + ) + .expect("event is valid, we just created it"); + + let send_join_response = server_server::send_request( + &db, + room_id.server_name().to_string(), + federation::membership::create_join_event::v1::Request { + room_id: room_id.clone(), + event_id, + pdu_stub: serde_json::from_value(join_event_stub_value) + .expect("Raw::from_value always works"), + }, + ) + .await?; + + dbg!(&send_join_response); + // todo!("Take send_join_response and 'create' the room using that data"); + + let mut event_map = send_join_response + .room_state + .state + .iter() + .map(|pdu| { + pdu.deserialize() + .map(StateEvent::Full) + .map(|ev| (ev.event_id(), ev)) + }) + .collect::, _>>() + .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; + + let _auth_chain = send_join_response + .room_state + .auth_chain + .iter() + .flat_map(|pdu| pdu.deserialize().ok()) + .map(StateEvent::Full) + .collect::>(); + + // TODO make StateResolution's methods free functions ? or no self param ? + let sorted_events_ids = state_res::StateResolution::default() + .reverse_topological_power_sort( + &room_id, + &event_map.keys().cloned().collect::>(), + &mut event_map, + &db.rooms, + &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + ); + + for ev_id in &sorted_events_ids { + // this is a `state_res::StateEvent` that holds a `ruma::Pdu` + let pdu = event_map + .get(ev_id) + .expect("Found event_id in sorted events that is not in resolved state"); + + // We do not rebuild the PDU in this case only insert to DB + db.rooms + .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; + } + } + + let event = member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }; + + db.rooms.build_and_append_pdu( + PduBuilder { + room_id: room_id.clone(), + sender: sender_id.clone(), + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &db.globals, + &db.account_data, + )?; + + Ok(join_room_by_id::Response::new(room_id.clone()).into()) +} From 5ccdd3694bd86b7048d089d6a2101ae740a1d0f9 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Thu, 20 Aug 2020 12:12:02 -0400 Subject: [PATCH 07/48] Add helper function for get_alias route --- src/client_server/alias.rs | 28 +++++++++++++++++--------- src/client_server/membership.rs | 35 +++++++++------------------------ 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 7dc9078..12bb8df 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -1,11 +1,14 @@ use super::State; use crate::{server_server, ConduitResult, Database, Error, Ruma}; -use ruma::api::{ - client::{ - error::ErrorKind, - r0::alias::{create_alias, delete_alias, get_alias}, +use ruma::{ + api::{ + client::{ + error::ErrorKind, + r0::alias::{create_alias, delete_alias, get_alias}, + }, + federation, }, - federation, + RoomAliasId, }; #[cfg(feature = "conduit_bin")] @@ -50,12 +53,19 @@ pub async fn get_alias_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - if body.room_alias.server_name() != db.globals.server_name() { + get_alias_helper(db, &body.room_alias).await +} + +pub async fn get_alias_helper( + db: State<'_, Database>, + room_alias: &RoomAliasId, +) -> ConduitResult { + if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( &db, - body.room_alias.server_name().to_string(), + room_alias.server_name().to_string(), federation::query::get_room_information::v1::Request { - room_alias: body.room_alias.to_string(), + room_alias: room_alias.to_string(), }, ) .await?; @@ -65,7 +75,7 @@ pub async fn get_alias_route( let room_id = db .rooms - .id_from_alias(&body.room_alias)? + .id_from_alias(&room_alias)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "Room with alias not found.", diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 0d9fa12..0075861 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -8,13 +8,10 @@ use ruma::{ api::{ client::{ error::ErrorKind, - r0::{ - alias, - membership::{ - ban_user, forget_room, get_member_events, invite_user, join_room_by_id, - join_room_by_id_or_alias, joined_members, joined_rooms, kick_user, leave_room, - unban_user, IncomingThirdPartySigned, - }, + r0::membership::{ + ban_user, forget_room, get_member_events, invite_user, join_room_by_id, + join_room_by_id_or_alias, joined_members, joined_rooms, kick_user, leave_room, + unban_user, IncomingThirdPartySigned, }, }, federation, @@ -58,24 +55,10 @@ pub async fn join_room_by_id_or_alias_route( let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, Err(room_alias) => { - client_server::get_alias_route( - db, - Ruma { - body: alias::get_alias::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ "room_alias": room_alias, }) - .to_string() - .as_bytes() - .to_vec(), - )) - .unwrap(), - sender_id: body.sender_id.clone(), - device_id: body.device_id.clone(), - json_body: None, - }, - ) - .await? - .0 - .room_id + client_server::get_alias_helper(db, &room_alias) + .await? + .0 + .room_id } }; @@ -495,7 +478,7 @@ async fn join_room_by_id_helper( room_id: room_id.clone(), event_id, pdu_stub: serde_json::from_value(join_event_stub_value) - .expect("Raw::from_value always works"), + .expect("we just created this event"), }, ) .await?; From fe795d38ead8b83c413187f68e83ccea4185c8c7 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 21 Aug 2020 17:19:18 -0400 Subject: [PATCH 08/48] Replace route calling routes with helpers This fixes the panic from ruma "index out of bounds" --- src/client_server/alias.rs | 4 +- src/client_server/membership.rs | 20 ++--- src/client_server/state.rs | 142 ++++++++++++++++++-------------- 3 files changed, 90 insertions(+), 76 deletions(-) diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 12bb8df..669f558 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -53,11 +53,11 @@ pub async fn get_alias_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - get_alias_helper(db, &body.room_alias).await + get_alias_helper(&db, &body.room_alias).await } pub async fn get_alias_helper( - db: State<'_, Database>, + db: &Database, room_alias: &RoomAliasId, ) -> ConduitResult { if room_alias.server_name() != db.globals.server_name() { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 0075861..996d3c4 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -49,13 +49,12 @@ pub async fn join_room_by_id_route( )] pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, - db2: State<'_, Database>, body: Ruma, ) -> ConduitResult { let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, Err(room_alias) => { - client_server::get_alias_helper(db, &room_alias) + client_server::get_alias_helper(&db, &room_alias) .await? .0 .room_id @@ -64,7 +63,7 @@ pub async fn join_room_by_id_or_alias_route( Ok(join_room_by_id_or_alias::Response { room_id: join_room_by_id_helper( - &db2, + &db, body.sender_id.as_ref(), &room_id, body.third_party_signed.as_ref(), @@ -507,14 +506,13 @@ async fn join_room_by_id_helper( .collect::>(); // TODO make StateResolution's methods free functions ? or no self param ? - let sorted_events_ids = state_res::StateResolution::default() - .reverse_topological_power_sort( - &room_id, - &event_map.keys().cloned().collect::>(), - &mut event_map, - &db.rooms, - &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort - ); + let sorted_events_ids = state_res::StateResolution::reverse_topological_power_sort( + &room_id, + &event_map.keys().cloned().collect::>(), + &mut event_map, + &db.rooms, + &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + ); for ev_id in &sorted_events_ids { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 2920de2..867b051 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -9,8 +9,8 @@ use ruma::{ }, }, events::{AnyStateEventContent, EventContent}, + RoomId, UserId, }; -use std::convert::TryFrom; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; @@ -33,45 +33,14 @@ pub fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = &body.content { - let mut aliases = canonical_alias.alt_aliases.clone(); - - if let Some(alias) = canonical_alias.alias.clone() { - aliases.push(alias); - } - - for alias in aliases { - if alias.server_name() != db.globals.server_name() - || db - .rooms - .id_from_alias(&alias)? - .filter(|room| room == &body.room_id) // Make sure it's the right room - .is_none() - { - return Err(Error::BadRequest( - ErrorKind::Forbidden, - "You are only allowed to send canonical_alias \ - events when it's aliases already exists", - )); - } - } - } - - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), - event_type: body.content.event_type().into(), - content, - unsigned: None, - state_key: Some(body.state_key.clone()), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; - - Ok(send_state_event_for_key::Response::new(event_id).into()) + send_state_event_for_key_helper( + &db, + sender_id, + &body.content, + content, + &body.room_id, + Some(body.state_key.clone()), + ) } #[cfg_attr( @@ -84,34 +53,30 @@ pub fn send_state_event_for_empty_key_route( ) -> ConduitResult { // This just calls send_state_event_for_key_route let Ruma { - body: - send_state_event_for_empty_key::IncomingRequest { - room_id, content, .. - }, + body, sender_id, - device_id, + device_id: _, json_body, } = body; + let json = serde_json::from_str::( + json_body + .as_ref() + .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? + .get(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; + Ok(send_state_event_for_empty_key::Response::new( - send_state_event_for_key_route( - db, - Ruma { - body: send_state_event_for_key::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ - "room_id": room_id, - "state_key": "", - "content": content, - }) - .to_string() - .as_bytes() - .to_vec(), - )) - .unwrap(), - sender_id, - device_id, - json_body, - }, + send_state_event_for_key_helper( + &db, + sender_id + .as_ref() + .expect("no user for send state empty key rout"), + &body.content, + json, + &body.room_id, + None, )? .0 .event_id, @@ -210,3 +175,54 @@ pub fn get_state_events_for_empty_key_route( } .into()) } + +pub fn send_state_event_for_key_helper( + db: &Database, + sender: &UserId, + content: &AnyStateEventContent, + json: serde_json::Value, + room_id: &RoomId, + state_key: Option, +) -> ConduitResult { + let sender_id = sender; + + if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = content { + let mut aliases = canonical_alias.alt_aliases.clone(); + + if let Some(alias) = canonical_alias.alias.clone() { + aliases.push(alias); + } + + for alias in aliases { + if alias.server_name() != db.globals.server_name() + || db + .rooms + .id_from_alias(&alias)? + .filter(|room| room == room_id) // Make sure it's the right room + .is_none() + { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "You are only allowed to send canonical_alias \ + events when it's aliases already exists", + )); + } + } + } + + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + room_id: room_id.clone(), + sender: sender_id.clone(), + event_type: content.event_type().into(), + content: json, + unsigned: None, + state_key, + redacts: None, + }, + &db.globals, + &db.account_data, + )?; + + Ok(send_state_event_for_key::Response::new(event_id).into()) +} From 672bf4f47376eea520c19fa1b9108478ec56974f Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 21 Aug 2020 20:18:56 -0400 Subject: [PATCH 09/48] Cargo lock update and a few doc additions --- Cargo.lock | 22 +++++++++++----------- src/client_server/config.rs | 6 +----- src/client_server/room.rs | 1 + src/database/rooms.rs | 2 +- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c5d836..0e5da6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,9 +144,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" @@ -247,9 +247,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518" +checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" [[package]] name = "cfg-if" @@ -960,9 +960,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.74" +version = "0.2.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10" +checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3" [[package]] name = "lock_api" @@ -1358,9 +1358,9 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" +checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" [[package]] name = "proc-macro-crate" @@ -2088,7 +2088,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res#4e9b428c0db50ac3a3421ced12a6fd202a1c36a3" +source = "git+https://github.com/ruma/state-res#d93a965ad17781fa9554bb3cea71673c054b9f3f" dependencies = [ "itertools", "js_int", @@ -2179,9 +2179,9 @@ checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" [[package]] name = "syn" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e69abc24912995b3038597a7a593be5053eb0fb44f3cc5beec0deb421790c1f4" +checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9" dependencies = [ "proc-macro2", "quote", diff --git a/src/client_server/config.rs b/src/client_server/config.rs index 8cb6a0d..45aec33 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -56,11 +56,7 @@ pub fn get_global_account_data_route( let data = db .account_data - .get::>( - None, - sender_id, - EventType::try_from(&body.event_type).expect("EventType::try_from can never fail"), - )? + .get::>(None, sender_id, body.event_type.clone().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?; Ok(get_global_account_data::Response { account_data: data }.into()) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 3ee21b6..c0603d3 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -195,6 +195,7 @@ pub fn create_room_route( content: match preset { create_room::RoomPreset::PublicChat => { serde_json::to_value(guest_access::GuestAccessEventContent::new( + // In a public room a joining is the only way to access guest_access::GuestAccess::Forbidden, )) .expect("event is valid, we just created it") diff --git a/src/database/rooms.rs b/src/database/rooms.rs index c8ff198..0129742 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -722,7 +722,7 @@ impl Rooms { } EventType::RoomMember => event_auth::is_membership_change_allowed( // TODO this is a bit of a hack but not sure how to have a type - // declared in `state_res` crate be + // declared in `state_res` crate easily convert to/from conduit::PduEvent Requester { prev_event_ids: prev_events.to_owned(), room_id: &room_id, From 972babbc795c9bfc6f3aee7b6351c0deb7e51b5d Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 19 Aug 2020 18:26:39 +0200 Subject: [PATCH 10/48] fix: set limited to true when skipping messages in /sync --- src/client_server/config.rs | 3 +-- src/client_server/search.rs | 27 +++++++++++++-------------- src/database/rooms.rs | 19 +++++++++++++++++-- src/pdu.rs | 24 +++++++++++++++++++++++- 4 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/client_server/config.rs b/src/client_server/config.rs index 45aec33..baa9381 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -5,10 +5,9 @@ use ruma::{ error::ErrorKind, r0::config::{get_global_account_data, set_global_account_data}, }, - events::{custom::CustomEventContent, BasicEvent, EventType}, + events::{custom::CustomEventContent, BasicEvent}, Raw, }; -use std::convert::TryFrom; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; diff --git a/src/client_server/search.rs b/src/client_server/search.rs index dec1ec9..082711d 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -14,7 +14,7 @@ use std::collections::BTreeMap; )] pub fn search_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,8 @@ pub fn search_events_route( result: db .rooms .get_pdu_from_id(&result)? - .map(|pdu| pdu.to_room_event()), + // TODO this is an awkward type conversion see method + .map(|pdu| pdu.to_any_event()), }) }) .filter_map(|r| r.ok()) @@ -70,17 +71,15 @@ pub fn search_events_route( Some((skip + limit).to_string()) }; - Ok(search_events::Response { - search_categories: ResultCategories { - room_events: Some(ResultRoomEvents { - count: uint!(0), // TODO - groups: BTreeMap::new(), // TODO - next_batch, - results, - state: BTreeMap::new(), // TODO - highlights: search.1, - }), - }, - } + Ok(search_events::Response::new(ResultCategories { + room_events: Some(ResultRoomEvents { + count: uint!(0), // TODO + groups: BTreeMap::new(), // TODO + next_batch, + results, + state: BTreeMap::new(), // TODO + highlights: search.1, + }), + }) .into()) } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0129742..d087d65 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -532,7 +532,7 @@ impl Rooms { self.append_state_pdu(&pdu.room_id, &pdu_id, state_key, &pdu.kind)?; } - match pdu.kind { + match &pdu.kind { EventType::RoomRedaction => { if let Some(redact_id) = &pdu.redacts { // TODO: Reason @@ -553,7 +553,7 @@ impl Rooms { } } EventType::RoomMember => { - if let Some(state_key) = &pdu.state_key { + if let Some(state_key) = pdu.state_key.as_ref() { // if the state_key fails let target_user_id = UserId::try_from(state_key.as_str()) .expect("This state_key was previously validated"); @@ -576,6 +576,21 @@ impl Rooms { )?; } } + EventType::RoomMessage => { + if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) { + for word in body + .split_terminator(|c: char| !c.is_alphanumeric()) + .map(str::to_lowercase) + { + let mut key = pdu.room_id.to_string().as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(word.as_bytes()); + key.push(0xff); + key.extend_from_slice(&pdu_id); + self.tokenids.insert(key, &[])?; + } + } + } _ => {} } self.edus.room_read_set(&pdu.room_id, &pdu.sender, index)?; diff --git a/src/pdu.rs b/src/pdu.rs index eec8e49..b565a24 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -2,7 +2,7 @@ use crate::{Error, Result}; use js_int::UInt; use ruma::{ events::{ - pdu::EventHash, room::member::MemberEventContent, AnyRoomEvent, AnyStateEvent, + pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, EventId, Raw, RoomId, ServerName, UserId, @@ -99,6 +99,28 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } + /// This only works for events that are also AnyRoomEvents. + pub fn to_any_event(&self) -> Raw { + let mut json = json!({ + "content": self.content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "unsigned": self.unsigned, + "room_id": self.room_id, + }); + + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &self.redacts { + json["redacts"] = json!(redacts); + } + + serde_json::from_value(json).expect("Raw::from_value always works") + } + pub fn to_room_event(&self) -> Raw { let mut json = json!({ "content": self.content, From 3c26166fb59d78cf887645144ffbb108328247df Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Aug 2020 21:22:59 +0200 Subject: [PATCH 11/48] improvement: device list works better The only situation that isn't working yet is sending `left` events for users when the sender leaves the room --- src/client_server/sync.rs | 20 ++++++-------------- src/database/rooms.rs | 9 ++++----- src/utils.rs | 6 ------ 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index ae4c224..accb199 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -149,15 +149,7 @@ pub async fn sync_events_route( device_list_updates.extend( db.rooms .room_members(&room_id) - .filter_map(|user_id| { - Some( - UserId::try_from(user_id.ok()?.clone()) - .map_err(|_| { - Error::bad_database("Invalid member event state key in db.") - }) - .ok()?, - ) - }) + .filter_map(|user_id| Some(user_id.ok()?)) .filter(|user_id| { // Don't send key updates from the sender to the sender sender_id != user_id @@ -491,9 +483,7 @@ pub async fn sync_events_route( } for user_id in left_encrypted_users { - // If the user doesn't share an encrypted room with the target anymore, we need to tell - // them - if db + let user_target_encrypted = db .rooms .get_shared_rooms(vec![sender_id.clone(), user_id.clone()]) .filter_map(|r| r.ok()) @@ -505,8 +495,10 @@ pub async fn sync_events_route( .is_some(), ) }) - .all(|encrypted| !encrypted) - { + .all(|encrypted| !encrypted); + // If the user doesn't share an encrypted room with the target anymore, we need to tell + // them + if user_target_encrypted { device_list_left.insert(user_id); } } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index d087d65..575a2bf 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -75,23 +75,23 @@ impl StateStore for Rooms { .map_err(|e| e.to_string())? .ok_or_else(|| "PDU via room_id and event_id not found in the db.".to_owned())?; - utils::deserialize( + serde_json::from_slice( &self .pduid_pdu .get(pid) .map_err(|e| e.to_string())? .ok_or_else(|| "PDU via pduid not found in db.".to_owned())?, ) + .map_err(|e| e.to_string()) .and_then(|pdu: StateEvent| { // conduit's PDU's always contain a room_id but some // of ruma's do not so this must be an Option if pdu.room_id() == Some(room_id) { Ok(pdu) } else { - Err(Error::bad_database("Found PDU for incorrect room in db.")) + Err("Found PDU for incorrect room in db.".into()) } }) - .map_err(|e| e.to_string()) } } @@ -1207,8 +1207,7 @@ impl Rooms { let roomid_index = key .iter() .enumerate() - .filter(|(_, &b)| b == 0xff) - .nth(0) + .find(|(_, &b)| b == 0xff) .ok_or_else(|| Error::bad_database("Invalid userroomid_joined in db."))? .0 + 1; // +1 because the room id starts AFTER the separator diff --git a/src/utils.rs b/src/utils.rs index b549153..8cf1b2c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,3 @@ -use crate::Error; use argon2::{Config, Variant}; use cmp::Ordering; use rand::prelude::*; @@ -91,8 +90,3 @@ pub fn common_elements( .all(|b| b) })) } - -pub fn deserialize<'de, T: serde::Deserialize<'de>>(val: &'de sled::IVec) -> Result { - serde_json::from_slice::(val.as_ref()) - .map_err(|_| Error::bad_database("Found invalid bytes as PDU in db.")) -} From 1848f08292875030f100bc4ffa987e31c15d6912 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 21 Aug 2020 21:44:55 -0400 Subject: [PATCH 12/48] Use full sorting algorithm on incoming PDU's in membership --- src/client_server/membership.rs | 50 +++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 996d3c4..3fa3b6a 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -497,7 +497,7 @@ async fn join_room_by_id_helper( .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - let _auth_chain = send_join_response + let auth_chain = send_join_response .room_state .auth_chain .iter() @@ -505,16 +505,54 @@ async fn join_room_by_id_helper( .map(StateEvent::Full) .collect::>(); - // TODO make StateResolution's methods free functions ? or no self param ? - let sorted_events_ids = state_res::StateResolution::reverse_topological_power_sort( + let power_events = event_map + .values() + .filter(|pdu| pdu.is_power_event()) + .map(|pdu| pdu.event_id()) + .collect::>(); + + // TODO these events are not guaranteed to be sorted but they are resolved, do + // we need the auth_chain + let sorted_power_events = state_res::StateResolution::reverse_topological_power_sort( &room_id, - &event_map.keys().cloned().collect::>(), + &power_events, &mut event_map, &db.rooms, - &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + &auth_chain // if we only use it here just build this list in the first place + .iter() + .map(|pdu| pdu.event_id()) + .collect::>(), ); - for ev_id in &sorted_events_ids { + // TODO we may be able to skip this since they are resolved according to spec + let resolved_power = state_res::StateResolution::iterative_auth_check( + room_id, + &RoomVersionId::Version6, + &sorted_power_events, + &BTreeMap::new(), // unconflicted events + &mut event_map, + &db.rooms, + ) + .expect("iterative auth check failed on resolved events"); + // TODO do we need to dedup them + + let events_to_sort = event_map + .keys() + .filter(|id| !sorted_power_events.contains(id)) + .cloned() + .collect::>(); + + let power_level = resolved_power.get(&(EventType::RoomPowerLevels, Some("".into()))); + + let sorted_event_ids = state_res::StateResolution::mainline_sort( + room_id, + &events_to_sort, + power_level, + &mut event_map, + &db.rooms, + ); + + for ev_id in &sorted_event_ids { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` let pdu = event_map .get(ev_id) From 27ffe778233370347d50b914862b6e866dae557b Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 23 Aug 2020 08:32:43 -0400 Subject: [PATCH 13/48] Use helper instead of route for get_public_rooms_filtered --- src/client_server/directory.rs | 224 ++++++++++++++++++--------------- src/client_server/sync.rs | 10 +- src/server_server.rs | 25 ++-- 3 files changed, 131 insertions(+), 128 deletions(-) diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 0aace15..5e03274 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,7 +14,7 @@ use ruma::{ }, federation, }, - directory::PublicRoomsChunk, + directory::{Filter, PublicRoomsChunk, RoomNetwork}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, @@ -33,17 +33,123 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - if let Some(other_server) = body - .server + let Ruma { + body: + get_public_rooms_filtered::IncomingRequest { + limit, + server, + since, + filter, + room_network, + }, + .. + } = body; + get_public_rooms_filtered_helper( + &db, + server.as_deref(), + limit, + since.as_deref(), + filter, // This is not used yet + Some(room_network), // This is not used + ) + .await +} + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/client/r0/publicRooms", data = "") +)] +pub async fn get_public_rooms_route( + db: State<'_, Database>, + body: Ruma, +) -> ConduitResult { + let Ruma { + body: + get_public_rooms::IncomingRequest { + limit, + server, + since, + }, + .. + } = body; + + let get_public_rooms_filtered::Response { + chunk, + prev_batch, + next_batch, + total_room_count_estimate, + } = get_public_rooms_filtered_helper( + &db, + server.as_deref(), + limit, + since.as_deref(), + None, // This is not used + None, // This is not used + ) + .await? + .0; + + Ok(get_public_rooms::Response { + chunk, + prev_batch, + next_batch, + total_room_count_estimate, + } + .into()) +} + +#[cfg_attr( + feature = "conduit_bin", + put("/_matrix/client/r0/directory/list/room/<_>", data = "") +)] +pub async fn set_room_visibility_route( + db: State<'_, Database>, + body: Ruma, +) -> ConduitResult { + match body.visibility { + room::Visibility::Public => db.rooms.set_public(&body.room_id, true)?, + room::Visibility::Private => db.rooms.set_public(&body.room_id, false)?, + } + + Ok(set_room_visibility::Response.into()) +} + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/client/r0/directory/list/room/<_>", data = "") +)] +pub async fn get_room_visibility_route( + db: State<'_, Database>, + body: Ruma, +) -> ConduitResult { + Ok(get_room_visibility::Response { + visibility: if db.rooms.is_public_room(&body.room_id)? { + room::Visibility::Public + } else { + room::Visibility::Private + }, + } + .into()) +} + +pub async fn get_public_rooms_filtered_helper( + db: &Database, + server: Option<&str>, + limit: Option, + since: Option<&str>, + _filter: Option, + _network: Option, +) -> ConduitResult { + if let Some(other_server) = server .clone() - .filter(|server| server != db.globals.server_name().as_str()) + .filter(|server| *server != db.globals.server_name().as_str()) { let response = server_server::send_request( &db, - other_server, + other_server.to_owned(), federation::directory::get_public_rooms::v1::Request { - limit: body.limit, - since: body.since.as_deref(), + limit, + since: since.as_deref(), room_network: ruma::directory::RoomNetwork::Matrix, }, ) @@ -73,10 +179,10 @@ pub async fn get_public_rooms_filtered_route( .into()); } - let limit = body.limit.map_or(10, u64::from); - let mut since = 0_u64; + let limit = limit.map_or(10, u64::from); + let mut num_since = 0_u64; - if let Some(s) = &body.since { + if let Some(s) = &since { let mut characters = s.chars(); let backwards = match characters.next() { Some('n') => false, @@ -89,13 +195,13 @@ pub async fn get_public_rooms_filtered_route( } }; - since = characters + num_since = characters .collect::() .parse() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `since` token."))?; if backwards { - since = since.saturating_sub(limit); + num_since = num_since.saturating_sub(limit); } } @@ -217,20 +323,20 @@ pub async fn get_public_rooms_filtered_route( let chunk = all_rooms .into_iter() - .skip(since as usize) + .skip(num_since as usize) .take(limit as usize) .collect::>(); - let prev_batch = if since == 0 { + let prev_batch = if num_since == 0 { None } else { - Some(format!("p{}", since)) + Some(format!("p{}", num_since)) }; let next_batch = if chunk.len() < limit as usize { None } else { - Some(format!("n{}", since + limit)) + Some(format!("n{}", num_since + limit)) }; Ok(get_public_rooms_filtered::Response { @@ -241,89 +347,3 @@ pub async fn get_public_rooms_filtered_route( } .into()) } - -#[cfg_attr( - feature = "conduit_bin", - get("/_matrix/client/r0/publicRooms", data = "") -)] -pub async fn get_public_rooms_route( - db: State<'_, Database>, - body: Ruma, -) -> ConduitResult { - let Ruma { - body: - get_public_rooms::IncomingRequest { - limit, - server, - since, - }, - sender_id, - device_id, - json_body, - } = body; - - let get_public_rooms_filtered::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } = get_public_rooms_filtered_route( - db, - Ruma { - body: get_public_rooms_filtered::IncomingRequest { - filter: None, - limit, - room_network: ruma::directory::RoomNetwork::Matrix, - server, - since, - }, - sender_id, - device_id, - json_body, - }, - ) - .await? - .0; - - Ok(get_public_rooms::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } - .into()) -} - -#[cfg_attr( - feature = "conduit_bin", - put("/_matrix/client/r0/directory/list/room/<_>", data = "") -)] -pub async fn set_room_visibility_route( - db: State<'_, Database>, - body: Ruma, -) -> ConduitResult { - match body.visibility { - room::Visibility::Public => db.rooms.set_public(&body.room_id, true)?, - room::Visibility::Private => db.rooms.set_public(&body.room_id, false)?, - } - - Ok(set_room_visibility::Response.into()) -} - -#[cfg_attr( - feature = "conduit_bin", - get("/_matrix/client/r0/directory/list/room/<_>", data = "") -)] -pub async fn get_room_visibility_route( - db: State<'_, Database>, - body: Ruma, -) -> ConduitResult { - Ok(get_room_visibility::Response { - visibility: if db.rooms.is_public_room(&body.room_id)? { - room::Visibility::Public - } else { - room::Visibility::Private - }, - } - .into()) -} diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index accb199..ccb25d1 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -415,15 +415,7 @@ pub async fn sync_events_route( device_list_left.extend( db.rooms .room_members(&room_id) - .filter_map(|user_id| { - Some( - UserId::try_from(user_id.ok()?.clone()) - .map_err(|_| { - Error::bad_database("Invalid member event state key in db.") - }) - .ok()?, - ) - }) + .filter_map(|user_id| Some(user_id.ok()?)) .filter(|user_id| { // Don't send key updates from the sender to the sender sender_id != user_id diff --git a/src/server_server.rs b/src/server_server.rs index e47b50a..ac4407b 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -216,9 +216,7 @@ pub async fn get_public_rooms_route( limit, since, }, - sender_id, - device_id, - json_body, + .. } = body; let client::r0::directory::get_public_rooms_filtered::Response { @@ -226,20 +224,13 @@ pub async fn get_public_rooms_route( prev_batch, next_batch, total_room_count_estimate, - } = client_server::get_public_rooms_filtered_route( - db, - Ruma { - body: client::r0::directory::get_public_rooms_filtered::IncomingRequest { - filter: None, - limit, - room_network: ruma::directory::RoomNetwork::Matrix, - server: None, - since, - }, - sender_id, - device_id, - json_body, - }, + } = client_server::get_public_rooms_filtered_helper( + &db, + None, + limit, + since.as_deref(), + None, + Some(ruma::directory::RoomNetwork::Matrix), ) .await? .0; From d9a29e3e5c2b8390bb1a4b99a631b3ac615b0763 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 25 Aug 2020 15:30:25 -0400 Subject: [PATCH 14/48] Fix state for empty key route Replace None with Some("") for state_key --- src/client_server/room.rs | 2 +- src/client_server/state.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index c0603d3..7f4a15f 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -195,7 +195,7 @@ pub fn create_room_route( content: match preset { create_room::RoomPreset::PublicChat => { serde_json::to_value(guest_access::GuestAccessEventContent::new( - // In a public room a joining is the only way to access + // In a public room, joining is the only way to access guest_access::GuestAccess::Forbidden, )) .expect("event is valid, we just created it") diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 867b051..12c5cac 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -76,7 +76,7 @@ pub fn send_state_event_for_empty_key_route( &body.content, json, &body.room_id, - None, + Some("".into()), )? .0 .event_id, From f46c2d1eec808658f671a4845b0ecd39221c8826 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 26 Aug 2020 11:15:52 -0400 Subject: [PATCH 15/48] Fix review issues, move state-res to spec-comp branch --- Cargo.lock | 79 ++++++++++++++------------------- Cargo.toml | 2 +- src/client_server/directory.rs | 31 ++++--------- src/client_server/membership.rs | 16 +++---- src/client_server/room.rs | 1 - src/client_server/state.rs | 35 ++++++++------- src/client_server/sync.rs | 4 +- src/database/rooms.rs | 76 +++++++++++++++---------------- 8 files changed, 106 insertions(+), 138 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e5da6f..62b13d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,7 +157,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.4.0", + "miniz_oxide 0.4.1", "object", "rustc-demangle", ] @@ -168,12 +168,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" -[[package]] -name = "base64" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" - [[package]] name = "base64" version = "0.12.3" @@ -265,7 +259,7 @@ checksum = "942f72db697d8767c22d46a598e01f2d3b475501ea43d0db4f16d90259182d0b" dependencies = [ "num-integer", "num-traits", - "time 0.1.43", + "time 0.1.44", ] [[package]] @@ -287,7 +281,7 @@ checksum = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd" name = "conduit" version = "0.1.0" dependencies = [ - "base64 0.12.3", + "base64", "directories", "http", "image", @@ -298,7 +292,7 @@ dependencies = [ "ring", "rocket", "ruma", - "rust-argon2 0.8.2", + "rust-argon2", "serde", "serde_json", "sled", @@ -320,7 +314,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ "aes-gcm", - "base64 0.12.3", + "base64", "hkdf", "percent-encoding", "rand", @@ -486,9 +480,9 @@ checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" [[package]] name = "encoding_rs" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" +checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" dependencies = [ "cfg-if", ] @@ -672,7 +666,7 @@ checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", ] [[package]] @@ -816,7 +810,7 @@ dependencies = [ "itoa", "pin-project", "socket2", - "time 0.1.43", + "time 0.1.44", "tokio", "tower-service", "tracing", @@ -1057,9 +1051,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" +checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722" dependencies = [ "adler", ] @@ -1462,13 +1456,13 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_users" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b23093265f8d200fa7b4c2c76297f47e681c655f6f1285a8780d6a022f7431" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" dependencies = [ "getrandom", "redox_syscall", - "rust-argon2 0.7.0", + "rust-argon2", ] [[package]] @@ -1527,11 +1521,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12427a5577082c24419c9c417db35cfeb65962efc7675bb6b0d5f1f9d315bfe6" +checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" dependencies = [ - "base64 0.12.3", + "base64", "bytes", "encoding_rs", "futures-core", @@ -1813,31 +1807,19 @@ name = "ruma-signatures" version = "0.6.0-dev.1" source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ - "base64 0.12.3", + "base64", "ring", "serde_json", "untrusted", ] -[[package]] -name = "rust-argon2" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017" -dependencies = [ - "base64 0.11.0", - "blake2b_simd", - "constant_time_eq", - "crossbeam-utils", -] - [[package]] name = "rust-argon2" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19" dependencies = [ - "base64 0.12.3", + "base64", "blake2b_simd", "constant_time_eq", "crossbeam-utils", @@ -1864,7 +1846,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" dependencies = [ - "base64 0.12.3", + "base64", "log", "ring", "sct", @@ -2072,9 +2054,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0437cfb83762844799a60e1e3b489d5ceb6a650fbacb86437badc1b6d87b246" +checksum = "33a71ea1ea5f8747d1af1979bfb7e65c3a025a70609f04ceb78425bc5adad8e6" dependencies = [ "version_check", ] @@ -2088,7 +2070,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res#d93a965ad17781fa9554bb3cea71673c054b9f3f" +source = "git+https://github.com/ruma/state-res?branch=spec-comp#17958665f6592af3ef478024fd1d75c384a30e7f" dependencies = [ "itertools", "js_int", @@ -2233,11 +2215,12 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi 0.3.9", ] @@ -2395,9 +2378,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db63662723c316b43ca36d833707cc93dff82a02ba3d7e354f342682cc8b3545" +checksum = "4f0e00789804e99b20f12bc7003ca416309d28a6f495d6af58d1e2c2842461b5" dependencies = [ "lazy_static", ] @@ -2550,6 +2533,12 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasm-bindgen" version = "0.2.67" diff --git a/Cargo.toml b/Cargo.toml index 78d8f76..15cee72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key # state-res = { path = "../../state-res" } -state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0" } +state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } ring = "0.16.15" [features] diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 5e03274..3b10686 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -63,26 +63,11 @@ pub async fn get_public_rooms_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - let Ruma { - body: - get_public_rooms::IncomingRequest { - limit, - server, - since, - }, - .. - } = body; - - let get_public_rooms_filtered::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } = get_public_rooms_filtered_helper( + let response = get_public_rooms_filtered_helper( &db, - server.as_deref(), - limit, - since.as_deref(), + body.body.server.as_deref(), + body.body.limit, + body.body.since.as_deref(), None, // This is not used None, // This is not used ) @@ -90,10 +75,10 @@ pub async fn get_public_rooms_route( .0; Ok(get_public_rooms::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, + chunk: response.chunk, + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, } .into()) } diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 3fa3b6a..90683e6 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -483,12 +483,12 @@ async fn join_room_by_id_helper( .await?; dbg!(&send_join_response); - // todo!("Take send_join_response and 'create' the room using that data"); let mut event_map = send_join_response .room_state .state .iter() + .chain(send_join_response.room_state.auth_chain.iter()) .map(|pdu| { pdu.deserialize() .map(StateEvent::Full) @@ -497,14 +497,6 @@ async fn join_room_by_id_helper( .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - let auth_chain = send_join_response - .room_state - .auth_chain - .iter() - .flat_map(|pdu| pdu.deserialize().ok()) - .map(StateEvent::Full) - .collect::>(); - let power_events = event_map .values() .filter(|pdu| pdu.is_power_event()) @@ -518,9 +510,11 @@ async fn join_room_by_id_helper( &power_events, &mut event_map, &db.rooms, - &auth_chain // if we only use it here just build this list in the first place + &send_join_response + .room_state + .auth_chain .iter() - .map(|pdu| pdu.event_id()) + .filter_map(|pdu| Some(StateEvent::Full(pdu.deserialize().ok()?).event_id())) .collect::>(), ); diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 7f4a15f..3ee21b6 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -195,7 +195,6 @@ pub fn create_room_route( content: match preset { create_room::RoomPreset::PublicChat => { serde_json::to_value(guest_access::GuestAccessEventContent::new( - // In a public room, joining is the only way to access guest_access::GuestAccess::Forbidden, )) .expect("event is valid, we just created it") diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 12c5cac..e7d2bcf 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -1,5 +1,5 @@ use super::State; -use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Ruma}; +use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Result, Ruma}; use ruma::{ api::client::{ error::ErrorKind, @@ -9,7 +9,7 @@ use ruma::{ }, }, events::{AnyStateEventContent, EventContent}, - RoomId, UserId, + EventId, RoomId, UserId, }; #[cfg(feature = "conduit_bin")] @@ -33,13 +33,16 @@ pub fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - send_state_event_for_key_helper( - &db, - sender_id, - &body.content, - content, - &body.room_id, - Some(body.state_key.clone()), + Ok( + send_state_event_for_key::Response::new(send_state_event_for_key_helper( + &db, + sender_id, + &body.content, + content, + &body.room_id, + Some(body.state_key.clone()), + )?) + .into(), ) } @@ -67,8 +70,8 @@ pub fn send_state_event_for_empty_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - Ok(send_state_event_for_empty_key::Response::new( - send_state_event_for_key_helper( + Ok( + send_state_event_for_empty_key::Response::new(send_state_event_for_key_helper( &db, sender_id .as_ref() @@ -77,11 +80,9 @@ pub fn send_state_event_for_empty_key_route( json, &body.room_id, Some("".into()), - )? - .0 - .event_id, + )?) + .into(), ) - .into()) } #[cfg_attr( @@ -183,7 +184,7 @@ pub fn send_state_event_for_key_helper( json: serde_json::Value, room_id: &RoomId, state_key: Option, -) -> ConduitResult { +) -> Result { let sender_id = sender; if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = content { @@ -224,5 +225,5 @@ pub fn send_state_event_for_key_helper( &db.account_data, )?; - Ok(send_state_event_for_key::Response::new(event_id).into()) + Ok(event_id) } diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index ccb25d1..7432960 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -475,7 +475,7 @@ pub async fn sync_events_route( } for user_id in left_encrypted_users { - let user_target_encrypted = db + let still_share_encrypted_room = db .rooms .get_shared_rooms(vec![sender_id.clone(), user_id.clone()]) .filter_map(|r| r.ok()) @@ -490,7 +490,7 @@ pub async fn sync_events_route( .all(|encrypted| !encrypted); // If the user doesn't share an encrypted room with the target anymore, we need to tell // them - if user_target_encrypted { + if still_share_encrypted_room { device_list_left.insert(user_id); } } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 575a2bf..66af736 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -4,7 +4,6 @@ pub use edus::RoomEdus; use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; use log::error; -// TODO if ruma-signatures re-exports `use ruma::signatures::digest;` use ring::digest; use ruma::{ api::client::error::ErrorKind, @@ -96,9 +95,9 @@ impl StateStore for Rooms { } impl Rooms { - /// Builds a `StateMap` by iterating over all keys that start - /// with `state_hash`, this gives the full state at event "x". - pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { + /// Builds a StateMap by iterating over all keys that start + /// with state_hash, this gives the full state for the given state_hash. + pub fn state_full(&self, state_hash: StateHashId) -> Result> { self.stateid_pduid .scan_prefix(&state_hash) .values() @@ -242,8 +241,6 @@ impl Rooms { /// Generate a new StateHash. /// /// A unique hash made from hashing the current states pduid's. - /// Because `append_state_pdu` handles the empty state db case it does not - /// have to be here. fn new_state_hash_id(&self, room_id: &RoomId) -> Result { // Use hashed roomId as the first StateHash key for first state event in room if self @@ -281,7 +278,7 @@ impl Rooms { /// Checks if a room exists. pub fn exists(&self, room_id: &RoomId) -> Result { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); // Look for PDUs in that room. @@ -300,7 +297,7 @@ impl Rooms { let mut hashmap = HashMap::new(); for pdu in self.roomstateid_pduid - .scan_prefix(&room_id.to_string().as_bytes()) + .scan_prefix(&room_id.as_bytes()) .values() .map(|value| { Ok::<_, Error>( @@ -322,13 +319,13 @@ impl Rooms { Ok(hashmap) } - /// Returns the all state entries for this type. + /// Returns all state entries for this type. pub fn room_state_type( &self, room_id: &RoomId, event_type: &EventType, ) -> Result> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(&event_type.to_string().as_bytes()); @@ -357,7 +354,7 @@ impl Rooms { Ok(hashmap) } - /// Returns a single PDU in `room_id` with key (`event_type`, `state_key`). + /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). pub fn room_state_get( &self, room_id: &RoomId, @@ -459,7 +456,7 @@ impl Rooms { /// Returns the leaf pdus of a room. pub fn get_pdu_leaves(&self, room_id: &RoomId) -> Result> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); let mut events = Vec::new(); @@ -582,7 +579,7 @@ impl Rooms { .split_terminator(|c: char| !c.is_alphanumeric()) .map(str::to_lowercase) { - let mut key = pdu.room_id.to_string().as_bytes().to_vec(); + let mut key = pdu.room_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(word.as_bytes()); key.push(0xff); @@ -752,7 +749,10 @@ impl Rooms { }) .collect::>>()?, ) - .ok_or(Error::Conflict("Found incoming PDU with invalid data."))?, + .map_err(|e| { + log::error!("{}", e); + Error::Conflict("Found incoming PDU with invalid data.") + })?, EventType::RoomCreate => prev_events.is_empty(), // Not allow any of the following events if the sender is not joined. _ if sender_membership != member::MembershipState::Join => false, @@ -982,13 +982,13 @@ impl Rooms { globals: &super::globals::Globals, ) -> Result<()> { let membership = member_content.membership; - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); - let mut roomuser_id = room_id.to_string().as_bytes().to_vec(); + let mut roomuser_id = room_id.as_bytes().to_vec(); roomuser_id.push(0xff); - roomuser_id.extend_from_slice(user_id.to_string().as_bytes()); + roomuser_id.extend_from_slice(user_id.as_bytes()); match &membership { member::MembershipState::Join => { @@ -1051,9 +1051,9 @@ impl Rooms { /// Makes a user forget a room. pub fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); self.userroomid_left.remove(userroom_id)?; @@ -1069,8 +1069,8 @@ impl Rooms { if let Some(room_id) = room_id { // New alias self.alias_roomid - .insert(alias.alias(), &*room_id.to_string())?; - let mut aliasid = room_id.to_string().as_bytes().to_vec(); + .insert(alias.alias(), room_id.as_bytes())?; + let mut aliasid = room_id.as_bytes().to_vec(); aliasid.extend_from_slice(&globals.next_count()?.to_be_bytes()); self.aliasid_alias.insert(aliasid, &*alias.alias())?; } else { @@ -1105,7 +1105,7 @@ impl Rooms { } pub fn room_aliases(&self, room_id: &RoomId) -> impl Iterator> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); self.aliasid_alias @@ -1119,16 +1119,16 @@ impl Rooms { pub fn set_public(&self, room_id: &RoomId, public: bool) -> Result<()> { if public { - self.publicroomids.insert(room_id.to_string(), &[])?; + self.publicroomids.insert(room_id.as_bytes(), &[])?; } else { - self.publicroomids.remove(room_id.to_string())?; + self.publicroomids.remove(room_id.as_bytes())?; } Ok(()) } pub fn is_public_room(&self, room_id: &RoomId) -> Result { - Ok(self.publicroomids.contains_key(room_id.to_string())?) + Ok(self.publicroomids.contains_key(room_id.as_bytes())?) } pub fn public_rooms(&self) -> impl Iterator> { @@ -1147,7 +1147,7 @@ impl Rooms { room_id: &RoomId, search_string: &str, ) -> Result<(impl Iterator + 'a, Vec)> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); let words = search_string @@ -1233,7 +1233,7 @@ impl Rooms { /// Returns an iterator over all joined members of a room. pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> { self.roomuserid_joined - .scan_prefix(room_id.to_string()) + .scan_prefix(room_id.as_bytes()) .keys() .map(|key| { Ok(UserId::try_from( @@ -1254,7 +1254,7 @@ impl Rooms { /// Returns an iterator over all invited members of a room. pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator> { self.roomuserid_invited - .scan_prefix(room_id.to_string()) + .scan_prefix(room_id.as_bytes()) .keys() .map(|key| { Ok(UserId::try_from( @@ -1275,7 +1275,7 @@ impl Rooms { /// Returns an iterator over all rooms this user joined. pub fn rooms_joined(&self, user_id: &UserId) -> impl Iterator> { self.userroomid_joined - .scan_prefix(user_id.to_string()) + .scan_prefix(user_id.as_bytes()) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1296,7 +1296,7 @@ impl Rooms { /// Returns an iterator over all rooms a user was invited to. pub fn rooms_invited(&self, user_id: &UserId) -> impl Iterator> { self.userroomid_invited - .scan_prefix(&user_id.to_string()) + .scan_prefix(&user_id.as_bytes()) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1317,7 +1317,7 @@ impl Rooms { /// Returns an iterator over all rooms a user left. pub fn rooms_left(&self, user_id: &UserId) -> impl Iterator> { self.userroomid_left - .scan_prefix(&user_id.to_string()) + .scan_prefix(&user_id.as_bytes()) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1336,25 +1336,25 @@ impl Rooms { } pub fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); Ok(self.userroomid_joined.get(userroom_id)?.is_some()) } pub fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); Ok(self.userroomid_invited.get(userroom_id)?.is_some()) } pub fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); Ok(self.userroomid_left.get(userroom_id)?.is_some()) } From 3b40f3d60ef9839c72763169f6ae1daed7b04895 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Thu, 27 Aug 2020 16:10:20 -0400 Subject: [PATCH 16/48] Update state-res crate --- Cargo.lock | 2 +- src/database/rooms.rs | 73 +++++++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62b13d0..eebaddc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2070,7 +2070,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res?branch=spec-comp#17958665f6592af3ef478024fd1d75c384a30e7f" +source = "git+https://github.com/ruma/state-res?branch=spec-comp#394d26744a6586ccdc01838964bb27dab289eee5" dependencies = [ "itertools", "js_int", diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 66af736..ee070b3 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -19,13 +19,12 @@ use ruma::{ EventId, Raw, RoomAliasId, RoomId, UserId, }; use sled::IVec; -use state_res::{event_auth, Requester, StateEvent, StateMap, StateStore}; +use state_res::{event_auth, Error as StateError, Requester, StateEvent, StateMap, StateStore}; use std::{ collections::{BTreeMap, HashMap}, convert::{TryFrom, TryInto}, mem, - result::Result as StdResult, }; /// The unique identifier of each state group. @@ -67,28 +66,32 @@ pub struct Rooms { } impl StateStore for Rooms { - fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> StdResult { + fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> state_res::Result { let pid = self .eventid_pduid .get(event_id.as_bytes()) - .map_err(|e| e.to_string())? - .ok_or_else(|| "PDU via room_id and event_id not found in the db.".to_owned())?; + .map_err(StateError::custom)? + .ok_or_else(|| { + StateError::NotFound("PDU via room_id and event_id not found in the db.".into()) + })?; serde_json::from_slice( &self .pduid_pdu .get(pid) - .map_err(|e| e.to_string())? - .ok_or_else(|| "PDU via pduid not found in db.".to_owned())?, + .map_err(StateError::custom)? + .ok_or_else(|| StateError::NotFound("PDU via pduid not found in db.".into()))?, ) - .map_err(|e| e.to_string()) + .map_err(Into::into) .and_then(|pdu: StateEvent| { // conduit's PDU's always contain a room_id but some // of ruma's do not so this must be an Option if pdu.room_id() == Some(room_id) { Ok(pdu) } else { - Err("Found PDU for incorrect room in db.".into()) + Err(StateError::NotFound( + "Found PDU for incorrect room in db.".into(), + )) } }) } @@ -732,27 +735,37 @@ impl Rooms { // Don't allow encryption events when it's disabled !globals.encryption_disabled() } - EventType::RoomMember => event_auth::is_membership_change_allowed( - // TODO this is a bit of a hack but not sure how to have a type - // declared in `state_res` crate easily convert to/from conduit::PduEvent - Requester { - prev_event_ids: prev_events.to_owned(), - room_id: &room_id, - content: &content, - state_key: Some(state_key.to_owned()), - sender: &sender, - }, - &auth_events - .iter() - .map(|((ty, key), pdu)| { - Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) - }) - .collect::>>()?, - ) - .map_err(|e| { - log::error!("{}", e); - Error::Conflict("Found incoming PDU with invalid data.") - })?, + EventType::RoomMember => { + let prev_event = self + .get_pdu(prev_events.iter().next().ok_or(Error::BadRequest( + ErrorKind::Unknown, + "Membership can't be the first event", + ))?)? + .map(|pdu| pdu.convert_for_state_res()) + .transpose()?; + event_auth::valid_membership_change( + // TODO this is a bit of a hack but not sure how to have a type + // declared in `state_res` crate easily convert to/from conduit::PduEvent + Requester { + prev_event_ids: prev_events.to_owned(), + room_id: &room_id, + content: &content, + state_key: Some(state_key.to_owned()), + sender: &sender, + }, + prev_event.as_ref(), + &auth_events + .iter() + .map(|((ty, key), pdu)| { + Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) + }) + .collect::>>()?, + ) + .map_err(|e| { + log::error!("{}", e); + Error::Conflict("Found incoming PDU with invalid data.") + })? + } EventType::RoomCreate => prev_events.is_empty(), // Not allow any of the following events if the sender is not joined. _ if sender_membership != member::MembershipState::Join => false, From 2a63d0955ab06d1d1cbcae8fdf078d02a49fa65b Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 30 Aug 2020 16:08:47 -0400 Subject: [PATCH 17/48] Sort and authenticate the events from /send_join response --- src/client_server/membership.rs | 49 +++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 90683e6..6d1931b 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -497,17 +497,18 @@ async fn join_room_by_id_helper( .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - let power_events = event_map + let control_events = event_map .values() .filter(|pdu| pdu.is_power_event()) .map(|pdu| pdu.event_id()) .collect::>(); - // TODO these events are not guaranteed to be sorted but they are resolved, do - // we need the auth_chain - let sorted_power_events = state_res::StateResolution::reverse_topological_power_sort( + // These events are not guaranteed to be sorted but they are resolved according to spec + // we auth them anyways to weed out faulty/malicious server. The following is basically the + // full state resolution algorithm. + let sorted_control_events = state_res::StateResolution::reverse_topological_power_sort( &room_id, - &power_events, + &control_events, &mut event_map, &db.rooms, &send_join_response @@ -518,26 +519,31 @@ async fn join_room_by_id_helper( .collect::>(), ); - // TODO we may be able to skip this since they are resolved according to spec - let resolved_power = state_res::StateResolution::iterative_auth_check( + // Auth check each event against the "partial" state created by the preceding events + let resolved_control_events = state_res::StateResolution::iterative_auth_check( room_id, &RoomVersionId::Version6, - &sorted_power_events, - &BTreeMap::new(), // unconflicted events + &sorted_control_events, + &BTreeMap::new(), // We have no "clean/resolved" events to add (these extend the `resolved_control_events`) &mut event_map, &db.rooms, ) .expect("iterative auth check failed on resolved events"); - // TODO do we need to dedup them + // This removes the control events that failed auth, leaving the resolved + // to be mainline sorted let events_to_sort = event_map .keys() - .filter(|id| !sorted_power_events.contains(id)) + .filter(|id| { + !sorted_control_events.contains(id) + || resolved_control_events.values().any(|rid| *id == rid) + }) .cloned() .collect::>(); - let power_level = resolved_power.get(&(EventType::RoomPowerLevels, Some("".into()))); - + let power_level = + resolved_control_events.get(&(EventType::RoomPowerLevels, Some("".into()))); + // Sort the remaining non control events let sorted_event_ids = state_res::StateResolution::mainline_sort( room_id, &events_to_sort, @@ -546,7 +552,22 @@ async fn join_room_by_id_helper( &db.rooms, ); - for ev_id in &sorted_event_ids { + let resolved_events = state_res::StateResolution::iterative_auth_check( + room_id, + &RoomVersionId::Version6, + &sorted_event_ids, + &resolved_control_events, + &mut event_map, + &db.rooms, + ) + .expect("iterative auth check failed on resolved events"); + + // filter the events that failed the auth check keeping the remaining events + // sorted correctly + for ev_id in sorted_event_ids + .iter() + .filter(|id| resolved_events.values().any(|rid| rid == *id)) + { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` let pdu = event_map .get(ev_id) From 1e8fbd8d50c1bf0fe80cb77b8df8b5a45c3698f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 8 Sep 2020 17:32:03 +0200 Subject: [PATCH 18/48] Update ruma version --- Cargo.lock | 18 +--------- Cargo.toml | 9 ++--- src/client_server/account.rs | 10 +++--- src/client_server/alias.rs | 8 ++--- src/client_server/backup.rs | 10 +++--- src/client_server/config.rs | 4 +-- src/client_server/context.rs | 2 +- src/client_server/device.rs | 8 ++--- src/client_server/directory.rs | 14 ++++---- src/client_server/filter.rs | 2 +- src/client_server/keys.rs | 18 +++++----- src/client_server/media.rs | 8 ++--- src/client_server/membership.rs | 32 +++++++++--------- src/client_server/message.rs | 8 ++--- src/client_server/presence.rs | 2 +- src/client_server/profile.rs | 10 +++--- src/client_server/read_marker.rs | 4 +-- src/client_server/redact.rs | 2 +- src/client_server/room.rs | 51 ++++++++++------------------- src/client_server/search.rs | 22 ++++++++----- src/client_server/session.rs | 2 +- src/client_server/state.rs | 6 ++-- src/client_server/sync.rs | 2 +- src/client_server/tag.rs | 6 ++-- src/client_server/to_device.rs | 2 +- src/client_server/typing.rs | 8 ++--- src/client_server/user_directory.rs | 2 +- src/database/media.rs | 4 +-- src/database/users.rs | 6 ++-- src/error.rs | 6 ++-- src/pdu.rs | 2 +- src/ruma_wrapper.rs | 22 ++++++++----- src/server_server.rs | 6 ++-- 33 files changed, 147 insertions(+), 169 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eebaddc..c88c578 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1631,7 +1631,6 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1647,7 +1646,6 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "http", "percent-encoding", @@ -1662,7 +1660,6 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1673,7 +1670,6 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-api", "ruma-common", @@ -1686,7 +1682,6 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "assign", "http", @@ -1705,9 +1700,9 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", + "ruma-api", "ruma-identifiers", "ruma-serde", "serde", @@ -1718,7 +1713,6 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-common", @@ -1733,7 +1727,6 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1744,7 +1737,6 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-api", @@ -1759,7 +1751,6 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1771,7 +1762,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro2", "quote", @@ -1782,18 +1772,14 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ - "ruma-serde", "serde", - "serde_json", "strum", ] [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "form_urlencoded", "itoa", @@ -1805,7 +1791,6 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "base64", "ring", @@ -2070,7 +2055,6 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res?branch=spec-comp#394d26744a6586ccdc01838964bb27dab289eee5" dependencies = [ "itertools", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 15cee72..5d35433 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,9 @@ edition = "2018" # TODO: This can become optional as soon as proper configs are supported #rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } - -ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers +#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers +#ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers +ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently log = "0.4.8" # Used for emitting log entries @@ -31,8 +32,8 @@ reqwest = "0.10.6" # Used to send requests thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key -# state-res = { path = "../../state-res" } -state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } +#state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } +state-res = { path = "../state-res" } ring = "0.16.15" [features] diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 9e52f6d..cb77a15 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -33,7 +33,7 @@ const GUEST_NAME_LENGTH: usize = 10; )] pub fn get_register_available_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { // Validate user id let user_id = UserId::parse_with_server_name(body.username.clone(), db.globals.server_name()) @@ -75,7 +75,7 @@ pub fn get_register_available_route( )] pub fn register_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { if db.globals.registration_disabled() { return Err(Error::BadRequest( @@ -84,7 +84,7 @@ pub fn register_route( )); } - let is_guest = matches!(body.kind, Some(RegistrationKind::Guest)); + let is_guest = body.kind == RegistrationKind::Guest; let mut missing_username = false; @@ -223,7 +223,7 @@ pub fn register_route( )] pub fn change_password_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -305,7 +305,7 @@ pub fn whoami_route(body: Ruma) -> ConduitResult, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 669f558..1d30261 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -20,7 +20,7 @@ use rocket::{delete, get, put}; )] pub fn create_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { if db.rooms.id_from_alias(&body.room_alias)?.is_some() { return Err(Error::Conflict("Alias already exists.")); @@ -38,7 +38,7 @@ pub fn create_alias_route( )] pub fn delete_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { db.rooms.set_alias(&body.room_alias, None, &db.globals)?; @@ -51,7 +51,7 @@ pub fn delete_alias_route( )] pub async fn get_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { get_alias_helper(&db, &body.room_alias).await } @@ -65,7 +65,7 @@ pub async fn get_alias_helper( &db, room_alias.server_name().to_string(), federation::query::get_room_information::v1::Request { - room_alias: room_alias.to_string(), + room_alias, }, ) .await?; diff --git a/src/client_server/backup.rs b/src/client_server/backup.rs index a104964..8966c01 100644 --- a/src/client_server/backup.rs +++ b/src/client_server/backup.rs @@ -33,7 +33,7 @@ pub fn create_backup_route( )] pub fn update_backup_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); db.key_backups @@ -75,7 +75,7 @@ pub fn get_latest_backup_route( )] pub fn get_backup_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let algorithm = db @@ -90,7 +90,7 @@ pub fn get_backup_route( algorithm, count: (db.key_backups.count_keys(sender_id, &body.version)? as u32).into(), etag: db.key_backups.get_etag(sender_id, &body.version)?, - version: body.version.clone(), + version: body.version.to_owned(), } .into()) } @@ -102,7 +102,7 @@ pub fn get_backup_route( )] pub fn add_backup_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -132,7 +132,7 @@ pub fn add_backup_keys_route( )] pub fn get_backup_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/config.rs b/src/client_server/config.rs index baa9381..515ad16 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -18,7 +18,7 @@ use rocket::{get, put}; )] pub fn set_global_account_data_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -49,7 +49,7 @@ pub fn set_global_account_data_route( )] pub fn get_global_account_data_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 7b1fad9..9593726 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -12,7 +12,7 @@ use rocket::get; )] pub fn get_context_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/device.rs b/src/client_server/device.rs index 89033f0..6352d0d 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -37,7 +37,7 @@ pub fn get_devices_route( )] pub fn get_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,7 @@ pub fn get_device_route( )] pub fn update_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -80,7 +80,7 @@ pub fn update_device_route( )] pub fn delete_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -127,7 +127,7 @@ pub fn delete_device_route( )] pub fn delete_devices_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 3b10686..34feb71 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,7 +14,7 @@ use ruma::{ }, federation, }, - directory::{Filter, PublicRoomsChunk, RoomNetwork}, + directory::{IncomingFilter, PublicRoomsChunk, IncomingRoomNetwork}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, @@ -31,7 +31,7 @@ use rocket::{get, post, put}; )] pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let Ruma { body: @@ -61,7 +61,7 @@ pub async fn get_public_rooms_filtered_route( )] pub async fn get_public_rooms_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let response = get_public_rooms_filtered_helper( &db, @@ -89,7 +89,7 @@ pub async fn get_public_rooms_route( )] pub async fn set_room_visibility_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { match body.visibility { room::Visibility::Public => db.rooms.set_public(&body.room_id, true)?, @@ -105,7 +105,7 @@ pub async fn set_room_visibility_route( )] pub async fn get_room_visibility_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { Ok(get_room_visibility::Response { visibility: if db.rooms.is_public_room(&body.room_id)? { @@ -122,8 +122,8 @@ pub async fn get_public_rooms_filtered_helper( server: Option<&str>, limit: Option, since: Option<&str>, - _filter: Option, - _network: Option, + _filter: Option, + _network: Option, ) -> ConduitResult { if let Some(other_server) = server .clone() diff --git a/src/client_server/filter.rs b/src/client_server/filter.rs index 4322de3..4b1c3a0 100644 --- a/src/client_server/filter.rs +++ b/src/client_server/filter.rs @@ -7,7 +7,7 @@ use rocket::{get, post}; #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))] pub fn get_filter_route() -> ConduitResult { // TODO - Ok(get_filter::Response::new(filter::FilterDefinition { + Ok(get_filter::Response::new(filter::IncomingFilterDefinition { event_fields: None, event_format: None, account_data: None, diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index 3311529..0e7b1ef 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -11,7 +11,7 @@ use ruma::{ uiaa::{AuthFlow, UiaaInfo}, }, }, - encryption::UnsignedDeviceInfo, + encryption::IncomingUnsignedDeviceInfo, }; use std::collections::{BTreeMap, HashSet}; @@ -24,7 +24,7 @@ use rocket::{get, post}; )] pub fn upload_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,7 @@ pub fn upload_keys_route( )] pub fn get_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -78,9 +78,9 @@ pub fn get_keys_route( Error::bad_database("all_device_keys contained nonexistent device.") })?; - keys.unsigned = Some(UnsignedDeviceInfo { + keys.unsigned = IncomingUnsignedDeviceInfo { device_display_name: metadata.display_name, - }); + }; container.insert(device_id, keys); } @@ -97,9 +97,9 @@ pub fn get_keys_route( ), )?; - keys.unsigned = Some(UnsignedDeviceInfo { + keys.unsigned = IncomingUnsignedDeviceInfo { device_display_name: metadata.display_name, - }); + }; container.insert(device_id.clone(), keys); } @@ -167,7 +167,7 @@ pub fn claim_keys_route( )] pub fn upload_signing_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -280,7 +280,7 @@ pub fn upload_signatures_route( )] pub fn get_key_changes_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 79c1f08..038012e 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -27,7 +27,7 @@ pub fn get_media_config_route( )] pub fn create_content_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let mxc = format!( "mxc://{}/{}", @@ -36,7 +36,7 @@ pub fn create_content_route( ); db.media.create( mxc.clone(), - body.filename.as_ref(), + &body.filename, &body.content_type, &body.file, )?; @@ -53,7 +53,7 @@ pub fn create_content_route( )] pub fn get_content_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _server_name: String, _media_id: String, ) -> ConduitResult { @@ -85,7 +85,7 @@ pub fn get_content_route( )] pub fn get_content_thumbnail_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _server_name: String, _media_id: String, ) -> ConduitResult { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 6d1931b..606e470 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -32,7 +32,7 @@ use rocket::{get, post}; )] pub async fn join_room_by_id_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { join_room_by_id_helper( &db, @@ -49,7 +49,7 @@ pub async fn join_room_by_id_route( )] pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, @@ -81,7 +81,7 @@ pub async fn join_room_by_id_or_alias_route( )] pub fn leave_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -127,11 +127,11 @@ pub fn leave_room_route( )] pub fn invite_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - if let invite_user::InvitationRecipient::UserId { user_id } = &body.recipient { + if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), @@ -165,7 +165,7 @@ pub fn invite_user_route( )] pub fn kick_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -212,7 +212,7 @@ pub fn kick_user_route( )] pub fn ban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -267,7 +267,7 @@ pub fn ban_user_route( )] pub fn unban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -313,7 +313,7 @@ pub fn unban_user_route( )] pub fn forget_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -348,7 +348,7 @@ pub fn joined_rooms_route( )] pub fn get_member_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -376,7 +376,7 @@ pub fn get_member_events_route( )] pub fn joined_members_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -422,9 +422,9 @@ async fn join_room_by_id_helper( &db, room_id.server_name().to_string(), federation::membership::create_join_event_template::v1::Request { - room_id: room_id.clone(), - user_id: sender_id.clone(), - ver: vec![RoomVersionId::Version5, RoomVersionId::Version6], + room_id, + user_id: sender_id, + ver: &[RoomVersionId::Version5, RoomVersionId::Version6], }, ) .await?; @@ -474,8 +474,8 @@ async fn join_room_by_id_helper( &db, room_id.server_name().to_string(), federation::membership::create_join_event::v1::Request { - room_id: room_id.clone(), - event_id, + room_id, + event_id: &event_id, pdu_stub: serde_json::from_value(join_event_stub_value) .expect("we just created this event"), }, diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 03832d8..09c3517 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -18,7 +18,7 @@ use rocket::{get, put}; )] pub fn send_message_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -53,7 +53,7 @@ pub fn send_message_event_route( )] pub fn get_message_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -96,7 +96,7 @@ pub fn get_message_events_route( .collect::>(); let mut resp = get_message_events::Response::new(); - resp.start = Some(body.from.clone()); + resp.start = Some(body.from.to_owned()); resp.end = end_token; resp.chunk = events_after; resp.state = Vec::new(); @@ -120,7 +120,7 @@ pub fn get_message_events_route( .collect::>(); let mut resp = get_message_events::Response::new(); - resp.start = Some(body.from.clone()); + resp.start = Some(body.from.to_owned()); resp.end = start_token; resp.chunk = events_before; resp.state = Vec::new(); diff --git a/src/client_server/presence.rs b/src/client_server/presence.rs index 0b6a51f..d105eb6 100644 --- a/src/client_server/presence.rs +++ b/src/client_server/presence.rs @@ -12,7 +12,7 @@ use rocket::put; )] pub fn set_presence_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 0707b34..386d898 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -21,7 +21,7 @@ use std::convert::TryInto; )] pub fn set_displayname_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -98,7 +98,7 @@ pub fn set_displayname_route( )] pub fn get_displayname_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { Ok(get_display_name::Response { displayname: db.users.displayname(&body.user_id)?, @@ -112,7 +112,7 @@ pub fn get_displayname_route( )] pub fn set_avatar_url_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -201,7 +201,7 @@ pub fn set_avatar_url_route( )] pub fn get_avatar_url_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { Ok(get_avatar_url::Response { avatar_url: db.users.avatar_url(&body.user_id)?, @@ -215,7 +215,7 @@ pub fn get_avatar_url_route( )] pub fn get_profile_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let avatar_url = db.users.avatar_url(&body.user_id)?; let displayname = db.users.displayname(&body.user_id)?; diff --git a/src/client_server/read_marker.rs b/src/client_server/read_marker.rs index ff72765..023eece 100644 --- a/src/client_server/read_marker.rs +++ b/src/client_server/read_marker.rs @@ -15,7 +15,7 @@ use std::{collections::BTreeMap, time::SystemTime}; )] pub fn set_read_marker_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -52,7 +52,7 @@ pub fn set_read_marker_route( ); let mut receipt_content = BTreeMap::new(); receipt_content.insert( - event.clone(), + event.to_owned(), ruma::events::receipt::Receipts { read: Some(user_receipts), }, diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 8708692..cd1b443 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -14,7 +14,7 @@ use rocket::put; )] pub fn redact_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 3ee21b6..9918123 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -22,7 +22,7 @@ use rocket::{get, post}; )] pub fn create_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -48,11 +48,8 @@ pub fn create_room_route( })?; let mut content = ruma::events::room::create::CreateEventContent::new(sender_id.clone()); - content.federate = body.creation_content.as_ref().map_or(true, |c| c.federate); - content.predecessor = body - .creation_content - .as_ref() - .and_then(|c| c.predecessor.clone()); + content.federate = body.creation_content.federate; + content.predecessor = body.creation_content.predecessor.clone(); content.room_version = RoomVersionId::Version6; // 1. The room create event @@ -80,7 +77,7 @@ pub fn create_room_route( membership: member::MembershipState::Join, displayname: db.users.displayname(&sender_id)?, avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: body.is_direct, + is_direct: Some(body.is_direct), third_party_invite: None, }) .expect("event is valid, we just created it"), @@ -137,8 +134,7 @@ pub fn create_room_route( // 4. Events set by preset // Figure out preset. We need it for preset specific events - let visibility = body.visibility.unwrap_or(room::Visibility::Private); - let preset = body.preset.unwrap_or_else(|| match visibility { + let preset = body.preset.unwrap_or_else(|| match body.visibility { room::Visibility::Private => create_room::RoomPreset::PrivateChat, room::Visibility::Public => create_room::RoomPreset::PublicChat, }); @@ -213,32 +209,19 @@ pub fn create_room_route( )?; // 5. Events listed in initial_state - for create_room::InitialStateEvent { - event_type, - state_key, - content, - } in &body.initial_state - { + for event in &body.initial_state { + let pdu_builder = serde_json::from_str::( + &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"), + ).map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; + // Silently skip encryption events if they are not allowed - if event_type == &EventType::RoomEncryption && db.globals.encryption_disabled() { + if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() + { continue; } - db.rooms.build_and_append_pdu( - PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), - event_type: event_type.clone(), - content: serde_json::from_str(content.get()).map_err(|_| { - Error::BadRequest(ErrorKind::BadJson, "Invalid initial_state content.") - })?, - unsigned: None, - state_key: state_key.clone(), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu(pdu_builder, &db.globals, &db.account_data)?; } // 6. Events implied by name and topic @@ -293,7 +276,7 @@ pub fn create_room_route( membership: member::MembershipState::Invite, displayname: db.users.displayname(&user)?, avatar_url: db.users.avatar_url(&user)?, - is_direct: body.is_direct, + is_direct: Some(body.is_direct), third_party_invite: None, }) .expect("event is valid, we just created it"), @@ -311,7 +294,7 @@ pub fn create_room_route( db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?; } - if let Some(room::Visibility::Public) = body.visibility { + if body.visibility == room::Visibility::Public { db.rooms.set_public(&room_id, true)?; } @@ -324,7 +307,7 @@ pub fn create_room_route( )] pub fn get_room_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/search.rs b/src/client_server/search.rs index 082711d..2967e00 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -1,11 +1,10 @@ use super::State; use crate::{ConduitResult, Database, Error, Ruma}; -use js_int::uint; use ruma::api::client::{error::ErrorKind, r0::search::search_events}; #[cfg(feature = "conduit_bin")] use rocket::post; -use search_events::{ResultCategories, ResultRoomEvents, SearchResult}; +use search_events::{EventContextResult, ResultCategories, ResultRoomEvents, SearchResult}; use std::collections::BTreeMap; #[cfg_attr( @@ -14,7 +13,7 @@ use std::collections::BTreeMap; )] pub fn search_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -51,13 +50,18 @@ pub fn search_events_route( .0 .map(|result| { Ok::<_, Error>(SearchResult { - context: None, + context: EventContextResult { + end: None, + events_after: Vec::new(), + events_before: Vec::new(), + profile_info: BTreeMap::new(), + start: None, + }, rank: None, result: db .rooms .get_pdu_from_id(&result)? - // TODO this is an awkward type conversion see method - .map(|pdu| pdu.to_any_event()), + .map(|pdu| pdu.to_room_event()), }) }) .filter_map(|r| r.ok()) @@ -72,14 +76,14 @@ pub fn search_events_route( }; Ok(search_events::Response::new(ResultCategories { - room_events: Some(ResultRoomEvents { - count: uint!(0), // TODO + room_events: ResultRoomEvents { + count: None, // TODO? maybe not groups: BTreeMap::new(), // TODO next_batch, results, state: BTreeMap::new(), // TODO highlights: search.1, - }), + }, }) .into()) } diff --git a/src/client_server/session.rs b/src/client_server/session.rs index 948b455..9cd051c 100644 --- a/src/client_server/session.rs +++ b/src/client_server/session.rs @@ -36,7 +36,7 @@ pub fn get_login_types_route() -> ConduitResult { )] pub fn login_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { // Validate login method let user_id = diff --git a/src/client_server/state.rs b/src/client_server/state.rs index e7d2bcf..75463cb 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -21,7 +21,7 @@ use rocket::{get, put}; )] pub fn send_state_event_for_key_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -40,7 +40,7 @@ pub fn send_state_event_for_key_route( &body.content, content, &body.room_id, - Some(body.state_key.clone()), + Some(body.state_key.to_owned()), )?) .into(), ) @@ -52,7 +52,7 @@ pub fn send_state_event_for_key_route( )] pub fn send_state_event_for_empty_key_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { // This just calls send_state_event_for_key_route let Ruma { diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 7432960..167ee75 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -31,7 +31,7 @@ use std::{ )] pub async fn sync_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/tag.rs b/src/client_server/tag.rs index 99ee6e3..d04dd3a 100644 --- a/src/client_server/tag.rs +++ b/src/client_server/tag.rs @@ -15,7 +15,7 @@ use rocket::{delete, get, put}; )] pub fn update_tag_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -49,7 +49,7 @@ pub fn update_tag_route( )] pub fn delete_tag_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -80,7 +80,7 @@ pub fn delete_tag_route( )] pub fn get_tags_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/to_device.rs b/src/client_server/to_device.rs index ca423fe..fe74101 100644 --- a/src/client_server/to_device.rs +++ b/src/client_server/to_device.rs @@ -14,7 +14,7 @@ use rocket::put; )] pub fn send_event_to_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/typing.rs b/src/client_server/typing.rs index 7eba13e..b15121c 100644 --- a/src/client_server/typing.rs +++ b/src/client_server/typing.rs @@ -1,5 +1,6 @@ use super::State; use crate::{utils, ConduitResult, Database, Ruma}; +use create_typing_event::Typing; use ruma::api::client::r0::typing::create_typing_event; #[cfg(feature = "conduit_bin")] @@ -11,16 +12,15 @@ use rocket::put; )] pub fn create_typing_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - if body.typing { + if let Typing::Yes(duration) = body.state { db.rooms.edus.roomactive_add( &sender_id, &body.room_id, - body.timeout.map(|d| d.as_millis() as u64).unwrap_or(30000) - + utils::millis_since_unix_epoch(), + duration.as_millis() as u64 + utils::millis_since_unix_epoch(), &db.globals, )?; } else { diff --git a/src/client_server/user_directory.rs b/src/client_server/user_directory.rs index f47643c..dcf48fe 100644 --- a/src/client_server/user_directory.rs +++ b/src/client_server/user_directory.rs @@ -11,7 +11,7 @@ use rocket::post; )] pub fn search_users_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let limit = u64::from(body.limit) as usize; diff --git a/src/database/media.rs b/src/database/media.rs index e9dcb4a..63fa11c 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -16,7 +16,7 @@ impl Media { pub fn create( &self, mxc: String, - filename: Option<&String>, + filename: &Option, content_type: &str, file: &[u8], ) -> Result<()> { @@ -25,7 +25,7 @@ impl Media { key.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail key.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail key.push(0xff); - key.extend_from_slice(filename.map(|f| f.as_bytes()).unwrap_or_default()); + key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default()); key.push(0xff); key.extend_from_slice(content_type.as_bytes()); diff --git a/src/database/users.rs b/src/database/users.rs index 1b6a681..10e1ef3 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -8,7 +8,7 @@ use ruma::{ keys::{CrossSigningKey, OneTimeKey}, }, }, - encryption::DeviceKeys, + encryption::IncomingDeviceKeys, events::{AnyToDeviceEvent, EventType}, DeviceId, DeviceKeyAlgorithm, DeviceKeyId, Raw, UserId, }; @@ -395,7 +395,7 @@ impl Users { &self, user_id: &UserId, device_id: &DeviceId, - device_keys: &DeviceKeys, + device_keys: &IncomingDeviceKeys, rooms: &super::rooms::Rooms, globals: &super::globals::Globals, ) -> Result<()> { @@ -625,7 +625,7 @@ impl Users { &self, user_id: &UserId, device_id: &DeviceId, - ) -> Result> { + ) -> Result> { let mut key = user_id.to_string().as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(device_id.as_bytes()); diff --git a/src/error.rs b/src/error.rs index 623aa0e..f521da4 100644 --- a/src/error.rs +++ b/src/error.rs @@ -70,14 +70,14 @@ where use ErrorKind::*; let (kind, status_code) = match self { Self::BadRequest(kind, _) => ( - kind, + kind.clone(), match kind { Forbidden | GuestAccessForbidden | ThreepidAuthFailed | ThreepidDenied => { StatusCode::FORBIDDEN } - Unauthorized | UnknownToken | MissingToken => StatusCode::UNAUTHORIZED, + Unauthorized | UnknownToken { .. } | MissingToken => StatusCode::UNAUTHORIZED, NotFound => StatusCode::NOT_FOUND, - LimitExceeded => StatusCode::TOO_MANY_REQUESTS, + LimitExceeded { .. } => StatusCode::TOO_MANY_REQUESTS, UserDeactivated => StatusCode::FORBIDDEN, TooLarge => StatusCode::PAYLOAD_TOO_LARGE, _ => StatusCode::BAD_REQUEST, diff --git a/src/pdu.rs b/src/pdu.rs index b565a24..1526484 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -255,7 +255,7 @@ impl PduEvent { } /// Build the start of a PDU in order to add it to the `Database`. -#[derive(Debug)] +#[derive(Debug, Deserialize)] pub struct PduBuilder { pub room_id: RoomId, pub sender: UserId, diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 80e6e58..734d214 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -1,9 +1,9 @@ use crate::Error; use ruma::{ - api::IncomingRequest, + api::{Outgoing, OutgoingRequest}, identifiers::{DeviceId, UserId}, }; -use std::{convert::TryInto, ops::Deref}; +use std::{convert::TryFrom, convert::TryInto, ops::Deref}; #[cfg(feature = "conduit_bin")] use { @@ -24,15 +24,21 @@ use { /// This struct converts rocket requests into ruma structs by converting them into http requests /// first. -pub struct Ruma { - pub body: T, +pub struct Ruma { + pub body: T::Incoming, pub sender_id: Option, pub device_id: Option>, pub json_body: Option>, // This is None when body is not a valid string } #[cfg(feature = "conduit_bin")] -impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { +impl<'a, T: Outgoing + OutgoingRequest> FromTransformedData<'a> for Ruma +where + ::Incoming: TryFrom>> + std::fmt::Debug, + <::Incoming as std::convert::TryFrom< + http::request::Request>, + >>::Error: std::fmt::Debug, +{ type Error = (); // TODO: Better error handling type Owned = Data; type Borrowed = Self::Owned; @@ -93,7 +99,7 @@ impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { let http_request = http_request.body(body.clone()).unwrap(); log::info!("{:?}", http_request); - match T::try_from(http_request) { + match ::Incoming::try_from(http_request) { Ok(t) => Success(Ruma { body: t, sender_id: user_id, @@ -112,8 +118,8 @@ impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { } } -impl Deref for Ruma { - type Target = T; +impl Deref for Ruma { + type Target = T::Incoming; fn deref(&self) -> &Self::Target { &self.body diff --git a/src/server_server.rs b/src/server_server.rs index ac4407b..d39abe6 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -207,7 +207,7 @@ pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json { )] pub async fn get_public_rooms_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let Ruma { body: @@ -230,7 +230,7 @@ pub async fn get_public_rooms_route( limit, since.as_deref(), None, - Some(ruma::directory::RoomNetwork::Matrix), + Some(ruma::directory::IncomingRoomNetwork::Matrix), ) .await? .0; @@ -264,7 +264,7 @@ pub async fn get_public_rooms_route( )] pub fn send_transaction_message_route<'a>( _db: State<'a, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { dbg!(&*body); Ok(send_transaction_message::v1::Response { From 12a8c9badd202f3019cab71a1f3a3134c8fb75ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 12 Sep 2020 21:30:07 +0200 Subject: [PATCH 19/48] fix: join rooms over federation --- Cargo.lock | 140 +++++++------ Cargo.toml | 8 +- src/client_server/account.rs | 4 +- src/client_server/alias.rs | 4 +- src/client_server/directory.rs | 2 +- src/client_server/media.rs | 8 +- src/client_server/membership.rs | 106 +++++----- src/client_server/message.rs | 5 +- src/client_server/profile.rs | 8 +- src/client_server/redact.rs | 4 +- src/client_server/room.rs | 51 ++--- src/client_server/search.rs | 2 +- src/client_server/state.rs | 4 +- src/database.rs | 1 - src/database/rooms.rs | 340 ++++++++++++++------------------ src/pdu.rs | 112 +++++------ src/server_server.rs | 1 + 17 files changed, 395 insertions(+), 405 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c88c578..dc215c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,9 +116,9 @@ checksum = "4af5687fe33aec5e70ef14caac5e0d363e335e5e5d6385fb75978d0c241b1d67" [[package]] name = "async-trait" -version = "0.1.38" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1a4a2f97ce50c9d0282c1468816208588441492b40d813b2e0419c22c05e7f" +checksum = "687c230d85c0a52504709705fc8a53e4a692b83a2184f03dae73e38e1e93a783" dependencies = [ "proc-macro2", "quote", @@ -223,9 +223,9 @@ checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" [[package]] name = "bytemuck" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7a1029718df60331e557c9e83a55523c955e5dd2a7bfeffad6bbd50b538ae9" +checksum = "41aa2ec95ca3b5c54cf73c91acf06d24f4495d5f1b1c12506ae3483d646177ac" [[package]] name = "byteorder" @@ -301,6 +301,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "const_fn" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -319,7 +325,7 @@ dependencies = [ "percent-encoding", "rand", "sha2", - "time 0.2.16", + "time 0.2.19", "version_check", ] @@ -660,9 +666,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ "cfg-if", "libc", @@ -721,12 +727,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.8.2" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b62f79061a0bc2e046024cb7ba44b08419ed238ecbd9adbd787434b9e8c25" -dependencies = [ - "autocfg", -] +checksum = "00d63df3d41950fb462ed38308eea019113ad1508da725bbedcd0fa5a85ef5f7" [[package]] name = "heck" @@ -843,9 +846,9 @@ dependencies = [ [[package]] name = "image" -version = "0.23.8" +version = "0.23.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "543904170510c1b5fb65140485d84de4a57fddb2ed685481e9020ce3d2c9f64c" +checksum = "974e194911d1f7efe3cd8a8f9db3b767e43536327e899e8bc9a12ef5711b74d2" dependencies = [ "bytemuck", "byteorder", @@ -859,9 +862,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b45e59b16c76b11bf9738fd5d38879d3bd28ad292d7b313608becb17ae2df9" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" dependencies = [ "autocfg", "hashbrown", @@ -920,9 +923,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.44" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a7e2c92a4804dd459b86c339278d0fe87cf93757fae222c3fa3ae75458bc73" +checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" dependencies = [ "wasm-bindgen", ] @@ -954,9 +957,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.76" +version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3" +checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" [[package]] name = "lock_api" @@ -1120,9 +1123,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.34" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" dependencies = [ "cfg-if", "libc", @@ -1379,9 +1382,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.19" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" +checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c" dependencies = [ "unicode-xid", ] @@ -1587,7 +1590,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "state", - "time 0.2.16", + "time 0.2.19", "tokio", "toml", "version_check", @@ -1622,7 +1625,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.16", + "time 0.2.19", "tokio", "tokio-rustls", "unicode-xid", @@ -1631,6 +1634,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1646,6 +1650,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "http", "percent-encoding", @@ -1660,6 +1665,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1670,6 +1676,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "ruma-api", "ruma-common", @@ -1682,6 +1689,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "assign", "http", @@ -1700,6 +1708,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "js_int", "ruma-api", @@ -1713,6 +1722,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "js_int", "ruma-common", @@ -1727,6 +1737,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1737,6 +1748,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "js_int", "ruma-api", @@ -1751,6 +1763,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1762,6 +1775,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "proc-macro2", "quote", @@ -1772,6 +1786,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "serde", "strum", @@ -1780,6 +1795,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "form_urlencoded", "itoa", @@ -1791,6 +1807,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "base64", "ring", @@ -1910,18 +1927,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.115" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54c9a88f2da7238af84b5101443f0c0d0a3bbdc455e34a5c9497b1903ed55d5" +checksum = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.115" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609feed1d0a73cc36a0182a840a9b37b4a82f0b1150369f0536a9e3f2a31dc48" +checksum = "f630a6370fd8e457873b4bd2ffdae75408bc291ba72be773772a4c2a065d9ae8" dependencies = [ "proc-macro2", "quote", @@ -2021,9 +2038,9 @@ checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" [[package]] name = "socket2" -version = "0.3.12" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" dependencies = [ "cfg-if", "libc", @@ -2055,6 +2072,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#0081081604b051d412a2365b68357e064c33320c" dependencies = [ "itertools", "js_int", @@ -2139,15 +2157,15 @@ dependencies = [ [[package]] name = "subtle" -version = "2.2.3" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" +checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" [[package]] name = "syn" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9" +checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350" dependencies = [ "proc-macro2", "quote", @@ -2210,11 +2228,11 @@ dependencies = [ [[package]] name = "time" -version = "0.2.16" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a51cadc5b1eec673a685ff7c33192ff7b7603d0b75446fb354939ee615acb15" +checksum = "80c1a1fd93112fc50b11c43a1def21f926be3c18884fad676ea879572da070a1" dependencies = [ - "cfg-if", + "const_fn", "libc", "standback", "stdweb", @@ -2288,9 +2306,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "228139ddd4fea3fa345a29233009635235833e52807af7ea6448ead03890d6a9" +checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", "rustls", @@ -2362,9 +2380,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f0e00789804e99b20f12bc7003ca416309d28a6f495d6af58d1e2c2842461b5" +checksum = "5bcf46c1f1f06aeea2d6b81f3c863d0930a596c86ad1920d4e5bad6dd1d7119a" dependencies = [ "lazy_static", ] @@ -2382,9 +2400,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6ccba2f8f16e0ed268fc765d9b7ff22e965e7185d32f8f1ec8294fe17d86e79" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" dependencies = [ "serde", "tracing-core", @@ -2392,9 +2410,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abd165311cc4d7a555ad11cc77a37756df836182db0d81aac908c8184c584f40" +checksum = "82bb5079aa76438620837198db8a5c529fb9878c730bc2b28179b0241cf04c10" dependencies = [ "ansi_term", "chrono", @@ -2525,9 +2543,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0563a9a4b071746dd5aedbc3a28c6fe9be4586fb3fbadb67c400d4f53c6b16c" +checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" dependencies = [ "cfg-if", "serde", @@ -2537,9 +2555,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc71e4c5efa60fb9e74160e89b93353bc24059999c0ae0fb03affc39770310b0" +checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" dependencies = [ "bumpalo", "lazy_static", @@ -2552,9 +2570,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95f8d235a77f880bcef268d379810ea6c0af2eacfa90b1ad5af731776e0c4699" +checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" dependencies = [ "cfg-if", "js-sys", @@ -2564,9 +2582,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97c57cefa5fa80e2ba15641578b44d36e7a64279bc5ed43c6dbaf329457a2ed2" +checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2574,9 +2592,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841a6d1c35c6f596ccea1f82504a192a60378f64b3bb0261904ad8f2f5657556" +checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" dependencies = [ "proc-macro2", "quote", @@ -2587,15 +2605,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93b162580e34310e5931c4b792560108b10fd14d64915d7fff8ff00180e70092" +checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" [[package]] name = "web-sys" -version = "0.3.44" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda38f4e5ca63eda02c059d243aa25b5f35ab98451e518c51612cd0f1bd19a47" +checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 5d35433..1b7a700 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,8 @@ edition = "2018" #rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } #ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers -#ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers -ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } +ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" } # Used for matrix spec type definitions and helpers +#ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently log = "0.4.8" # Used for emitting log entries @@ -32,8 +32,8 @@ reqwest = "0.10.6" # Used to send requests thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key -#state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } -state-res = { path = "../state-res" } +state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } +#state-res = { path = "../state-res", features = ["unstable-pre-spec"] } ring = "0.16.15" [features] diff --git a/src/client_server/account.rs b/src/client_server/account.rs index cb77a15..3db933c 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -356,14 +356,14 @@ pub fn deactivate_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 1d30261..bfdaeca 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -64,9 +64,7 @@ pub async fn get_alias_helper( let response = server_server::send_request( &db, room_alias.server_name().to_string(), - federation::query::get_room_information::v1::Request { - room_alias, - }, + federation::query::get_room_information::v1::Request { room_alias }, ) .await?; diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 34feb71..2764d2c 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,7 +14,7 @@ use ruma::{ }, federation, }, - directory::{IncomingFilter, PublicRoomsChunk, IncomingRoomNetwork}, + directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 038012e..d077447 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -34,12 +34,8 @@ pub fn create_content_route( db.globals.server_name(), utils::random_string(MXC_LENGTH) ); - db.media.create( - mxc.clone(), - &body.filename, - &body.content_type, - &body.file, - )?; + db.media + .create(mxc.clone(), &body.filename, &body.content_type, &body.file)?; Ok(create_content::Response { content_uri: mxc }.into()) } diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 606e470..ea2271b 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -4,6 +4,7 @@ use crate::{ pdu::{PduBuilder, PduEvent}, server_server, utils, ConduitResult, Database, Error, Ruma, }; +use log::warn; use ruma::{ api::{ client::{ @@ -20,8 +21,7 @@ use ruma::{ EventId, Raw, RoomId, RoomVersionId, UserId, }; use state_res::StateEvent; - -use std::{collections::BTreeMap, convert::TryFrom}; +use std::{collections::BTreeMap, convert::TryFrom, sync::Arc}; #[cfg(feature = "conduit_bin")] use rocket::{get, post}; @@ -106,14 +106,14 @@ pub fn leave_room_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -134,8 +134,6 @@ pub fn invite_user_route( if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Invite, @@ -149,6 +147,8 @@ pub fn invite_user_route( state_key: Some(user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -191,14 +191,14 @@ pub fn kick_user_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -246,14 +246,14 @@ pub fn ban_user_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -292,14 +292,14 @@ pub fn unban_user_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -473,7 +473,7 @@ async fn join_room_by_id_helper( let send_join_response = server_server::send_request( &db, room_id.server_name().to_string(), - federation::membership::create_join_event::v1::Request { + federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, pdu_stub: serde_json::from_value(join_event_stub_value) @@ -482,25 +482,39 @@ async fn join_room_by_id_helper( ) .await?; - dbg!(&send_join_response); - let mut event_map = send_join_response .room_state .state .iter() .chain(send_join_response.room_state.auth_chain.iter()) .map(|pdu| { - pdu.deserialize() - .map(StateEvent::Full) - .map(|ev| (ev.event_id(), ev)) + let mut value = serde_json::from_str(pdu.json().get()) + .expect("converting raw jsons to values always works"); + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value + .as_object_mut() + .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? + .insert("event_id".to_owned(), event_id.to_string().into()); + + serde_json::from_value::(value) + .map(|ev| (event_id, Arc::new(ev))) + .map_err(|e| { + warn!("{}", e); + Error::BadServerResponse("Invalid PDU bytes in send_join response.") + }) }) - .collect::, _>>() - .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; + .collect::>, _>>()?; let control_events = event_map .values() .filter(|pdu| pdu.is_power_event()) - .map(|pdu| pdu.event_id()) + .map(|pdu| pdu.event_id().clone()) .collect::>(); // These events are not guaranteed to be sorted but they are resolved according to spec @@ -515,7 +529,9 @@ async fn join_room_by_id_helper( .room_state .auth_chain .iter() - .filter_map(|pdu| Some(StateEvent::Full(pdu.deserialize().ok()?).event_id())) + .filter_map(|pdu| { + Some(StateEvent::Full(pdu.deserialize().ok()?).event_id().clone()) + }) .collect::>(), ); @@ -575,31 +591,31 @@ async fn join_room_by_id_helper( // We do not rebuild the PDU in this case only insert to DB db.rooms - .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; + .append_pdu(PduEvent::from(&**pdu), &db.globals, &db.account_data)?; } + } else { + let event = member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }; + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.account_data, + )?; } - let event = member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }; - - db.rooms.build_and_append_pdu( - PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; - Ok(join_room_by_id::Response::new(room_id.clone()).into()) } diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 09c3517..025331e 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -27,11 +27,10 @@ pub fn send_message_event_route( let event_id = db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: body.content.event_type().into(), content: serde_json::from_str( body.json_body + .as_ref() .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? .get(), ) @@ -40,6 +39,8 @@ pub fn send_message_event_route( state_key: None, redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 386d898..2a2e05e 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -33,8 +33,6 @@ pub fn set_displayname_route( let room_id = room_id?; db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(ruma::events::room::member::MemberEventContent { displayname: body.displayname.clone(), @@ -62,6 +60,8 @@ pub fn set_displayname_route( state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -136,8 +136,6 @@ pub fn set_avatar_url_route( let room_id = room_id?; db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(ruma::events::room::member::MemberEventContent { avatar_url: body.avatar_url.clone(), @@ -165,6 +163,8 @@ pub fn set_avatar_url_route( state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index cd1b443..5117348 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -20,8 +20,6 @@ pub fn redact_event_route( let event_id = db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomRedaction, content: serde_json::to_value(redaction::RedactionEventContent { reason: body.reason.clone(), @@ -31,6 +29,8 @@ pub fn redact_event_route( state_key: None, redacts: Some(body.event_id.clone()), }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 9918123..9a83f81 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -55,14 +55,14 @@ pub fn create_room_route( // 1. The room create event db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomCreate, content: serde_json::to_value(content).expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -70,8 +70,6 @@ pub fn create_room_route( // 2. Let the room creator join db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Join, @@ -85,6 +83,8 @@ pub fn create_room_route( state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -119,14 +119,14 @@ pub fn create_room_route( }; db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomPowerLevels, content: power_levels_content, unsigned: None, state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -142,8 +142,6 @@ pub fn create_room_route( // 4.1 Join Rules db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomJoinRules, content: match preset { create_room::RoomPreset::PublicChat => serde_json::to_value( @@ -160,6 +158,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -167,8 +167,6 @@ pub fn create_room_route( // 4.2 History Visibility db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomHistoryVisibility, content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( history_visibility::HistoryVisibility::Shared, @@ -178,6 +176,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -185,8 +185,6 @@ pub fn create_room_route( // 4.3 Guest Access db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomGuestAccess, content: match preset { create_room::RoomPreset::PublicChat => { @@ -204,6 +202,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -212,24 +212,27 @@ pub fn create_room_route( for event in &body.initial_state { let pdu_builder = serde_json::from_str::( &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"), - ).map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; // Silently skip encryption events if they are not allowed - if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() - { + if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() { continue; } - db.rooms - .build_and_append_pdu(pdu_builder, &db.globals, &db.account_data)?; + db.rooms.build_and_append_pdu( + pdu_builder, + &sender_id, + &room_id, + &db.globals, + &db.account_data, + )?; } // 6. Events implied by name and topic if let Some(name) = &body.name { db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomName, content: serde_json::to_value( name::NameEventContent::new(name.clone()).map_err(|_| { @@ -241,6 +244,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -249,8 +254,6 @@ pub fn create_room_route( if let Some(topic) = &body.topic { db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomTopic, content: serde_json::to_value(topic::TopicEventContent { topic: topic.clone(), @@ -260,6 +263,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -269,8 +274,6 @@ pub fn create_room_route( for user in &body.invite { db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Invite, @@ -284,6 +287,8 @@ pub fn create_room_route( state_key: Some(user.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/search.rs b/src/client_server/search.rs index 2967e00..3b03e7a 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -77,7 +77,7 @@ pub fn search_events_route( Ok(search_events::Response::new(ResultCategories { room_events: ResultRoomEvents { - count: None, // TODO? maybe not + count: None, // TODO? maybe not groups: BTreeMap::new(), // TODO next_batch, results, diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 75463cb..1fe3cd6 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -213,14 +213,14 @@ pub fn send_state_event_for_key_helper( let event_id = db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: content.event_type().into(), content: json, unsigned: None, state_key, redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/database.rs b/src/database.rs index a105058..0d18020 100644 --- a/src/database.rs +++ b/src/database.rs @@ -97,7 +97,6 @@ impl Database { }, pduid_pdu: db.open_tree("pduid_pdu")?, eventid_pduid: db.open_tree("eventid_pduid")?, - roomstateid_pduid: db.open_tree("roomstateid_pduid")?, roomid_pduleaves: db.open_tree("roomid_pduleaves")?, alias_roomid: db.open_tree("alias_roomid")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index ee070b3..6033378 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -25,6 +25,7 @@ use std::{ collections::{BTreeMap, HashMap}, convert::{TryFrom, TryInto}, mem, + sync::Arc, }; /// The unique identifier of each state group. @@ -33,10 +34,6 @@ use std::{ /// hashing the entire state. pub type StateHashId = Vec; -/// This identifier consists of roomId + count. It represents a -/// unique event, it will never be overwritten or removed. -pub type PduId = IVec; - pub struct Rooms { pub edus: edus::RoomEdus, pub(super) pduid_pdu: sled::Tree, // PduId = RoomId + Count @@ -54,22 +51,22 @@ pub struct Rooms { pub(super) roomuserid_invited: sled::Tree, pub(super) userroomid_left: sled::Tree, - // STATE TREES - /// This holds the full current state, including the latest event. - pub(super) roomstateid_pduid: sled::Tree, // RoomStateId = Room + StateType + StateKey - /// This holds the full room state minus the latest event. - pub(super) pduid_statehash: sled::Tree, // PDU id -> StateHash - /// Also holds the full room state minus the latest event. - pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + (EventType, StateKey) - /// The room_id -> the latest StateHash + /// Remember the current state hash of a room. pub(super) roomid_statehash: sled::Tree, + /// Remember the state hash at events in the past. + pub(super) pduid_statehash: sled::Tree, + /// The state for a given state hash. + pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + EventType + StateKey } impl StateStore for Rooms { - fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> state_res::Result { + fn get_event( + &self, + room_id: &RoomId, + event_id: &EventId, + ) -> state_res::Result> { let pid = self - .eventid_pduid - .get(event_id.as_bytes()) + .get_pdu_id(event_id) .map_err(StateError::custom)? .ok_or_else(|| { StateError::NotFound("PDU via room_id and event_id not found in the db.".into()) @@ -87,7 +84,7 @@ impl StateStore for Rooms { // conduit's PDU's always contain a room_id but some // of ruma's do not so this must be an Option if pdu.room_id() == Some(room_id) { - Ok(pdu) + Ok(Arc::new(pdu)) } else { Err(StateError::NotFound( "Found PDU for incorrect room in db.".into(), @@ -136,53 +133,12 @@ impl Rooms { None } - /// Fetch the current State using the `roomstateid_pduid` tree. - pub fn current_state_pduids(&self, room_id: &RoomId) -> Result> { - // TODO this could also scan roomstateid_pduid if we passed in room_id ? - self.roomstateid_pduid - .scan_prefix(room_id.as_bytes()) - .values() - .map(|pduid| { - let pduid = &pduid?; - self.pduid_pdu.get(pduid)?.map_or_else( - || { - Err(Error::bad_database( - "Failed to find current state of pduid's.", - )) - }, - |b| { - Ok(( - serde_json::from_slice::(&b) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - pduid.clone(), - )) - }, - ) - }) - .map(|pair| { - let (pdu, id) = pair?; - Ok(((pdu.kind, pdu.state_key), id)) - }) - .collect::>>() - } - /// Returns the last state hash key added to the db. - pub fn current_state_hash(&self, room_id: &RoomId) -> Result { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xff); - - // We must check here because this method is called outside and before - // `append_state_pdu` so the DB can be empty - if self.pduid_statehash.scan_prefix(prefix).next().is_none() { - // return the hash of the room_id, this represents a room with no state - return self.new_state_hash_id(room_id); - } - - self.pduid_statehash - .iter() - .next_back() - .map(|pair| Ok(pair?.1.to_vec())) - .ok_or_else(|| Error::bad_database("No PDU's found for this room."))? + pub fn current_state_hash(&self, room_id: &RoomId) -> Result> { + Ok(self + .roomid_statehash + .get(room_id.as_bytes())? + .map(|bytes| bytes.to_vec())) } /// This fetches auth event_ids from the current state using the @@ -243,39 +199,11 @@ impl Rooms { /// Generate a new StateHash. /// - /// A unique hash made from hashing the current states pduid's. - fn new_state_hash_id(&self, room_id: &RoomId) -> Result { - // Use hashed roomId as the first StateHash key for first state event in room - if self - .pduid_statehash - .scan_prefix(room_id.as_bytes()) - .next() - .is_none() - { - return Ok(digest::digest(&digest::SHA256, room_id.as_bytes()) - .as_ref() - .to_vec()); - } - - let pdu_ids_to_hash = self - .pduid_statehash - .scan_prefix(room_id.as_bytes()) - .values() - .next_back() - .unwrap() // We just checked if the tree was empty - .map(|hash| { - self.stateid_pduid - .scan_prefix(hash) - .values() - // pduid is roomId + count so just hash the whole thing - .map(|pid| Ok(pid?.to_vec())) - .collect::>>>() - })??; - - let hash = digest::digest( - &digest::SHA256, - &pdu_ids_to_hash.into_iter().flatten().collect::>(), - ); + /// A unique hash made from hashing all PDU ids of the state joined with 0xff. + fn calculate_hash(&self, pdu_id_bytes: &[&[u8]]) -> Result { + // We only hash the pdu's event ids, not the whole pdu + let bytes = pdu_id_bytes.join(&0xff); + let hash = digest::digest(&digest::SHA256, &bytes); Ok(hash.as_ref().to_vec()) } @@ -297,29 +225,38 @@ impl Rooms { &self, room_id: &RoomId, ) -> Result> { - let mut hashmap = HashMap::new(); - for pdu in - self.roomstateid_pduid - .scan_prefix(&room_id.as_bytes()) + if let Some(current_state_hash) = self.current_state_hash(room_id)? { + let mut prefix = current_state_hash; + prefix.push(0xff); + + let mut hashmap = HashMap::new(); + for pdu in self + .stateid_pduid + .scan_prefix(prefix) .values() - .map(|value| { + .map(|pdu_id| { Ok::<_, Error>( serde_json::from_slice::( - &self.pduid_pdu.get(value?)?.ok_or_else(|| { - Error::bad_database("PDU not found for ID in db.") + &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { + Error::bad_database("PDU in state not found in database.") })?, ) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + .map_err(|_| { + Error::bad_database("Invalid PDU bytes in current room state.") + })?, ) }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert((pdu.kind.clone(), state_key), pdu); + { + let pdu = pdu?; + let state_key = pdu.state_key.clone().ok_or_else(|| { + Error::bad_database("Room state contains event without state_key.") + })?; + hashmap.insert((pdu.kind.clone(), state_key), pdu); + } + Ok(hashmap) + } else { + Ok(HashMap::new()) } - Ok(hashmap) } /// Returns all state entries for this type. @@ -328,33 +265,40 @@ impl Rooms { room_id: &RoomId, event_type: &EventType, ) -> Result> { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xff); - prefix.extend_from_slice(&event_type.to_string().as_bytes()); + if let Some(current_state_hash) = self.current_state_hash(room_id)? { + let mut prefix = current_state_hash; + prefix.push(0xff); + prefix.extend_from_slice(&event_type.to_string().as_bytes()); + prefix.push(0xff); - let mut hashmap = HashMap::new(); - for pdu in - self.roomstateid_pduid + let mut hashmap = HashMap::new(); + for pdu in self + .stateid_pduid .scan_prefix(&prefix) .values() - .map(|value| { + .map(|pdu_id| { Ok::<_, Error>( serde_json::from_slice::( - &self.pduid_pdu.get(value?)?.ok_or_else(|| { - Error::bad_database("PDU not found for ID in db.") + &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { + Error::bad_database("PDU in state not found in database.") })?, ) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + .map_err(|_| { + Error::bad_database("Invalid PDU bytes in current room state.") + })?, ) }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert(state_key, pdu); + { + let pdu = pdu?; + let state_key = pdu.state_key.clone().ok_or_else(|| { + Error::bad_database("Room state contains event without state_key.") + })?; + hashmap.insert(state_key, pdu); + } + Ok(hashmap) + } else { + Ok(HashMap::new()) } - Ok(hashmap) } /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). @@ -364,23 +308,24 @@ impl Rooms { event_type: &EventType, state_key: &str, ) -> Result> { - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(&event_type.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(&state_key.as_bytes()); + if let Some(current_state_hash) = self.current_state_hash(room_id)? { + let mut key = current_state_hash; + key.push(0xff); + key.extend_from_slice(&event_type.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(&state_key.as_bytes()); - self.roomstateid_pduid.get(&key)?.map_or(Ok(None), |value| { - Ok::<_, Error>(Some( - serde_json::from_slice::( - &self - .pduid_pdu - .get(value)? - .ok_or_else(|| Error::bad_database("PDU not found for ID in db."))?, - ) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - )) - }) + self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| { + Ok::<_, Error>(Some( + serde_json::from_slice::(&self.pduid_pdu.get(pdu_id)?.ok_or_else( + || Error::bad_database("PDU in state not found in database."), + )?) + .map_err(|_| Error::bad_database("Invalid PDU bytes in current room state."))?, + )) + }) + } else { + Ok(None) + } } /// Returns the `count` of this pdu's id. @@ -528,8 +473,8 @@ impl Rooms { self.eventid_pduid .insert(pdu.event_id.as_bytes(), &*pdu_id)?; - if let Some(state_key) = &pdu.state_key { - self.append_state_pdu(&pdu.room_id, &pdu_id, state_key, &pdu.kind)?; + if pdu.state_key.is_some() { + self.append_to_state(&pdu_id, &pdu)?; } match &pdu.kind { @@ -603,59 +548,69 @@ impl Rooms { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. /// The incoming event is the `pdu_id` passed to this method. - fn append_state_pdu( - &self, - room_id: &RoomId, - pdu_id: &[u8], - state_key: &str, - kind: &EventType, - ) -> Result { - let state_hash = self.new_state_hash_id(room_id)?; - let state = self.current_state_pduids(room_id)?; + fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result { + let old_state = + if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? { + // Store state for event. The state does not include the event itself. + // Instead it's the state before the pdu, so the room's old state. + self.pduid_statehash.insert(new_pdu_id, &old_state_hash)?; + if new_pdu.state_key.is_none() { + return Ok(old_state_hash.to_vec()); + } - let mut key = state_hash.to_vec(); - key.push(0xff); + let mut prefix = old_state_hash.to_vec(); + prefix.push(0xff); + self.stateid_pduid + .scan_prefix(&prefix) + .filter_map(|pdu| pdu.map_err(|e| error!("{}", e)).ok()) + .map(|(k, v)| (k.subslice(prefix.len(), k.len() - prefix.len()), v)) + .collect::>() + } else { + HashMap::new() + }; - // TODO eventually we could avoid writing to the DB so much on every event - // by keeping track of the delta and write that every so often - for ((ev_ty, state_key), pid) in state { - let mut state_id = key.to_vec(); - state_id.extend_from_slice(ev_ty.to_string().as_bytes()); - key.push(0xff); - state_id.extend_from_slice(state_key.expect("state event").as_bytes()); + if let Some(state_key) = &new_pdu.state_key { + let mut new_state = old_state; + let mut pdu_key = new_pdu.kind.as_str().as_bytes().to_vec(); + pdu_key.push(0xff); + pdu_key.extend_from_slice(state_key.as_bytes()); + new_state.insert(pdu_key.into(), new_pdu_id.into()); + + let new_state_hash = + self.calculate_hash(&new_state.values().map(|b| &**b).collect::>())?; + + let mut key = new_state_hash.to_vec(); key.push(0xff); - self.stateid_pduid.insert(&state_id, &pid)?; + // TODO: we could avoid writing to the DB on every state event by keeping + // track of the delta and write that every so often + for (key_without_prefix, pdu_id) in new_state { + let mut state_id = key.clone(); + state_id.extend_from_slice(&key_without_prefix); + self.stateid_pduid.insert(&state_id, &pdu_id)?; + } + + self.roomid_statehash + .insert(new_pdu.room_id.as_bytes(), &*new_state_hash)?; + + Ok(new_state_hash) + } else { + Err(Error::bad_database( + "Tried to insert non-state event into room without a state.", + )) } - - // This event's state does not include the event itself. `current_state_pduids` - // uses `roomstateid_pduid` before the current event is inserted to the tree so the state - // will be everything up to but not including the incoming event. - self.pduid_statehash.insert(pdu_id, state_hash.as_slice())?; - - self.roomid_statehash - .insert(room_id.as_bytes(), state_hash.as_slice())?; - - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(kind.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pduid.insert(key, pdu_id)?; - - Ok(state_hash) } /// Creates a new persisted data unit and adds it to a room. pub fn build_and_append_pdu( &self, pdu_builder: PduBuilder, + sender: &UserId, + room_id: &RoomId, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, ) -> Result { let PduBuilder { - room_id, - sender, event_type, content, unsigned, @@ -741,8 +696,7 @@ impl Rooms { ErrorKind::Unknown, "Membership can't be the first event", ))?)? - .map(|pdu| pdu.convert_for_state_res()) - .transpose()?; + .map(|pdu| pdu.convert_for_state_res()); event_auth::valid_membership_change( // TODO this is a bit of a hack but not sure how to have a type // declared in `state_res` crate easily convert to/from conduit::PduEvent @@ -753,11 +707,12 @@ impl Rooms { state_key: Some(state_key.to_owned()), sender: &sender, }, - prev_event.as_ref(), + prev_event, + None, &auth_events .iter() .map(|((ty, key), pdu)| { - Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) + Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res())) }) .collect::>>()?, ) @@ -812,9 +767,8 @@ impl Rooms { let mut pdu = PduEvent { event_id: EventId::try_from("$thiswillbefilledinlater").expect("we know this is valid"), - room_id, - sender, - origin: globals.server_name().to_owned(), + room_id: room_id.clone(), + sender: sender.clone(), origin_server_ts: utils::millis_since_unix_epoch() .try_into() .expect("time is valid"), @@ -834,7 +788,7 @@ impl Rooms { hashes: ruma::events::pdu::EventHash { sha256: "aaa".to_owned(), }, - signatures: HashMap::new(), + signatures: BTreeMap::new(), }; // Generate event id @@ -1028,8 +982,6 @@ impl Rooms { self.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: user_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member_content) .expect("event is valid, we just created it"), @@ -1037,6 +989,8 @@ impl Rooms { state_key: Some(user_id.to_string()), redacts: None, }, + &user_id, + &room_id, globals, account_data, )?; diff --git a/src/pdu.rs b/src/pdu.rs index 1526484..f23e688 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -5,18 +5,17 @@ use ruma::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, - EventId, Raw, RoomId, ServerName, UserId, + EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, }; use serde::{Deserialize, Serialize}; use serde_json::json; -use std::{collections::HashMap, convert::TryFrom}; +use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH}; -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, Debug)] pub struct PduEvent { pub event_id: EventId, pub room_id: RoomId, pub sender: UserId, - pub origin: Box, pub origin_server_ts: UInt, #[serde(rename = "type")] pub kind: EventType, @@ -31,7 +30,7 @@ pub struct PduEvent { #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] pub unsigned: serde_json::Map, pub hashes: EventHash, - pub signatures: HashMap>, + pub signatures: BTreeMap, BTreeMap>, } impl PduEvent { @@ -199,66 +198,69 @@ impl PduEvent { } } -impl TryFrom<&state_res::StateEvent> for PduEvent { - type Error = Error; - fn try_from(pdu: &state_res::StateEvent) -> Result { - serde_json::from_value(json!({ - "event_id": pdu.event_id(), - "room_id": pdu.room_id(), - "sender": pdu.sender(), - "origin": pdu.origin(), - "origin_server_ts": pdu.origin_server_ts(), - "event_type": pdu.kind(), - "content": pdu.content(), - "state_key": pdu.state_key(), - "prev_events": pdu.prev_event_ids(), - "depth": pdu.depth(), - "auth_events": pdu.auth_events(), - "redacts": pdu.redacts(), - "unsigned": pdu.unsigned(), - "hashes": pdu.hashes(), - "signatures": pdu.signatures(), - })) - .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) +impl From<&state_res::StateEvent> for PduEvent { + fn from(pdu: &state_res::StateEvent) -> Self { + Self { + event_id: pdu.event_id().clone(), + room_id: pdu.room_id().unwrap().clone(), + sender: pdu.sender().clone(), + origin_server_ts: (pdu + .origin_server_ts() + .duration_since(UNIX_EPOCH) + .expect("time is valid") + .as_millis() as u64) + .try_into() + .expect("time is valid"), + kind: pdu.kind(), + content: pdu.content().clone(), + state_key: pdu.state_key(), + prev_events: pdu.prev_event_ids(), + depth: pdu.depth().clone(), + auth_events: pdu.auth_events(), + redacts: pdu.redacts().cloned(), + unsigned: pdu.unsigned().clone().into_iter().collect(), + hashes: pdu.hashes().clone(), + signatures: pdu.signatures(), + } } } impl PduEvent { - pub fn convert_for_state_res(&self) -> Result { - serde_json::from_value(json!({ - "event_id": self.event_id, - "room_id": self.room_id, - "sender": self.sender, - "origin": self.origin, - "origin_server_ts": self.origin_server_ts, - "type": self.kind, - "content": self.content, - "state_key": self.state_key, - "prev_events": self.prev_events - .iter() - // TODO How do we create one of these - .map(|id| (id, EventHash { sha256: "hello".into() })) - .collect::>(), - "depth": self.depth, - "auth_events": self.auth_events - .iter() - // TODO How do we create one of these - .map(|id| (id, EventHash { sha256: "hello".into() })) - .collect::>(), - "redacts": self.redacts, - "unsigned": self.unsigned, - "hashes": self.hashes, - "signatures": self.signatures, - })) - .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) + pub fn convert_for_state_res(&self) -> Arc { + Arc::new( + serde_json::from_value(json!({ + "event_id": self.event_id, + "room_id": self.room_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "type": self.kind, + "content": self.content, + "state_key": self.state_key, + "prev_events": self.prev_events + .iter() + // TODO How do we create one of these + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "depth": self.depth, + "auth_events": self.auth_events + .iter() + // TODO How do we create one of these + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "redacts": self.redacts, + "unsigned": self.unsigned, + "hashes": self.hashes, + "signatures": self.signatures, + })) + .expect("all conduit PDUs are state events"), + ) } } /// Build the start of a PDU in order to add it to the `Database`. #[derive(Debug, Deserialize)] pub struct PduBuilder { - pub room_id: RoomId, - pub sender: UserId, + #[serde(rename = "type")] pub event_type: EventType, pub content: serde_json::Value, pub unsigned: Option>, diff --git a/src/server_server.rs b/src/server_server.rs index d39abe6..ffa5c5b 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -139,6 +139,7 @@ where .unwrap() .into_iter() .collect(); + Ok( T::IncomingResponse::try_from(http_response.body(body).unwrap()) .expect("TODO: error handle other server errors"), From 1f2843498863065ca2775bd0a02a37db3f8f0002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 12 Sep 2020 22:41:33 +0200 Subject: [PATCH 20/48] feat: hacky transactions --- src/server_server.rs | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index ffa5c5b..d7d0e23 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,16 +1,19 @@ use crate::{client_server, ConduitResult, Database, Error, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; -use ruma::api::{ - client, - federation::{ - directory::get_public_rooms, - discovery::{ - get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, +use ruma::{ + api::{ + client, + federation::{ + directory::get_public_rooms, + discovery::{ + get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, + }, + transactions::send_transaction_message, }, - transactions::send_transaction_message, + OutgoingRequest, }, - OutgoingRequest, + EventId, }; use serde_json::json; use std::{ @@ -264,10 +267,29 @@ pub async fn get_public_rooms_route( put("/_matrix/federation/v1/send/<_>", data = "") )] pub fn send_transaction_message_route<'a>( - _db: State<'a, Database>, + db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { dbg!(&*body); + for pdu in &body.pdus { + let mut value = serde_json::to_value(pdu).expect("all ruma pdus are json values"); + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value).expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value + .as_object_mut() + .expect("ruma pdus are json objects") + .insert("event_id".to_owned(), event_id.to_string().into()); + + db.rooms.append_pdu( + serde_json::from_value(value).expect("all ruma pdus are conduit pdus"), + &db.globals, + &db.account_data, + )?; + } Ok(send_transaction_message::v1::Response { pdus: BTreeMap::new(), } From af53485d70d474468b02d10badd6959fa05f1068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 12 Sep 2020 23:38:52 +0200 Subject: [PATCH 21/48] fix: avoid pdus without event ids --- src/client_server/membership.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index ea2271b..20cf315 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -502,8 +502,10 @@ async fn join_room_by_id_helper( .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? .insert("event_id".to_owned(), event_id.to_string().into()); + dbg!(&value); + serde_json::from_value::(value) - .map(|ev| (event_id, Arc::new(ev))) + .map(|ev| (dbg!(&ev).event_id().clone(), Arc::new(ev))) .map_err(|e| { warn!("{}", e); Error::BadServerResponse("Invalid PDU bytes in send_join response.") @@ -520,19 +522,14 @@ async fn join_room_by_id_helper( // These events are not guaranteed to be sorted but they are resolved according to spec // we auth them anyways to weed out faulty/malicious server. The following is basically the // full state resolution algorithm. + let event_ids = event_map.keys().cloned().collect::>(); + let sorted_control_events = state_res::StateResolution::reverse_topological_power_sort( &room_id, &control_events, &mut event_map, &db.rooms, - &send_join_response - .room_state - .auth_chain - .iter() - .filter_map(|pdu| { - Some(StateEvent::Full(pdu.deserialize().ok()?).event_id().clone()) - }) - .collect::>(), + &event_ids, ); // Auth check each event against the "partial" state created by the preceding events From 1f292c09f2e8a0679467ec5f3b699918d60cea64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 13 Sep 2020 22:24:36 +0200 Subject: [PATCH 22/48] improvement: better federation joins --- Cargo.lock | 30 ++++++------ src/client_server/membership.rs | 81 +++++++++++++++++++++---------- src/database/rooms.rs | 86 +++++++++++++++++++++------------ src/server_server.rs | 18 ++++--- 4 files changed, 136 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 646cdcc..6ffd347 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1634,7 +1634,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1650,7 +1650,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "http", "percent-encoding", @@ -1665,7 +1665,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "ruma-api", "ruma-common", @@ -1689,7 +1689,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "assign", "http", @@ -1708,7 +1708,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "js_int", "ruma-api", @@ -1722,7 +1722,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "js_int", "ruma-common", @@ -1737,7 +1737,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1748,7 +1748,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "js_int", "ruma-api", @@ -1763,7 +1763,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1775,7 +1775,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "proc-macro2", "quote", @@ -1786,7 +1786,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "serde", "strum", @@ -1795,7 +1795,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "form_urlencoded", "itoa", @@ -1807,7 +1807,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "base64", "ring", @@ -2072,7 +2072,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#0081081604b051d412a2365b68357e064c33320c" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#a9186476b748c901fbf4356414247a0b3ac01b5f" dependencies = [ "itertools", "js_int", diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 20cf315..9285648 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -2,7 +2,7 @@ use super::State; use crate::{ client_server, pdu::{PduBuilder, PduEvent}, - server_server, utils, ConduitResult, Database, Error, Ruma, + server_server, utils, ConduitResult, Database, Error, Result, Ruma, }; use log::warn; use ruma::{ @@ -17,11 +17,14 @@ use ruma::{ }, federation, }, + events::pdu::Pdu, events::{room::member, EventType}, EventId, Raw, RoomId, RoomVersionId, UserId, }; use state_res::StateEvent; -use std::{collections::BTreeMap, convert::TryFrom, sync::Arc}; +use std::{ + collections::BTreeMap, collections::HashMap, collections::HashSet, convert::TryFrom, sync::Arc, +}; #[cfg(feature = "conduit_bin")] use rocket::{get, post}; @@ -482,36 +485,49 @@ async fn join_room_by_id_helper( ) .await?; - let mut event_map = send_join_response + let add_event_id = |pdu: &Raw| { + let mut value = serde_json::from_str(pdu.json().get()) + .expect("converting raw jsons to values always works"); + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value + .as_object_mut() + .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? + .insert("event_id".to_owned(), event_id.to_string().into()); + + Ok((event_id, value)) + }; + + let room_state = send_join_response.room_state.state.iter().map(add_event_id); + + let state_events = room_state + .clone() + .map(|pdu: Result<(EventId, serde_json::Value)>| Ok(pdu?.0)) + .collect::>>()?; + + let auth_chain = send_join_response .room_state - .state + .auth_chain .iter() - .chain(send_join_response.room_state.auth_chain.iter()) - .map(|pdu| { - let mut value = serde_json::from_str(pdu.json().get()) - .expect("converting raw jsons to values always works"); - let event_id = EventId::try_from(&*format!( - "${}", - ruma::signatures::reference_hash(&value) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - value - .as_object_mut() - .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? - .insert("event_id".to_owned(), event_id.to_string().into()); - - dbg!(&value); + .map(add_event_id); + let mut event_map = room_state + .chain(auth_chain) + .map(|r| { + let (event_id, value) = r?; serde_json::from_value::(value) - .map(|ev| (dbg!(&ev).event_id().clone(), Arc::new(ev))) + .map(|ev| (event_id, Arc::new(ev))) .map_err(|e| { warn!("{}", e); Error::BadServerResponse("Invalid PDU bytes in send_join response.") }) }) - .collect::>, _>>()?; + .collect::>>>()?; let control_events = event_map .values() @@ -575,6 +591,8 @@ async fn join_room_by_id_helper( ) .expect("iterative auth check failed on resolved events"); + let mut state = HashMap::new(); + // filter the events that failed the auth check keeping the remaining events // sorted correctly for ev_id in sorted_event_ids @@ -587,9 +605,22 @@ async fn join_room_by_id_helper( .expect("Found event_id in sorted events that is not in resolved state"); // We do not rebuild the PDU in this case only insert to DB - db.rooms - .append_pdu(PduEvent::from(&**pdu), &db.globals, &db.account_data)?; + let pdu_id = + db.rooms + .append_pdu(&PduEvent::from(&**pdu), &db.globals, &db.account_data)?; + + if state_events.contains(ev_id) { + state.insert( + ( + pdu.kind(), + pdu.state_key().expect("State events have a state key"), + ), + pdu_id, + ); + } } + + db.rooms.force_state(room_id, state)?; } else { let event = member::MemberEventContent { membership: member::MembershipState::Join, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 87f4dcd..b538c85 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -220,6 +220,31 @@ impl Rooms { .is_some()) } + /// Returns the full room state. + pub fn force_state( + &self, + room_id: &RoomId, + state: HashMap<(EventType, String), Vec>, + ) -> Result<()> { + let state_hash = + self.calculate_hash(&state.values().map(|pdu_id| &**pdu_id).collect::>())?; + let mut prefix = state_hash.clone(); + prefix.push(0xff); + + for ((event_type, state_key), pdu_id) in state { + let mut state_id = prefix.clone(); + state_id.extend_from_slice(&event_type.as_str().as_bytes()); + state_id.push(0xff); + state_id.extend_from_slice(&state_key.as_bytes()); + self.stateid_pduid.insert(state_id, pdu_id)?; + } + + self.roomid_statehash + .insert(room_id.as_bytes(), &*state_hash)?; + + Ok(()) + } + /// Returns the full room state. pub fn room_state_full( &self, @@ -446,10 +471,10 @@ impl Rooms { /// Creates a new persisted data unit and adds it to a room. pub fn append_pdu( &self, - pdu: PduEvent, + pdu: &PduEvent, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, - ) -> Result { + ) -> Result> { let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); ruma::signatures::hash_and_sign_event( globals.server_name().as_str(), @@ -473,10 +498,6 @@ impl Rooms { self.eventid_pduid .insert(pdu.event_id.as_bytes(), &*pdu_id)?; - if pdu.state_key.is_some() { - self.append_to_state(&pdu_id, &pdu)?; - } - match pdu.kind { EventType::RoomRedaction => { if let Some(redact_id) = &pdu.redacts { @@ -484,23 +505,22 @@ impl Rooms { } } EventType::RoomMember => { - if let Some(state_key) = pdu.state_key { + if let Some(state_key) = &pdu.state_key { // if the state_key fails - let target_user_id = UserId::try_from(state_key) + let target_user_id = UserId::try_from(state_key.clone()) .expect("This state_key was previously validated"); // Update our membership info, we do this here incase a user is invited // and immediately leaves we need the DB to record the invite event for auth self.update_membership( &pdu.room_id, &target_user_id, - serde_json::from_value::(pdu.content).map_err( - |_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid redaction event content.", - ) - }, - )?, + serde_json::from_value::(pdu.content.clone()) + .map_err(|_| { + Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid redaction event content.", + ) + })?, &pdu.sender, account_data, globals, @@ -528,7 +548,7 @@ impl Rooms { self.edus .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; - Ok(pdu.event_id) + Ok(pdu_id) } /// Generates a new StateHash and associates it with the incoming event. @@ -789,7 +809,13 @@ impl Rooms { )) .expect("ruma's reference hashes are valid event ids"); - self.append_pdu(pdu, globals, account_data) + let pdu_id = self.append_pdu(&pdu, globals, account_data)?; + + if pdu.state_key.is_some() { + self.append_to_state(&pdu_id, &pdu)?; + } + + Ok(pdu.event_id) } /// Returns an iterator over all PDUs in a room. @@ -953,19 +979,17 @@ impl Rooms { self.roomuseroncejoinedids.insert(&userroom_id, &[])?; // Check if the room has a predecessor - if let Some(predecessor) = serde_json::from_value::< - Raw, - >( - self.room_state_get(&room_id, &EventType::RoomCreate, "")? - .ok_or_else(|| { - Error::bad_database("Found room without m.room.create event.") - })? - .content, - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid room event in database."))? - .predecessor + if let Some(predecessor) = self + .room_state_get(&room_id, &EventType::RoomCreate, "")? + .and_then(|create| { + serde_json::from_value::< + Raw, + >(create.content) + .expect("Raw::from_value always works") + .deserialize() + .ok() + }) + .and_then(|content| content.predecessor) { // Copy user settings from predecessor to the current room: // - Push rules diff --git a/src/server_server.rs b/src/server_server.rs index d7d0e23..6634d5a 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,4 +1,4 @@ -use crate::{client_server, ConduitResult, Database, Error, Result, Ruma}; +use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ @@ -270,9 +270,11 @@ pub fn send_transaction_message_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - dbg!(&*body); + //dbg!(&*body); for pdu in &body.pdus { - let mut value = serde_json::to_value(pdu).expect("all ruma pdus are json values"); + let mut value = serde_json::from_str(pdu.json().get()) + .expect("converting raw jsons to values always works"); + let event_id = EventId::try_from(&*format!( "${}", ruma::signatures::reference_hash(&value).expect("ruma can calculate reference hashes") @@ -284,11 +286,11 @@ pub fn send_transaction_message_route<'a>( .expect("ruma pdus are json objects") .insert("event_id".to_owned(), event_id.to_string().into()); - db.rooms.append_pdu( - serde_json::from_value(value).expect("all ruma pdus are conduit pdus"), - &db.globals, - &db.account_data, - )?; + let pdu = + serde_json::from_value::(value).expect("all ruma pdus are conduit pdus"); + if db.rooms.exists(&pdu.room_id)? { + db.rooms.append_pdu(&pdu, &db.globals, &db.account_data)?; + } } Ok(send_transaction_message::v1::Response { pdus: BTreeMap::new(), From c5313b3e8f8d024c540d0428a6c75828d82f95c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 11:00:31 +0200 Subject: [PATCH 23/48] improvement: try out multiple servers when joining remote rooms --- Cargo.lock | 35 +++++++++++----------- src/client_server/alias.rs | 4 +-- src/client_server/directory.rs | 12 ++++---- src/client_server/membership.rs | 52 +++++++++++++++++++++------------ src/server_server.rs | 10 +++---- 5 files changed, 65 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ffd347..28a4395 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,7 +157,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.4.1", + "miniz_oxide 0.4.2", "object", "rustc-demangle", ] @@ -1054,11 +1054,12 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722" +checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" dependencies = [ "adler", + "autocfg", ] [[package]] @@ -1634,7 +1635,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1650,7 +1651,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "http", "percent-encoding", @@ -1665,7 +1666,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1676,7 +1677,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "ruma-api", "ruma-common", @@ -1689,7 +1690,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "assign", "http", @@ -1708,7 +1709,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "js_int", "ruma-api", @@ -1722,7 +1723,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "js_int", "ruma-common", @@ -1737,7 +1738,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1748,7 +1749,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "js_int", "ruma-api", @@ -1763,7 +1764,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1775,7 +1776,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "proc-macro2", "quote", @@ -1786,7 +1787,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "serde", "strum", @@ -1795,7 +1796,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "form_urlencoded", "itoa", @@ -1807,7 +1808,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "base64", "ring", diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index bfdaeca..0ec43f5 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -63,7 +63,7 @@ pub async fn get_alias_helper( if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( &db, - room_alias.server_name().to_string(), + room_alias.server_name(), federation::query::get_room_information::v1::Request { room_alias }, ) .await?; @@ -79,5 +79,5 @@ pub async fn get_alias_helper( "Room with alias not found.", ))?; - Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_string()]).into()) + Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_owned()]).into()) } diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 2764d2c..a68d8dd 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -19,7 +19,7 @@ use ruma::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, }, - Raw, + Raw, ServerName, }; #[cfg(feature = "conduit_bin")] @@ -65,9 +65,9 @@ pub async fn get_public_rooms_route( ) -> ConduitResult { let response = get_public_rooms_filtered_helper( &db, - body.body.server.as_deref(), - body.body.limit, - body.body.since.as_deref(), + body.server.as_deref(), + body.limit, + body.since.as_deref(), None, // This is not used None, // This is not used ) @@ -119,7 +119,7 @@ pub async fn get_room_visibility_route( pub async fn get_public_rooms_filtered_helper( db: &Database, - server: Option<&str>, + server: Option<&ServerName>, limit: Option, since: Option<&str>, _filter: Option, @@ -131,7 +131,7 @@ pub async fn get_public_rooms_filtered_helper( { let response = server_server::send_request( &db, - other_server.to_owned(), + other_server, federation::directory::get_public_rooms::v1::Request { limit, since: since.as_deref(), diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 9285648..8d19402 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -19,7 +19,7 @@ use ruma::{ }, events::pdu::Pdu, events::{room::member, EventType}, - EventId, Raw, RoomId, RoomVersionId, UserId, + EventId, Raw, RoomId, RoomVersionId, ServerName, UserId, }; use state_res::StateEvent; use std::{ @@ -41,6 +41,7 @@ pub async fn join_room_by_id_route( &db, body.sender_id.as_ref(), &body.room_id, + &[body.room_id.server_name().to_owned()], body.third_party_signed.as_ref(), ) .await @@ -54,13 +55,12 @@ pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { - Ok(room_id) => room_id, + let (servers, room_id) = match RoomId::try_from(body.room_id_or_alias.clone()) { + Ok(room_id) => (vec![room_id.server_name().to_owned()], room_id), Err(room_alias) => { - client_server::get_alias_helper(&db, &room_alias) - .await? - .0 - .room_id + let response = client_server::get_alias_helper(&db, &room_alias).await?; + + (response.0.servers, response.0.room_id) } }; @@ -69,6 +69,7 @@ pub async fn join_room_by_id_or_alias_route( &db, body.sender_id.as_ref(), &room_id, + &servers, body.third_party_signed.as_ref(), ) .await? @@ -415,22 +416,37 @@ async fn join_room_by_id_helper( db: &Database, sender_id: Option<&UserId>, room_id: &RoomId, + servers: &[Box], _third_party_signed: Option<&IncomingThirdPartySigned>, ) -> ConduitResult { let sender_id = sender_id.expect("user is authenticated"); // Ask a remote server if we don't have this room if !db.rooms.exists(&room_id)? && room_id.server_name() != db.globals.server_name() { - let make_join_response = server_server::send_request( - &db, - room_id.server_name().to_string(), - federation::membership::create_join_event_template::v1::Request { - room_id, - user_id: sender_id, - ver: &[RoomVersionId::Version5, RoomVersionId::Version6], - }, - ) - .await?; + let mut make_join_response_and_server = Err(Error::BadServerResponse( + "No server available to assist in joining.", + )); + + for remote_server in servers { + let make_join_response = server_server::send_request( + &db, + remote_server, + federation::membership::create_join_event_template::v1::Request { + room_id, + user_id: sender_id, + ver: &[RoomVersionId::Version5, RoomVersionId::Version6], + }, + ) + .await; + + make_join_response_and_server = make_join_response.map(|r| (r, remote_server)); + + if make_join_response_and_server.is_ok() { + break; + } + } + + let (make_join_response, remote_server) = make_join_response_and_server?; let mut join_event_stub_value = serde_json::from_str::(make_join_response.event.json().get()) @@ -475,7 +491,7 @@ async fn join_room_by_id_helper( let send_join_response = server_server::send_request( &db, - room_id.server_name().to_string(), + remote_server, federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, diff --git a/src/server_server.rs b/src/server_server.rs index 6634d5a..fc1da00 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -13,7 +13,7 @@ use ruma::{ }, OutgoingRequest, }, - EventId, + EventId, ServerName, }; use serde_json::json; use std::{ @@ -44,16 +44,16 @@ pub async fn request_well_known(db: &crate::Database, destination: &str) -> Opti pub async fn send_request( db: &crate::Database, - destination: String, + destination: &ServerName, request: T, ) -> Result where T: Debug, { let actual_destination = "https://".to_owned() - + &request_well_known(db, &destination) + + &request_well_known(db, &destination.as_str()) .await - .unwrap_or(destination.clone() + ":8448"); + .unwrap_or(destination.as_str().to_owned() + ":8448"); let mut http_request = request .try_into_http_request(&actual_destination, Some("")) @@ -82,7 +82,7 @@ where "origin".to_owned(), db.globals.server_name().as_str().into(), ); - request_map.insert("destination".to_owned(), destination.into()); + request_map.insert("destination".to_owned(), destination.as_str().into()); let mut request_json = request_map.into(); ruma::signatures::sign_json( From 4e44fedbcd823876862315ac3590cc3d21a2825e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 11:42:16 +0200 Subject: [PATCH 24/48] fix: room list over federation --- Cargo.lock | 28 +++++------ src/client_server/directory.rs | 32 +++++-------- src/server_server.rs | 88 +++++++++++++++++++++++----------- 3 files changed, 84 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28a4395..865540c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1635,7 +1635,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1651,7 +1651,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "http", "percent-encoding", @@ -1666,7 +1666,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1677,7 +1677,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "ruma-api", "ruma-common", @@ -1690,7 +1690,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "assign", "http", @@ -1709,7 +1709,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "js_int", "ruma-api", @@ -1723,7 +1723,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "js_int", "ruma-common", @@ -1738,7 +1738,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1749,7 +1749,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "js_int", "ruma-api", @@ -1764,7 +1764,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1776,7 +1776,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "proc-macro2", "quote", @@ -1787,7 +1787,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "serde", "strum", @@ -1796,7 +1796,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "form_urlencoded", "itoa", @@ -1808,7 +1808,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "base64", "ring", diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index a68d8dd..f30825d 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,6 +14,7 @@ use ruma::{ }, federation, }, + directory::RoomNetwork, directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, @@ -33,24 +34,13 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - let Ruma { - body: - get_public_rooms_filtered::IncomingRequest { - limit, - server, - since, - filter, - room_network, - }, - .. - } = body; get_public_rooms_filtered_helper( &db, - server.as_deref(), - limit, - since.as_deref(), - filter, // This is not used yet - Some(room_network), // This is not used + body.server.as_deref(), + body.limit, + body.since.as_deref(), + &body.filter, + &body.room_network, ) .await } @@ -68,8 +58,8 @@ pub async fn get_public_rooms_route( body.server.as_deref(), body.limit, body.since.as_deref(), - None, // This is not used - None, // This is not used + &IncomingFilter::default(), + &IncomingRoomNetwork::Matrix, ) .await? .0; @@ -122,8 +112,8 @@ pub async fn get_public_rooms_filtered_helper( server: Option<&ServerName>, limit: Option, since: Option<&str>, - _filter: Option, - _network: Option, + _filter: &IncomingFilter, + _network: &IncomingRoomNetwork, ) -> ConduitResult { if let Some(other_server) = server .clone() @@ -135,7 +125,7 @@ pub async fn get_public_rooms_filtered_helper( federation::directory::get_public_rooms::v1::Request { limit, since: since.as_deref(), - room_network: ruma::directory::RoomNetwork::Matrix, + room_network: RoomNetwork::Matrix, }, ) .await?; diff --git a/src/server_server.rs b/src/server_server.rs index fc1da00..6c53aed 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -2,8 +2,8 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Rum use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ + api::federation::directory::get_public_rooms_filtered, api::{ - client, federation::{ directory::get_public_rooms, discovery::{ @@ -13,6 +13,7 @@ use ruma::{ }, OutgoingRequest, }, + directory::{IncomingFilter, IncomingRoomNetwork}, EventId, ServerName, }; use serde_json::json; @@ -209,38 +210,24 @@ pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json { feature = "conduit_bin", post("/_matrix/federation/v1/publicRooms", data = "") )] -pub async fn get_public_rooms_route( +pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, - body: Ruma>, -) -> ConduitResult { - let Ruma { - body: - get_public_rooms::v1::IncomingRequest { - room_network: _room_network, // TODO - limit, - since, - }, - .. - } = body; - - let client::r0::directory::get_public_rooms_filtered::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } = client_server::get_public_rooms_filtered_helper( + body: Ruma>, +) -> ConduitResult { + let response = client_server::get_public_rooms_filtered_helper( &db, None, - limit, - since.as_deref(), - None, - Some(ruma::directory::IncomingRoomNetwork::Matrix), + body.limit, + body.since.as_deref(), + &body.filter, + &body.room_network, ) .await? .0; - Ok(get_public_rooms::v1::Response { - chunk: chunk + Ok(get_public_rooms_filtered::v1::Response { + chunk: response + .chunk .into_iter() .map(|c| { // Convert ruma::api::federation::directory::get_public_rooms::v1::PublicRoomsChunk @@ -255,9 +242,52 @@ pub async fn get_public_rooms_route( }) .filter_map(|r| r.ok()) .collect(), - prev_batch, - next_batch, - total_room_count_estimate, + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, + } + .into()) +} + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/federation/v1/publicRooms", data = "") +)] +pub async fn get_public_rooms_route( + db: State<'_, Database>, + body: Ruma>, +) -> ConduitResult { + let response = client_server::get_public_rooms_filtered_helper( + &db, + None, + body.limit, + body.since.as_deref(), + &IncomingFilter::default(), + &IncomingRoomNetwork::Matrix, + ) + .await? + .0; + + Ok(get_public_rooms::v1::Response { + chunk: response + .chunk + .into_iter() + .map(|c| { + // Convert ruma::api::federation::directory::get_public_rooms::v1::PublicRoomsChunk + // to ruma::api::client::r0::directory::PublicRoomsChunk + Ok::<_, Error>( + serde_json::from_str( + &serde_json::to_string(&c) + .expect("PublicRoomsChunk::to_string always works"), + ) + .expect("federation and client-server PublicRoomsChunk are the same type"), + ) + }) + .filter_map(|r| r.ok()) + .collect(), + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, } .into()) } From aa5e9e607ecc739d0d991ea7221dadd0125f6d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 14:20:38 +0200 Subject: [PATCH 25/48] feat: download media and thumbnails over federation --- src/client_server/media.rs | 69 +++++++++++++++++++++++++++++++++++--- src/database/media.rs | 28 ++++++++++++++-- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index d077447..8f33743 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -1,5 +1,7 @@ use super::State; -use crate::{database::media::FileMeta, utils, ConduitResult, Database, Error, Ruma}; +use crate::{ + database::media::FileMeta, server_server, utils, ConduitResult, Database, Error, Ruma, +}; use ruma::api::client::{ error::ErrorKind, r0::media::{create_content, get_content, get_content_thumbnail, get_media_config}, @@ -35,7 +37,7 @@ pub fn create_content_route( utils::random_string(MXC_LENGTH) ); db.media - .create(mxc.clone(), &body.filename, &body.content_type, &body.file)?; + .create(mxc.clone(), &body.filename.as_deref(), &body.content_type, &body.file)?; Ok(create_content::Response { content_uri: mxc }.into()) } @@ -47,19 +49,25 @@ pub fn create_content_route( data = "" ) )] -pub fn get_content_route( +pub async fn get_content_route( db: State<'_, Database>, body: Ruma>, _server_name: String, _media_id: String, ) -> ConduitResult { + let mxc = format!( + "mxc://{}/{}", + db.globals.server_name(), + utils::random_string(MXC_LENGTH) + ); + if let Some(FileMeta { filename, content_type, file, }) = db .media - .get(format!("mxc://{}/{}", body.server_name, body.media_id))? + .get(&mxc)? { Ok(get_content::Response { file, @@ -67,6 +75,26 @@ pub fn get_content_route( content_disposition: filename.unwrap_or_default(), // TODO: Spec says this should be optional } .into()) + } else if body.allow_remote { + let get_content_response = server_server::send_request( + &db, + body.server_name.as_ref(), + get_content::Request { + allow_remote: false, + server_name: &body.server_name, + media_id: &body.media_id, + }, + ) + .await?; + + db.media.create( + mxc, + &Some(&get_content_response.content_disposition), + &get_content_response.content_type, + &get_content_response.file, + )?; + + Ok(get_content_response.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } @@ -79,7 +107,7 @@ pub fn get_content_route( data = "" ) )] -pub fn get_content_thumbnail_route( +pub async fn get_content_thumbnail_route( db: State<'_, Database>, body: Ruma>, _server_name: String, @@ -97,6 +125,37 @@ pub fn get_content_thumbnail_route( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, )? { Ok(get_content_thumbnail::Response { file, content_type }.into()) + } else if body.allow_remote { + let get_thumbnail_response = server_server::send_request( + &db, + body.server_name.as_ref(), + get_content_thumbnail::Request { + allow_remote: false, + height: body.height, + width: body.width, + method: body.method, + server_name: &body.server_name, + media_id: &body.media_id, + }, + ) + .await?; + + let mxc = format!( + "mxc://{}/{}", + db.globals.server_name(), + utils::random_string(MXC_LENGTH) + ); + + db.media.upload_thumbnail( + mxc, + &None, + &get_thumbnail_response.content_type, + body.width.try_into().expect("all UInts are valid u32s"), + body.height.try_into().expect("all UInts are valid u32s"), + &get_thumbnail_response.file, + )?; + + Ok(get_thumbnail_response.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } diff --git a/src/database/media.rs b/src/database/media.rs index 63fa11c..869d5d8 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -16,7 +16,7 @@ impl Media { pub fn create( &self, mxc: String, - filename: &Option, + filename: &Option<&str>, content_type: &str, file: &[u8], ) -> Result<()> { @@ -34,8 +34,32 @@ impl Media { Ok(()) } + /// Uploads or replaces a file thumbnail. + pub fn upload_thumbnail( + &self, + mxc: String, + filename: &Option, + content_type: &str, + width: u32, + height: u32, + file: &[u8], + ) -> Result<()> { + let mut key = mxc.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(&width.to_be_bytes()); + key.extend_from_slice(&height.to_be_bytes()); + key.push(0xff); + key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default()); + key.push(0xff); + key.extend_from_slice(content_type.as_bytes()); + + self.mediaid_file.insert(key, file)?; + + Ok(()) + } + /// Downloads a file. - pub fn get(&self, mxc: String) -> Result> { + pub fn get(&self, mxc: &str) -> Result> { let mut prefix = mxc.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail From d1099e9224f3b47d9de9135ab751edd9152dc3b0 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 25 Aug 2020 11:49:51 +0200 Subject: [PATCH 26/48] Update dependencies --- Cargo.lock | 274 +------------------------------------ Cargo.toml | 57 +++++--- src/client_server/media.rs | 22 +-- 3 files changed, 56 insertions(+), 297 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 865540c..bde0b0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,80 +1,11 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "addr2line" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" - [[package]] name = "adler32" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" -[[package]] -name = "aead" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" -dependencies = [ - "generic-array", -] - -[[package]] -name = "aes" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7001367fde4c768a19d1029f0a8be5abd9308e1119846d5bd9ad26297b8faf5" -dependencies = [ - "aes-soft", - "aesni", - "block-cipher", -] - -[[package]] -name = "aes-gcm" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f5007801316299f922a6198d1d09a0bae95786815d066d5880d13f7c45ead1" -dependencies = [ - "aead", - "aes", - "block-cipher", - "ghash", - "subtle", -] - -[[package]] -name = "aes-soft" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4925647ee64e5056cf231608957ce7c81e12d6d6e316b9ce1404778cc1d35fa7" -dependencies = [ - "block-cipher", - "byteorder", - "opaque-debug 0.2.3", -] - -[[package]] -name = "aesni" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050d39b0b7688b3a3254394c3e30a9d66c41dcf9b05b0e2dbdc623f6505d264" -dependencies = [ - "block-cipher", - "opaque-debug 0.2.3", -] - [[package]] name = "ansi_term" version = "0.12.1" @@ -90,12 +21,6 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" -[[package]] -name = "array-init" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30bbe2f5e3d117f55bd8c7a1f9191e4a5deba9f15f595bbea4f670c59c765db" - [[package]] name = "arrayref" version = "0.3.6" @@ -148,20 +73,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" -[[package]] -name = "backtrace" -version = "0.3.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide 0.4.2", - "object", - "rustc-demangle", -] - [[package]] name = "base-x" version = "0.2.6" @@ -197,24 +108,6 @@ dependencies = [ "constant_time_eq", ] -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - -[[package]] -name = "block-cipher" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa136449e765dc7faa244561ccae839c394048667929af599b5d931ebe7b7f10" -dependencies = [ - "generic-array", -] - [[package]] name = "bumpalo" version = "3.4.0" @@ -319,12 +212,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ - "aes-gcm", - "base64", - "hkdf", "percent-encoding", - "rand", - "sha2", "time 0.2.19", "version_check", ] @@ -345,12 +233,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" -[[package]] -name = "cpuid-bool" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" - [[package]] name = "crc32fast" version = "1.2.0" @@ -386,16 +268,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "crypto-mac" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "deflate" version = "0.8.6" @@ -436,22 +308,12 @@ dependencies = [ "syn", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "directories" -version = "2.0.2" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "551a778172a450d7fc12e629ca3b0428d00f6afa9a43da1b630d54604e97371c" +checksum = "f8fed639d60b58d0f53498ab13d26f621fd77569cc6edb031f4cc36a2ad9da0f" dependencies = [ - "cfg-if", "dirs-sys", ] @@ -654,16 +516,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "generic-array" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.1.15" @@ -675,15 +527,6 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1", ] -[[package]] -name = "ghash" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6e27f0689a6e15944bdce7e45425efb87eaa8ab0c6e87f11d0987a9133e2531" -dependencies = [ - "polyval", -] - [[package]] name = "gif" version = "0.10.3" @@ -694,12 +537,6 @@ dependencies = [ "lzw", ] -[[package]] -name = "gimli" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" - [[package]] name = "glob" version = "0.3.0" @@ -749,26 +586,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hkdf" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe1149865383e4526a43aee8495f9a325f0b806c63ce6427d06336a590abbbc9" -dependencies = [ - "digest", - "hmac", -] - -[[package]] -name = "hmac" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" -dependencies = [ - "crypto-mac", - "digest", -] - [[package]] name = "http" version = "0.2.1" @@ -1052,16 +869,6 @@ dependencies = [ "adler32", ] -[[package]] -name = "miniz_oxide" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" -dependencies = [ - "adler", - "autocfg", -] - [[package]] name = "mio" version = "0.6.22" @@ -1184,30 +991,12 @@ dependencies = [ "libc", ] -[[package]] -name = "object" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" - [[package]] name = "once_cell" version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" version = "0.10.30" @@ -1341,17 +1130,7 @@ dependencies = [ "bitflags", "crc32fast", "deflate", - "miniz_oxide 0.3.7", -] - -[[package]] -name = "polyval" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a50142b55ab3ed0e9f68dfb3709f1d90d29da24e91033f28b96330643107dc" -dependencies = [ - "cfg-if", - "universal-hash", + "miniz_oxide", ] [[package]] @@ -1828,12 +1607,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rustc-demangle" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" - [[package]] name = "rustc_version" version = "0.2.3" @@ -1975,19 +1748,6 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" -[[package]] -name = "sha2" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1" -dependencies = [ - "block-buffer", - "cfg-if", - "cpuid-bool", - "digest", - "opaque-debug 0.3.0", -] - [[package]] name = "sharded-slab" version = "0.0.9" @@ -2015,12 +1775,10 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "sled" -version = "0.32.1" +version = "0.34.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3dbbb8ee10611bd1d020767c27599ccbbf8365f7e0ed7e54429cc8b9433ad8" +checksum = "f72c064e63fbca3138ad07f3588c58093f1684f3a99f60dcfa6d46b87e60fde7" dependencies = [ - "array-init", - "backtrace", "crc32fast", "crossbeam-epoch", "crossbeam-utils", @@ -2156,12 +1914,6 @@ dependencies = [ "syn", ] -[[package]] -name = "subtle" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" - [[package]] name = "syn" version = "1.0.40" @@ -2436,12 +2188,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" -[[package]] -name = "typenum" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" - [[package]] name = "unicase" version = "2.6.0" @@ -2481,16 +2227,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" -[[package]] -name = "universal-hash" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "untrusted" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index 1b7a700..60296a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,28 +12,49 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +# Used to handle requests # TODO: This can become optional as soon as proper configs are supported -#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests -rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } -#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" } # Used for matrix spec type definitions and helpers +#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", default-features = false, features = ["tls"] } # Used to handle requests +rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] } + +# Used for matrix spec type definitions and helpers +#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } +ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" } #ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } -tokio = "0.2.22" # Used for long polling -sled = "0.32.0" # Used for storing data permanently -log = "0.4.8" # Used for emitting log entries -http = "0.2.1" # Used for rocket<->ruma conversions -directories = "2.0.2" # Used to find data directory for default db path -js_int = "0.1.5" # Used for number types for ruma -serde_json = { version = "1.0.53", features = ["raw_value"] } # Used for ruma wrapper -serde = "1.0.111" # Used for pdu definition -rand = "0.7.3" # Used for secure identifiers -rust-argon2 = "0.8.2" # Used to hash passwords -reqwest = "0.10.6" # Used to send requests -thiserror = "1.0.19" # Used for conduit::Error type -image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images -base64 = "0.12.3" # Used to encode server public key + +# Used when doing state resolution state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } #state-res = { path = "../state-res", features = ["unstable-pre-spec"] } + +# Used for long polling +tokio = "0.2.22" +# Used for storing data permanently +sled = "0.34.4" +# Used for emitting log entries +log = "0.4.11" +# Used for rocket<->ruma conversions +http = "0.2.1" +# Used to find data directory for default db path +directories = "3.0.1" +# Used for number types for ruma +js_int = "0.1.9" +# Used for ruma wrapper +serde_json = { version = "1.0.57", features = ["raw_value"] } +# Used for pdu definition +serde = "1.0.116" +# Used for secure identifiers +rand = "0.7.3" +# Used to hash passwords +rust-argon2 = "0.8.2" +# Used to send requests +reqwest = "0.10.8" +# Used for conduit::Error type +thiserror = "1.0.20" +# Used to generate thumbnails for images +image = { version = "0.23.9", default-features = false, features = ["jpeg", "png", "gif"] } +# Used to encode server public key +base64 = "0.12.3" +# Used when hashing the state ring = "0.16.15" [features] diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 8f33743..f897a67 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -36,8 +36,12 @@ pub fn create_content_route( db.globals.server_name(), utils::random_string(MXC_LENGTH) ); - db.media - .create(mxc.clone(), &body.filename.as_deref(), &body.content_type, &body.file)?; + db.media.create( + mxc.clone(), + &body.filename.as_deref(), + &body.content_type, + &body.file, + )?; Ok(create_content::Response { content_uri: mxc }.into()) } @@ -55,19 +59,17 @@ pub async fn get_content_route( _server_name: String, _media_id: String, ) -> ConduitResult { - let mxc = format!( - "mxc://{}/{}", - db.globals.server_name(), - utils::random_string(MXC_LENGTH) - ); + let mxc = format!( + "mxc://{}/{}", + db.globals.server_name(), + utils::random_string(MXC_LENGTH) + ); if let Some(FileMeta { filename, content_type, file, - }) = db - .media - .get(&mxc)? + }) = db.media.get(&mxc)? { Ok(get_content::Response { file, From 9f05ef926af6f0c7c7d2886b99b87ccd6e218e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 16:23:15 +0200 Subject: [PATCH 27/48] fix: filter public room dir --- src/client_server/directory.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index f30825d..871a780 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,6 +14,7 @@ use ruma::{ }, federation, }, + directory::Filter, directory::RoomNetwork, directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk}, events::{ @@ -112,7 +113,7 @@ pub async fn get_public_rooms_filtered_helper( server: Option<&ServerName>, limit: Option, since: Option<&str>, - _filter: &IncomingFilter, + filter: &IncomingFilter, _network: &IncomingRoomNetwork, ) -> ConduitResult { if let Some(other_server) = server @@ -122,9 +123,12 @@ pub async fn get_public_rooms_filtered_helper( let response = server_server::send_request( &db, other_server, - federation::directory::get_public_rooms::v1::Request { + federation::directory::get_public_rooms_filtered::v1::Request { limit, since: since.as_deref(), + filter: Filter { + generic_search_term: filter.generic_search_term.as_deref(), + }, room_network: RoomNetwork::Matrix, }, ) From f7816b11de0889fca761f55510a3313dcfa78a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 20:23:19 +0200 Subject: [PATCH 28/48] feat: send messages over federation --- Cargo.lock | 32 ++--- src/client_server/account.rs | 4 +- src/client_server/alias.rs | 2 +- src/client_server/directory.rs | 2 +- src/client_server/media.rs | 4 +- src/client_server/membership.rs | 206 +++++++++++++++++--------------- src/client_server/message.rs | 4 +- src/client_server/profile.rs | 8 +- src/client_server/redact.rs | 4 +- src/client_server/room.rs | 37 +++--- src/client_server/state.rs | 12 +- src/database.rs | 1 + src/database/rooms.rs | 177 ++++++++++++++++++--------- src/pdu.rs | 27 ++++- src/server_server.rs | 22 ++-- 15 files changed, 324 insertions(+), 218 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bde0b0d..e0de2a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1414,7 +1414,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1430,7 +1430,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "http", "percent-encoding", @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1456,7 +1456,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "ruma-api", "ruma-common", @@ -1469,7 +1469,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "assign", "http", @@ -1488,7 +1488,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "js_int", "ruma-api", @@ -1502,7 +1502,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "js_int", "ruma-common", @@ -1517,7 +1517,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1528,7 +1528,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "js_int", "ruma-api", @@ -1543,7 +1543,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1555,7 +1555,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "proc-macro2", "quote", @@ -1566,7 +1566,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "serde", "strum", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "form_urlencoded", "itoa", @@ -1587,7 +1587,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "base64", "ring", @@ -1916,9 +1916,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.40" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350" +checksum = "6690e3e9f692504b941dc6c3b188fd28df054f7fb8469ab40680df52fdcc842b" dependencies = [ "proc-macro2", "quote", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 3db933c..2ec9282 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -303,7 +303,7 @@ pub fn whoami_route(body: Ruma) -> ConduitResult, body: Ruma>, ) -> ConduitResult { @@ -366,7 +366,7 @@ pub fn deactivate_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // Remove devices and mark account as deactivated diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 0ec43f5..c5c514e 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -62,7 +62,7 @@ pub async fn get_alias_helper( ) -> ConduitResult { if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( - &db, + &db.globals, room_alias.server_name(), federation::query::get_room_information::v1::Request { room_alias }, ) diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 871a780..372ce98 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -121,7 +121,7 @@ pub async fn get_public_rooms_filtered_helper( .filter(|server| *server != db.globals.server_name().as_str()) { let response = server_server::send_request( - &db, + &db.globals, other_server, federation::directory::get_public_rooms_filtered::v1::Request { limit, diff --git a/src/client_server/media.rs b/src/client_server/media.rs index f897a67..8f7a9b9 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -79,7 +79,7 @@ pub async fn get_content_route( .into()) } else if body.allow_remote { let get_content_response = server_server::send_request( - &db, + &db.globals, body.server_name.as_ref(), get_content::Request { allow_remote: false, @@ -129,7 +129,7 @@ pub async fn get_content_thumbnail_route( Ok(get_content_thumbnail::Response { file, content_type }.into()) } else if body.allow_remote { let get_thumbnail_response = server_server::send_request( - &db, + &db.globals, body.server_name.as_ref(), get_content_thumbnail::Request { allow_remote: false, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 8d19402..18fb5a9 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -83,7 +83,7 @@ pub async fn join_room_by_id_or_alias_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/leave", data = "") )] -pub fn leave_room_route( +pub async fn leave_room_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -108,19 +108,21 @@ pub fn leave_room_route( event.membership = member::MembershipState::Leave; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(leave_room::Response::new().into()) } @@ -129,33 +131,35 @@ pub fn leave_room_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/invite", data = "") )] -pub fn invite_user_route( +pub async fn invite_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user_id)?, - avatar_url: db.users.avatar_url(&user_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user_id)?, + avatar_url: db.users.avatar_url(&user_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(invite_user::Response.into()) } else { @@ -167,7 +171,7 @@ pub fn invite_user_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/kick", data = "") )] -pub fn kick_user_route( +pub async fn kick_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -193,19 +197,21 @@ pub fn kick_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; // TODO: reason - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(kick_user::Response::new().into()) } @@ -214,7 +220,7 @@ pub fn kick_user_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/ban", data = "") )] -pub fn ban_user_route( +pub async fn ban_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -248,19 +254,21 @@ pub fn ban_user_route( }, )?; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(ban_user::Response::new().into()) } @@ -269,7 +277,7 @@ pub fn ban_user_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/unban", data = "") )] -pub fn unban_user_route( +pub async fn unban_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -294,19 +302,21 @@ pub fn unban_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(unban_user::Response::new().into()) } @@ -429,7 +439,7 @@ async fn join_room_by_id_helper( for remote_server in servers { let make_join_response = server_server::send_request( - &db, + &db.globals, remote_server, federation::membership::create_join_event_template::v1::Request { room_id, @@ -490,7 +500,7 @@ async fn join_room_by_id_helper( .expect("event is valid, we just created it"); let send_join_response = server_server::send_request( - &db, + &db.globals, remote_server, federation::membership::create_join_event::v2::Request { room_id, @@ -621,9 +631,12 @@ async fn join_room_by_id_helper( .expect("Found event_id in sorted events that is not in resolved state"); // We do not rebuild the PDU in this case only insert to DB - let pdu_id = - db.rooms - .append_pdu(&PduEvent::from(&**pdu), &db.globals, &db.account_data)?; + let pdu_id = db.rooms.append_pdu( + &PduEvent::from(&**pdu), + &serde_json::to_value(&**pdu).expect("PDU is valid value"), + &db.globals, + &db.account_data, + )?; if state_events.contains(ev_id) { state.insert( @@ -646,19 +659,22 @@ async fn join_room_by_id_helper( third_party_invite: None, }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.account_data, + ) + .await?; } Ok(join_room_by_id::Response::new(room_id.clone()).into()) diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 8a09aba..4ba0d9f 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -17,7 +17,7 @@ use rocket::{get, put}; feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/send/<_>/<_>", data = "") )] -pub fn send_message_event_route( +pub async fn send_message_event_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -67,7 +67,7 @@ pub fn send_message_event_route( &body.room_id, &db.globals, &db.account_data, - )?; + ).await?; db.transaction_ids .add_txnid(sender_id, device_id, &body.txn_id, event_id.as_bytes())?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index c1c0253..be893e1 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -19,7 +19,7 @@ use std::convert::TryInto; feature = "conduit_bin", put("/_matrix/client/r0/profile/<_>/displayname", data = "") )] -pub fn set_displayname_route( +pub async fn set_displayname_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -64,7 +64,7 @@ pub fn set_displayname_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // Presence update db.rooms.edus.update_presence( @@ -110,7 +110,7 @@ pub fn get_displayname_route( feature = "conduit_bin", put("/_matrix/client/r0/profile/<_>/avatar_url", data = "") )] -pub fn set_avatar_url_route( +pub async fn set_avatar_url_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -167,7 +167,7 @@ pub fn set_avatar_url_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // Presence update db.rooms.edus.update_presence( diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 5117348..701fc00 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -12,7 +12,7 @@ use rocket::put; feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/redact/<_>/<_>", data = "") )] -pub fn redact_event_route( +pub async fn redact_event_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -33,7 +33,7 @@ pub fn redact_event_route( &body.room_id, &db.globals, &db.account_data, - )?; + ).await?; Ok(redact_event::Response { event_id }.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index a5280cf..0e5c571 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -20,7 +20,7 @@ use rocket::{get, post}; feature = "conduit_bin", post("/_matrix/client/r0/createRoom", data = "") )] -pub fn create_room_route( +pub async fn create_room_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -65,7 +65,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 2. Let the room creator join db.rooms.build_and_append_pdu( @@ -87,7 +87,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 3. Power levels let mut users = BTreeMap::new(); @@ -129,7 +129,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 4. Events set by preset @@ -162,7 +162,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 4.2 History Visibility db.rooms.build_and_append_pdu( @@ -180,7 +180,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 4.3 Guest Access db.rooms.build_and_append_pdu( @@ -206,7 +206,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 5. Events listed in initial_state for event in &body.initial_state { @@ -226,7 +226,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // 6. Events implied by name and topic @@ -248,7 +248,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } if let Some(topic) = &body.topic { @@ -267,7 +267,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // 7. Events implied by invite (and TODO: invite_3pid) @@ -291,7 +291,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // Homeserver specific stuff @@ -337,7 +337,7 @@ pub fn get_room_event_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_room_id>/upgrade", data = "") )] -pub fn upgrade_room_route( +pub async fn upgrade_room_route( db: State<'_, Database>, body: Ruma>, _room_id: String, @@ -379,7 +379,7 @@ pub fn upgrade_room_route( &body.room_id, &db.globals, &db.account_data, - )?; + ).await?; // Get the old room federations status let federate = serde_json::from_value::>( @@ -419,7 +419,7 @@ pub fn upgrade_room_route( &replacement_room, &db.globals, &db.account_data, - )?; + ).await?; // Join the new room db.rooms.build_and_append_pdu( @@ -441,7 +441,7 @@ pub fn upgrade_room_route( &replacement_room, &db.globals, &db.account_data, - )?; + ).await?; // Recommended transferable state events list from the specs let transferable_state_events = vec![ @@ -475,7 +475,7 @@ pub fn upgrade_room_route( &replacement_room, &db.globals, &db.account_data, - )?; + ).await?; } // Moves any local aliases to the new room @@ -505,7 +505,7 @@ pub fn upgrade_room_route( power_levels_event_content.invite = new_level; // Modify the power levels in the old room to prevent sending of events and inviting new users - db.rooms + let _ = db.rooms .build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, @@ -519,8 +519,7 @@ pub fn upgrade_room_route( &body.room_id, &db.globals, &db.account_data, - ) - .ok(); + ).await; // Return the replacement room id Ok(upgrade_room::Response { replacement_room }.into()) diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 1fe3cd6..e9d20e2 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -19,7 +19,7 @@ use rocket::{get, put}; feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/state/<_>/<_>", data = "") )] -pub fn send_state_event_for_key_route( +pub async fn send_state_event_for_key_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -41,7 +41,7 @@ pub fn send_state_event_for_key_route( content, &body.room_id, Some(body.state_key.to_owned()), - )?) + ).await?) .into(), ) } @@ -50,7 +50,7 @@ pub fn send_state_event_for_key_route( feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/state/<_>", data = "") )] -pub fn send_state_event_for_empty_key_route( +pub async fn send_state_event_for_empty_key_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -80,7 +80,7 @@ pub fn send_state_event_for_empty_key_route( json, &body.room_id, Some("".into()), - )?) + ).await?) .into(), ) } @@ -177,7 +177,7 @@ pub fn get_state_events_for_empty_key_route( .into()) } -pub fn send_state_event_for_key_helper( +pub async fn send_state_event_for_key_helper( db: &Database, sender: &UserId, content: &AnyStateEventContent, @@ -223,7 +223,7 @@ pub fn send_state_event_for_key_helper( &room_id, &db.globals, &db.account_data, - )?; + ).await?; Ok(event_id) } diff --git a/src/database.rs b/src/database.rs index 83f30c9..e1a356c 100644 --- a/src/database.rs +++ b/src/database.rs @@ -109,6 +109,7 @@ impl Database { tokenids: db.open_tree("tokenids")?, + roomserverids: db.open_tree("roomserverids")?, userroomid_joined: db.open_tree("userroomid_joined")?, roomuserid_joined: db.open_tree("roomuserid_joined")?, roomuseroncejoinedids: db.open_tree("roomuseroncejoinedids")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index b538c85..ba54e7f 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -2,11 +2,12 @@ mod edus; pub use edus::RoomEdus; -use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; +use crate::{pdu::PduBuilder, server_server, utils, Error, PduEvent, Result}; use log::error; use ring::digest; use ruma::{ api::client::error::ErrorKind, + api::federation, events::{ ignored_user_list, room::{ @@ -15,7 +16,7 @@ use ruma::{ }, EventType, }, - EventId, Raw, RoomAliasId, RoomId, UserId, + EventId, Raw, RoomAliasId, RoomId, ServerName, UserId, }; use sled::IVec; use state_res::{event_auth, Error as StateError, Requester, StateEvent, StateMap, StateStore}; @@ -25,6 +26,7 @@ use std::{ convert::{TryFrom, TryInto}, mem, sync::Arc, + time::SystemTime, }; /// The unique identifier of each state group. @@ -44,6 +46,8 @@ pub struct Rooms { pub(super) tokenids: sled::Tree, // TokenId = RoomId + Token + PduId + /// Participating servers in a room. + pub(super) roomserverids: sled::Tree, // RoomServerId = RoomId + ServerName pub(super) userroomid_joined: sled::Tree, pub(super) roomuserid_joined: sled::Tree, pub(super) roomuseroncejoinedids: sled::Tree, @@ -169,8 +173,7 @@ impl Rooms { Ok(events) } - // This fetches auth events from the current state using the - /// full `roomstateid_pdu` tree. + /// This fetches auth events from the current state. pub fn get_auth_events( &self, room_id: &RoomId, @@ -472,17 +475,10 @@ impl Rooms { pub fn append_pdu( &self, pdu: &PduEvent, + pdu_json: &serde_json::Value, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, ) -> Result> { - let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); - ruma::signatures::hash_and_sign_event( - globals.server_name().as_str(), - globals.keypair(), - &mut pdu_json, - ) - .expect("event is valid, we just created it"); - self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?; // Increment the last index and use that @@ -610,7 +606,7 @@ impl Rooms { } /// Creates a new persisted data unit and adds it to a room. - pub fn build_and_append_pdu( + pub async fn build_and_append_pdu( &self, pdu_builder: PduBuilder, sender: &UserId, @@ -799,22 +795,59 @@ impl Rooms { signatures: BTreeMap::new(), }; + // Hash and sign + let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + ruma::signatures::hash_and_sign_event( + globals.server_name().as_str(), + globals.keypair(), + &mut pdu_json, + ) + .expect("event is valid, we just created it"); + // Generate event id pdu.event_id = EventId::try_from(&*format!( "${}", - ruma::signatures::reference_hash( - &serde_json::to_value(&pdu).expect("event is valid, we just created it") - ) - .expect("ruma can calculate reference hashes") + ruma::signatures::reference_hash(&pdu_json) + .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); - let pdu_id = self.append_pdu(&pdu, globals, account_data)?; + pdu_json + .as_object_mut() + .expect("json is object") + .insert("event_id".to_owned(), pdu.event_id.to_string().into()); + + let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data)?; if pdu.state_key.is_some() { self.append_to_state(&pdu_id, &pdu)?; } + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + let response = server_server::send_request( + &globals, + "koesters.xyz".try_into().unwrap(), + federation::transactions::send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus: &[serde_json::from_value(pdu_json).expect("Raw::from_value always works")], + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &utils::random_string(16), + }, + ) + .await; + + let _ = dbg!(response); + Ok(pdu.event_id) } @@ -957,12 +990,17 @@ impl Rooms { &self, room_id: &RoomId, user_id: &UserId, - mut member_content: member::MemberEventContent, + member_content: member::MemberEventContent, sender: &UserId, account_data: &super::account_data::AccountData, globals: &super::globals::Globals, ) -> Result<()> { let membership = member_content.membership; + + let mut roomserver_id = room_id.as_bytes().to_vec(); + roomserver_id.push(0xff); + roomserver_id.extend_from_slice(user_id.server_name().as_bytes()); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -1056,6 +1094,7 @@ impl Rooms { } } + self.roomserverids.insert(&roomserver_id, &[])?; self.userroomid_joined.insert(&userroom_id, &[])?; self.roomuserid_joined.insert(&roomuser_id, &[])?; self.userroomid_invited.remove(&userroom_id)?; @@ -1075,25 +1114,10 @@ impl Rooms { }); if is_ignored { - member_content.membership = member::MembershipState::Leave; - - self.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &user_id, - &room_id, - globals, - account_data, - )?; - return Ok(()); } + + self.roomserverids.insert(&roomserver_id, &[])?; self.userroomid_invited.insert(&userroom_id, &[])?; self.roomuserid_invited.insert(&roomuser_id, &[])?; self.userroomid_joined.remove(&userroom_id)?; @@ -1101,6 +1125,14 @@ impl Rooms { self.userroomid_left.remove(&userroom_id)?; } member::MembershipState::Leave | member::MembershipState::Ban => { + if self + .room_members(room_id) + .chain(self.room_members_invited(room_id)) + .filter_map(|r| r.ok()) + .all(|u| u.server_name() != user_id.server_name()) + { + self.roomserverids.remove(&roomserver_id)?; + } self.userroomid_left.insert(&userroom_id, &[])?; self.userroomid_joined.remove(&userroom_id)?; self.roomuserid_joined.remove(&roomuser_id)?; @@ -1294,10 +1326,34 @@ impl Rooms { }) } + /// Returns an iterator over all joined members of a room. + pub fn room_servers(&self, room_id: &RoomId) -> impl Iterator>> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + + self.roomserverids.scan_prefix(prefix).keys().map(|key| { + Ok(Box::::try_from( + utils::string_from_bytes( + &key? + .rsplit(|&b| b == 0xff) + .next() + .expect("rsplit always returns an element"), + ) + .map_err(|_| { + Error::bad_database("Server name in roomserverids is invalid unicode.") + })?, + ) + .map_err(|_| Error::bad_database("Server name in roomserverids is invalid."))?) + }) + } + /// Returns an iterator over all joined members of a room. pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + self.roomuserid_joined - .scan_prefix(room_id.as_bytes()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(UserId::try_from( @@ -1317,8 +1373,11 @@ impl Rooms { /// Returns an iterator over all User IDs who ever joined a room. pub fn room_useroncejoined(&self, room_id: &RoomId) -> impl Iterator> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + self.roomuseroncejoinedids - .scan_prefix(room_id.to_string()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(UserId::try_from( @@ -1338,8 +1397,11 @@ impl Rooms { /// Returns an iterator over all invited members of a room. pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + self.roomuserid_invited - .scan_prefix(room_id.as_bytes()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(UserId::try_from( @@ -1380,8 +1442,11 @@ impl Rooms { /// Returns an iterator over all rooms a user was invited to. pub fn rooms_invited(&self, user_id: &UserId) -> impl Iterator> { + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xff); + self.userroomid_invited - .scan_prefix(&user_id.as_bytes()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1401,23 +1466,23 @@ impl Rooms { /// Returns an iterator over all rooms a user left. pub fn rooms_left(&self, user_id: &UserId) -> impl Iterator> { - self.userroomid_left - .scan_prefix(&user_id.as_bytes()) - .keys() - .map(|key| { - Ok(RoomId::try_from( - utils::string_from_bytes( - &key? - .rsplit(|&b| b == 0xff) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| { - Error::bad_database("Room ID in userroomid_left is invalid unicode.") - })?, + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xff); + + self.userroomid_left.scan_prefix(prefix).keys().map(|key| { + Ok(RoomId::try_from( + utils::string_from_bytes( + &key? + .rsplit(|&b| b == 0xff) + .next() + .expect("rsplit always returns an element"), ) - .map_err(|_| Error::bad_database("Room ID in userroomid_left is invalid."))?) - }) + .map_err(|_| { + Error::bad_database("Room ID in userroomid_left is invalid unicode.") + })?, + ) + .map_err(|_| Error::bad_database("Room ID in userroomid_left is invalid."))?) + }) } pub fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { diff --git a/src/pdu.rs b/src/pdu.rs index 7f842e2..c904230 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -6,7 +6,7 @@ use ruma::{ AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, -}; +events::pdu::PduStub}; use serde::{Deserialize, Serialize}; use serde_json::json; use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH}; @@ -198,6 +198,31 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } + + pub fn to_outgoing_federation_event(&self) -> Raw { + let mut json = json!({ + "room_id": self.room_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "type": self.kind, + "content": self.content, + "prev_events": self.prev_events, + "depth": self.depth, + "auth_events": self.auth_events, + "unsigned": self.unsigned, + "hashes": self.hashes, + "signatures": self.signatures, + }); + + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &self.redacts { + json["redacts"] = json!(redacts); + } + + serde_json::from_value(json).expect("Raw::from_value always works") + } } impl From<&state_res::StateEvent> for PduEvent { diff --git a/src/server_server.rs b/src/server_server.rs index 6c53aed..9f4be13 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -24,9 +24,9 @@ use std::{ time::{Duration, SystemTime}, }; -pub async fn request_well_known(db: &crate::Database, destination: &str) -> Option { +pub async fn request_well_known(globals: &crate::database::globals::Globals, destination: &str) -> Option { let body: serde_json::Value = serde_json::from_str( - &db.globals + &globals .reqwest_client() .get(&format!( "https://{}/.well-known/matrix/server", @@ -44,7 +44,7 @@ pub async fn request_well_known(db: &crate::Database, destination: &str) -> Opti } pub async fn send_request( - db: &crate::Database, + globals: &crate::database::globals::Globals, destination: &ServerName, request: T, ) -> Result @@ -52,7 +52,7 @@ where T: Debug, { let actual_destination = "https://".to_owned() - + &request_well_known(db, &destination.as_str()) + + &request_well_known(globals, &destination.as_str()) .await .unwrap_or(destination.as_str().to_owned() + ":8448"); @@ -81,14 +81,14 @@ where ); request_map.insert( "origin".to_owned(), - db.globals.server_name().as_str().into(), + globals.server_name().as_str().into(), ); request_map.insert("destination".to_owned(), destination.as_str().into()); let mut request_json = request_map.into(); ruma::signatures::sign_json( - db.globals.server_name().as_str(), - db.globals.keypair(), + globals.server_name().as_str(), + globals.keypair(), &mut request_json, ) .unwrap(); @@ -110,7 +110,7 @@ where AUTHORIZATION, HeaderValue::from_str(&format!( "X-Matrix origin={},key=\"{}\",sig=\"{}\"", - db.globals.server_name(), + globals.server_name(), s.0, s.1 )) @@ -122,7 +122,7 @@ where let reqwest_request = reqwest::Request::try_from(http_request) .expect("all http requests are valid reqwest requests"); - let reqwest_response = db.globals.reqwest_client().execute(reqwest_request).await; + let reqwest_response = globals.reqwest_client().execute(reqwest_request).await; // Because reqwest::Response -> http::Response is complicated: match reqwest_response { @@ -317,9 +317,9 @@ pub fn send_transaction_message_route<'a>( .insert("event_id".to_owned(), event_id.to_string().into()); let pdu = - serde_json::from_value::(value).expect("all ruma pdus are conduit pdus"); + serde_json::from_value::(value.clone()).expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - db.rooms.append_pdu(&pdu, &db.globals, &db.account_data)?; + db.rooms.append_pdu(&pdu, &value, &db.globals, &db.account_data)?; } } Ok(send_transaction_message::v1::Response { From 71500b14b902321e91cab432d55dd3f3ae7aedfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 08:16:20 +0200 Subject: [PATCH 29/48] fix: send to all servers and fix media store --- src/client_server/alias.rs | 2 +- src/client_server/directory.rs | 2 +- src/client_server/media.rs | 38 ++++++++-------------------- src/client_server/membership.rs | 4 +-- src/database/rooms.rs | 44 ++++++++++++++++++++++----------- src/server_server.rs | 29 ++++++++++++---------- 6 files changed, 60 insertions(+), 59 deletions(-) diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index c5c514e..c2c3eb9 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -63,7 +63,7 @@ pub async fn get_alias_helper( if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( &db.globals, - room_alias.server_name(), + room_alias.server_name().to_owned(), federation::query::get_room_information::v1::Request { room_alias }, ) .await?; diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 372ce98..c82a15f 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -122,7 +122,7 @@ pub async fn get_public_rooms_filtered_helper( { let response = server_server::send_request( &db.globals, - other_server, + other_server.to_owned(), federation::directory::get_public_rooms_filtered::v1::Request { limit, since: since.as_deref(), diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 8f7a9b9..8a93d49 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -48,22 +48,13 @@ pub fn create_content_route( #[cfg_attr( feature = "conduit_bin", - get( - "/_matrix/media/r0/download/<_server_name>/<_media_id>", - data = "" - ) + get("/_matrix/media/r0/download/<_>/<_>", data = "") )] pub async fn get_content_route( db: State<'_, Database>, body: Ruma>, - _server_name: String, - _media_id: String, ) -> ConduitResult { - let mxc = format!( - "mxc://{}/{}", - db.globals.server_name(), - utils::random_string(MXC_LENGTH) - ); + let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { filename, @@ -77,10 +68,10 @@ pub async fn get_content_route( content_disposition: filename.unwrap_or_default(), // TODO: Spec says this should be optional } .into()) - } else if body.allow_remote { + } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_content_response = server_server::send_request( &db.globals, - body.server_name.as_ref(), + body.server_name.clone(), get_content::Request { allow_remote: false, server_name: &body.server_name, @@ -104,21 +95,18 @@ pub async fn get_content_route( #[cfg_attr( feature = "conduit_bin", - get( - "/_matrix/media/r0/thumbnail/<_server_name>/<_media_id>", - data = "" - ) + get("/_matrix/media/r0/thumbnail/<_>/<_>", data = "") )] pub async fn get_content_thumbnail_route( db: State<'_, Database>, body: Ruma>, - _server_name: String, - _media_id: String, ) -> ConduitResult { + let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); + if let Some(FileMeta { content_type, file, .. }) = db.media.get_thumbnail( - format!("mxc://{}/{}", body.server_name, body.media_id), + mxc.clone(), body.width .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, @@ -127,10 +115,10 @@ pub async fn get_content_thumbnail_route( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, )? { Ok(get_content_thumbnail::Response { file, content_type }.into()) - } else if body.allow_remote { + } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_thumbnail_response = server_server::send_request( &db.globals, - body.server_name.as_ref(), + body.server_name.clone(), get_content_thumbnail::Request { allow_remote: false, height: body.height, @@ -142,12 +130,6 @@ pub async fn get_content_thumbnail_route( ) .await?; - let mxc = format!( - "mxc://{}/{}", - db.globals.server_name(), - utils::random_string(MXC_LENGTH) - ); - db.media.upload_thumbnail( mxc, &None, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 18fb5a9..f60601f 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -440,7 +440,7 @@ async fn join_room_by_id_helper( for remote_server in servers { let make_join_response = server_server::send_request( &db.globals, - remote_server, + remote_server.clone(), federation::membership::create_join_event_template::v1::Request { room_id, user_id: sender_id, @@ -501,7 +501,7 @@ async fn join_room_by_id_helper( let send_join_response = server_server::send_request( &db.globals, - remote_server, + remote_server.clone(), federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index ba54e7f..3c3a0b2 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1,9 +1,10 @@ mod edus; pub use edus::RoomEdus; +use rocket::futures; use crate::{pdu::PduBuilder, server_server, utils, Error, PduEvent, Result}; -use log::error; +use log::{error, warn}; use ring::digest; use ruma::{ api::client::error::ErrorKind, @@ -833,20 +834,35 @@ impl Rooms { .expect("json is object") .remove("event_id"); - let response = server_server::send_request( - &globals, - "koesters.xyz".try_into().unwrap(), - federation::transactions::send_transaction_message::v1::Request { - origin: globals.server_name(), - pdus: &[serde_json::from_value(pdu_json).expect("Raw::from_value always works")], - edus: &[], - origin_server_ts: SystemTime::now(), - transaction_id: &utils::random_string(16), - }, - ) - .await; + let raw_json = + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - let _ = dbg!(response); + let pdus = &[raw_json]; + let transaction_id = utils::random_string(16); + + for result in futures::future::join_all( + self.room_servers(room_id) + .filter_map(|r| r.ok()) + .filter(|server| &**server != globals.server_name()) + .map(|server| { + server_server::send_request( + &globals, + server, + federation::transactions::send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus, + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &transaction_id, + }, + ) + }), + ) + .await { + if let Err(e) = result { + warn!("{}", e); + } + } Ok(pdu.event_id) } diff --git a/src/server_server.rs b/src/server_server.rs index 9f4be13..da5a6c1 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,5 +1,6 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; +use log::warn; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ api::federation::directory::get_public_rooms_filtered, @@ -24,7 +25,10 @@ use std::{ time::{Duration, SystemTime}, }; -pub async fn request_well_known(globals: &crate::database::globals::Globals, destination: &str) -> Option { +pub async fn request_well_known( + globals: &crate::database::globals::Globals, + destination: &str, +) -> Option { let body: serde_json::Value = serde_json::from_str( &globals .reqwest_client() @@ -45,7 +49,7 @@ pub async fn request_well_known(globals: &crate::database::globals::Globals, des pub async fn send_request( globals: &crate::database::globals::Globals, - destination: &ServerName, + destination: Box, request: T, ) -> Result where @@ -79,10 +83,7 @@ where .to_string() .into(), ); - request_map.insert( - "origin".to_owned(), - globals.server_name().as_str().into(), - ); + request_map.insert("origin".to_owned(), globals.server_name().as_str().into()); request_map.insert("destination".to_owned(), destination.as_str().into()); let mut request_json = request_map.into(); @@ -144,10 +145,11 @@ where .into_iter() .collect(); - Ok( - T::IncomingResponse::try_from(http_response.body(body).unwrap()) - .expect("TODO: error handle other server errors"), - ) + let response = T::IncomingResponse::try_from(http_response.body(body).unwrap()); + response.map_err(|e| { + warn!("{}", e); + Error::BadServerResponse("Server returned bad response.") + }) } Err(e) => Err(e.into()), } @@ -316,10 +318,11 @@ pub fn send_transaction_message_route<'a>( .expect("ruma pdus are json objects") .insert("event_id".to_owned(), event_id.to_string().into()); - let pdu = - serde_json::from_value::(value.clone()).expect("all ruma pdus are conduit pdus"); + let pdu = serde_json::from_value::(value.clone()) + .expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - db.rooms.append_pdu(&pdu, &value, &db.globals, &db.account_data)?; + db.rooms + .append_pdu(&pdu, &value, &db.globals, &db.account_data)?; } } Ok(send_transaction_message::v1::Response { From 0b263208e39a735fcb6970d168494783ff9994a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 08:55:02 +0200 Subject: [PATCH 30/48] fix: don't panic on bad server names --- src/server_server.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index da5a6c1..40ad654 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -62,14 +62,18 @@ where let mut http_request = request .try_into_http_request(&actual_destination, Some("")) - .unwrap(); + .map_err(|e| { + warn!("{}: {}", actual_destination, e); + Error::BadServerResponse("Invalid destination") + })?; let mut request_map = serde_json::Map::new(); if !http_request.body().is_empty() { request_map.insert( "content".to_owned(), - serde_json::from_slice(http_request.body()).unwrap(), + serde_json::from_slice(http_request.body()) + .expect("body is valid json, we just created it"), ); }; @@ -92,7 +96,7 @@ where globals.keypair(), &mut request_json, ) - .unwrap(); + .expect("our request json is what ruma expects"); let signatures = request_json["signatures"] .as_object() @@ -145,7 +149,11 @@ where .into_iter() .collect(); - let response = T::IncomingResponse::try_from(http_response.body(body).unwrap()); + let response = T::IncomingResponse::try_from( + http_response + .body(body) + .expect("reqwest body is valid http body"), + ); response.map_err(|e| { warn!("{}", e); Error::BadServerResponse("Server returned bad response.") From b7ab57897bc96e468421cf82ecd7d49e75c3f7dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 16:13:54 +0200 Subject: [PATCH 31/48] fix: sending slowness --- Cargo.lock | 2 +- src/client_server/account.rs | 30 +- src/client_server/membership.rs | 6 + src/client_server/message.rs | 42 +-- src/client_server/profile.rs | 138 +++++---- src/client_server/redact.rs | 36 ++- src/client_server/room.rs | 516 ++++++++++++++++++-------------- src/client_server/state.rs | 48 +-- src/database.rs | 5 + src/database/globals.rs | 21 +- src/database/rooms.rs | 58 ++-- src/database/rooms/edus.rs | 1 + src/database/sending.rs | 83 +++++ src/main.rs | 2 + src/pdu.rs | 3 +- 15 files changed, 574 insertions(+), 417 deletions(-) create mode 100644 src/database/sending.rs diff --git a/Cargo.lock b/Cargo.lock index e0de2a7..30144ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1831,7 +1831,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#a9186476b748c901fbf4356414247a0b3ac01b5f" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#1d01b6e65b6afd50e65085fb40f1e7d2782f519e" dependencies = [ "itertools", "js_int", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 2ec9282..7e0f942 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -354,19 +354,23 @@ pub async fn deactivate_route( third_party_invite: None, }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // Remove devices and mark account as deactivated diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index f60601f..c4eed95 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -120,6 +120,7 @@ pub async fn leave_room_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -157,6 +158,7 @@ pub async fn invite_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -209,6 +211,7 @@ pub async fn kick_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -266,6 +269,7 @@ pub async fn ban_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -314,6 +318,7 @@ pub async fn unban_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -672,6 +677,7 @@ async fn join_room_by_id_helper( &sender_id, &room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 4ba0d9f..3944d5b 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -49,25 +49,29 @@ pub async fn send_message_event_route( let mut unsigned = serde_json::Map::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into()); - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: body.content.event_type().into(), - content: serde_json::from_str( - body.json_body - .as_ref() - .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? - .get(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, - unsigned: Some(unsigned), - state_key: None, - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - ).await?; + let event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: body.content.event_type().into(), + content: serde_json::from_str( + body.json_body + .as_ref() + .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? + .get(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, + unsigned: Some(unsigned), + state_key: None, + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; db.transaction_ids .add_txnid(sender_id, device_id, &body.txn_id, event_id.as_bytes())?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index be893e1..53893c0 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -31,40 +31,43 @@ pub async fn set_displayname_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - displayname: body.displayname.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + displayname: body.displayname.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( "Tried to send displayname update for user not in the room.", ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Presence update db.rooms.edus.update_presence( @@ -134,40 +137,43 @@ pub async fn set_avatar_url_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - avatar_url: body.avatar_url.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( - "Tried to send avatar url update for user not in the room.", - ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + avatar_url: body.avatar_url.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( + "Tried to send avatar url update for user not in the room.", + ) + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Presence update db.rooms.edus.update_presence( diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 701fc00..24df8dd 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -18,22 +18,26 @@ pub async fn redact_event_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomRedaction, - content: serde_json::to_value(redaction::RedactionEventContent { - reason: body.reason.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: Some(body.event_id.clone()), - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - ).await?; + let event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomRedaction, + content: serde_json::to_value(redaction::RedactionEventContent { + reason: body.reason.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: Some(body.event_id.clone()), + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; Ok(redact_event::Response { event_id }.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 0e5c571..d21148b 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -53,41 +53,47 @@ pub async fn create_room_route( content.room_version = RoomVersionId::Version6; // 1. The room create event - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 2. Let the room creator join - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 3. Power levels let mut users = BTreeMap::new(); @@ -117,19 +123,22 @@ pub async fn create_room_route( }) .expect("event is valid, we just created it") }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomPowerLevels, - content: power_levels_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: power_levels_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 4. Events set by preset @@ -140,73 +149,84 @@ pub async fn create_room_route( }); // 4.1 Join Rules - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomJoinRules, - content: match preset { - create_room::RoomPreset::PublicChat => serde_json::to_value( - join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), - ) - .expect("event is valid, we just created it"), - // according to spec "invite" is the default - _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( - join_rules::JoinRule::Invite, - )) - .expect("event is valid, we just created it"), + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomJoinRules, + content: match preset { + create_room::RoomPreset::PublicChat => serde_json::to_value( + join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), + ) + .expect("event is valid, we just created it"), + // according to spec "invite" is the default + _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( + join_rules::JoinRule::Invite, + )) + .expect("event is valid, we just created it"), + }, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 4.2 History Visibility - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomHistoryVisibility, - content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( - history_visibility::HistoryVisibility::Shared, - )) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomHistoryVisibility, + content: serde_json::to_value( + history_visibility::HistoryVisibilityEventContent::new( + history_visibility::HistoryVisibility::Shared, + ), + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 4.3 Guest Access - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomGuestAccess, - content: match preset { - create_room::RoomPreset::PublicChat => { - serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::Forbidden, + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomGuestAccess, + content: match preset { + create_room::RoomPreset::PublicChat => { + serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::Forbidden, + )) + .expect("event is valid, we just created it") + } + _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::CanJoin, )) - .expect("event is valid, we just created it") - } - _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::CanJoin, - )) - .expect("event is valid, we just created it"), + .expect("event is valid, we just created it"), + }, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 5. Events listed in initial_state for event in &body.initial_state { @@ -220,78 +240,90 @@ pub async fn create_room_route( continue; } - db.rooms.build_and_append_pdu( - pdu_builder, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + pdu_builder, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // 6. Events implied by name and topic if let Some(name) = &body.name { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomName, - content: serde_json::to_value( - name::NameEventContent::new(name.clone()).map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") - })?, - ) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomName, + content: serde_json::to_value( + name::NameEventContent::new(name.clone()).map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") + })?, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } if let Some(topic) = &body.topic { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTopic, - content: serde_json::to_value(topic::TopicEventContent { - topic: topic.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTopic, + content: serde_json::to_value(topic::TopicEventContent { + topic: topic.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // 7. Events implied by invite (and TODO: invite_3pid) for user in &body.invite { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user)?, - avatar_url: db.users.avatar_url(&user)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user)?, + avatar_url: db.users.avatar_url(&user)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // Homeserver specific stuff @@ -363,23 +395,29 @@ pub async fn upgrade_room_route( // Send a m.room.tombstone event to the old room to indicate that it is not intended to be used any further // Fail if the sender does not have the required permissions - let tombstone_event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTombstone, - content: serde_json::to_value(ruma::events::room::tombstone::TombstoneEventContent { - body: "This room has been replaced".to_string(), - replacement_room: replacement_room.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &body.room_id, - &db.globals, - &db.account_data, - ).await?; + let tombstone_event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTombstone, + content: serde_json::to_value( + ruma::events::room::tombstone::TombstoneEventContent { + body: "This room has been replaced".to_string(), + replacement_room: replacement_room.clone(), + }, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Get the old room federations status let federate = serde_json::from_value::>( @@ -406,42 +444,48 @@ pub async fn upgrade_room_route( create_event_content.room_version = new_version; create_event_content.predecessor = predecessor; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(create_event_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(create_event_content) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Join the new room - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Recommended transferable state events list from the specs let transferable_state_events = vec![ @@ -463,19 +507,22 @@ pub async fn upgrade_room_route( None => continue, // Skipping missing events. }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type, - content: event_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type, + content: event_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // Moves any local aliases to the new room @@ -505,7 +552,8 @@ pub async fn upgrade_room_route( power_levels_event_content.invite = new_level; // Modify the power levels in the old room to prevent sending of events and inviting new users - let _ = db.rooms + let _ = db + .rooms .build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, @@ -518,8 +566,10 @@ pub async fn upgrade_room_route( sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, - ).await; + ) + .await; // Return the replacement room id Ok(upgrade_room::Response { replacement_room }.into()) diff --git a/src/client_server/state.rs b/src/client_server/state.rs index e9d20e2..46182a1 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -33,17 +33,18 @@ pub async fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - Ok( - send_state_event_for_key::Response::new(send_state_event_for_key_helper( + Ok(send_state_event_for_key::Response::new( + send_state_event_for_key_helper( &db, sender_id, &body.content, content, &body.room_id, Some(body.state_key.to_owned()), - ).await?) - .into(), + ) + .await?, ) + .into()) } #[cfg_attr( @@ -70,8 +71,8 @@ pub async fn send_state_event_for_empty_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - Ok( - send_state_event_for_empty_key::Response::new(send_state_event_for_key_helper( + Ok(send_state_event_for_empty_key::Response::new( + send_state_event_for_key_helper( &db, sender_id .as_ref() @@ -80,9 +81,10 @@ pub async fn send_state_event_for_empty_key_route( json, &body.room_id, Some("".into()), - ).await?) - .into(), + ) + .await?, ) + .into()) } #[cfg_attr( @@ -211,19 +213,23 @@ pub async fn send_state_event_for_key_helper( } } - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: content.event_type().into(), - content: json, - unsigned: None, - state_key, - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + let event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: content.event_type().into(), + content: json, + unsigned: None, + state_key, + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; Ok(event_id) } diff --git a/src/database.rs b/src/database.rs index e1a356c..4b2cba1 100644 --- a/src/database.rs +++ b/src/database.rs @@ -3,6 +3,7 @@ pub mod globals; pub mod key_backups; pub mod media; pub mod rooms; +pub mod sending; pub mod transaction_ids; pub mod uiaa; pub mod users; @@ -25,6 +26,7 @@ pub struct Database { pub media: media::Media, pub key_backups: key_backups::KeyBackups, pub transaction_ids: transaction_ids::TransactionIds, + pub sending: sending::Sending, pub _db: sled::Db, } @@ -135,6 +137,9 @@ impl Database { transaction_ids: transaction_ids::TransactionIds { userdevicetxnid_response: db.open_tree("userdevicetxnid_response")?, }, + sending: sending::Sending { + serverpduids: db.open_tree("serverpduids")?, + }, _db: db, }) } diff --git a/src/database/globals.rs b/src/database/globals.rs index 5db2806..8951425 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -1,12 +1,13 @@ use crate::{utils, Error, Result}; use ruma::ServerName; -use std::convert::TryInto; +use std::{convert::TryInto, sync::Arc}; pub const COUNTER: &str = "c"; +#[derive(Clone)] pub struct Globals { pub(super) globals: sled::Tree, - keypair: ruma::signatures::Ed25519KeyPair, + keypair: Arc, reqwest_client: reqwest::Client, server_name: Box, max_request_size: u32, @@ -16,13 +17,15 @@ pub struct Globals { impl Globals { pub fn load(globals: sled::Tree, config: &rocket::Config) -> Result { - let keypair = ruma::signatures::Ed25519KeyPair::new( - &*globals - .update_and_fetch("keypair", utils::generate_keypair)? - .expect("utils::generate_keypair always returns Some"), - "key1".to_owned(), - ) - .map_err(|_| Error::bad_database("Private or public keys are invalid."))?; + let keypair = Arc::new( + ruma::signatures::Ed25519KeyPair::new( + &*globals + .update_and_fetch("keypair", utils::generate_keypair)? + .expect("utils::generate_keypair always returns Some"), + "key1".to_owned(), + ) + .map_err(|_| Error::bad_database("Private or public keys are invalid."))?, + ); Ok(Self { globals, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 3c3a0b2..2246a61 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1,14 +1,12 @@ mod edus; pub use edus::RoomEdus; -use rocket::futures; -use crate::{pdu::PduBuilder, server_server, utils, Error, PduEvent, Result}; -use log::{error, warn}; +use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; +use log::error; use ring::digest; use ruma::{ api::client::error::ErrorKind, - api::federation, events::{ ignored_user_list, room::{ @@ -27,7 +25,6 @@ use std::{ convert::{TryFrom, TryInto}, mem, sync::Arc, - time::SystemTime, }; /// The unique identifier of each state group. @@ -36,6 +33,7 @@ use std::{ /// hashing the entire state. pub type StateHashId = Vec; +#[derive(Clone)] pub struct Rooms { pub edus: edus::RoomEdus, pub(super) pduid_pdu: sled::Tree, // PduId = RoomId + Count @@ -415,6 +413,16 @@ impl Rooms { }) } + /// Returns the pdu. + pub fn get_pdu_json_from_id(&self, pdu_id: &IVec) -> Result> { + self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| { + Ok(Some( + serde_json::from_slice(&pdu) + .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + )) + }) + } + /// Removes a pdu and creates a new one with the same id. fn replace_pdu(&self, pdu_id: &IVec, pdu: &PduEvent) -> Result<()> { if self.pduid_pdu.get(&pdu_id)?.is_some() { @@ -613,6 +621,7 @@ impl Rooms { sender: &UserId, room_id: &RoomId, globals: &super::globals::Globals, + sending: &super::sending::Sending, account_data: &super::account_data::AccountData, ) -> Result { let PduBuilder { @@ -829,39 +838,12 @@ impl Rooms { self.append_to_state(&pdu_id, &pdu)?; } - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); - - let raw_json = - serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - - let pdus = &[raw_json]; - let transaction_id = utils::random_string(16); - - for result in futures::future::join_all( - self.room_servers(room_id) - .filter_map(|r| r.ok()) - .filter(|server| &**server != globals.server_name()) - .map(|server| { - server_server::send_request( - &globals, - server, - federation::transactions::send_transaction_message::v1::Request { - origin: globals.server_name(), - pdus, - edus: &[], - origin_server_ts: SystemTime::now(), - transaction_id: &transaction_id, - }, - ) - }), - ) - .await { - if let Err(e) = result { - warn!("{}", e); - } + for server in self + .room_servers(room_id) + .filter_map(|r| r.ok()) + .filter(|server| &**server != globals.server_name()) + { + sending.send_pdu(server, &pdu_id)?; } Ok(pdu.event_id) diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index d60e1f1..a794c69 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -13,6 +13,7 @@ use std::{ convert::{TryFrom, TryInto}, }; +#[derive(Clone)] pub struct RoomEdus { pub(in super::super) readreceiptid_readreceipt: sled::Tree, // ReadReceiptId = RoomId + Count + UserId pub(in super::super) roomuserid_privateread: sled::Tree, // RoomUserId = Room + User, PrivateRead = Count diff --git a/src/database/sending.rs b/src/database/sending.rs new file mode 100644 index 0000000..187fd57 --- /dev/null +++ b/src/database/sending.rs @@ -0,0 +1,83 @@ +use std::{convert::TryFrom, time::SystemTime}; + +use crate::{server_server, utils, Error, Result}; +use rocket::futures::stream::{FuturesUnordered, StreamExt}; +use ruma::{api::federation, Raw, ServerName}; +use tokio::select; + +pub struct Sending { + /// The state for a given state hash. + pub(super) serverpduids: sled::Tree, // ServerPduId = ServerName + PduId +} + +impl Sending { + pub fn start_handler(&self, globals: &super::globals::Globals, rooms: &super::rooms::Rooms) { + let serverpduids = self.serverpduids.clone(); + let rooms = rooms.clone(); + let globals = globals.clone(); + + tokio::spawn(async move { + let mut futures = FuturesUnordered::new(); + let mut subscriber = serverpduids.watch_prefix(b""); + loop { + select! { + Some(_) = futures.next() => {}, + Some(event) = &mut subscriber => { + let serverpduid = if let sled::Event::Insert {key, ..} = event { + key + } else + { return Err::<(), Error>(Error::bad_database("")); }; + let mut parts = serverpduid.splitn(2, |&b| b == 0xff); + let server = Box::::try_from( + utils::string_from_bytes(parts.next().expect("splitn will always return 1 or more elements")) + .map_err(|_| Error::bad_database("ServerName in serverpduid bytes are invalid."))? + ).map_err(|_| Error::bad_database("ServerName in serverpduid is invalid."))?; + + let pdu_id = parts.next().ok_or_else(|| Error::bad_database("Invalid serverpduid in db."))?; + let mut pdu_json = rooms.get_pdu_json_from_id(&pdu_id.into())?.ok_or_else(|| Error::bad_database("Event in serverpduids not found in db."))?; + + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + let raw_json = + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); + + let globals = &globals; + + futures.push( + async move { + let pdus = vec![raw_json]; + let transaction_id = utils::random_string(16); + + server_server::send_request( + &globals, + server, + federation::transactions::send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus: &pdus, + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &transaction_id, + }, + ).await + } + ); + }, + } + } + }); + } + /* + */ + + pub fn send_pdu(&self, server: Box, pdu_id: &[u8]) -> Result<()> { + let mut key = server.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(pdu_id); + self.serverpduids.insert(key, b"")?; + + Ok(()) + } +} diff --git a/src/main.rs b/src/main.rs index eb060e3..2817ab9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -130,6 +130,8 @@ fn setup_rocket() -> rocket::Rocket { .attach(AdHoc::on_attach("Config", |mut rocket| async { let data = Database::load_or_create(rocket.config().await).expect("valid config"); + data.sending.start_handler(&data.globals, &data.rooms); + Ok(rocket.manage(data)) })) } diff --git a/src/pdu.rs b/src/pdu.rs index c904230..6d78092 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,12 +1,13 @@ use crate::{Error, Result}; use js_int::UInt; use ruma::{ + events::pdu::PduStub, events::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, -events::pdu::PduStub}; +}; use serde::{Deserialize, Serialize}; use serde_json::json; use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH}; From 1bf614b0f57a023cbc467f1e5bf03d1eae87b755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 17:02:20 +0200 Subject: [PATCH 32/48] fix: remove transaction_id from pdus over federation --- src/database/sending.rs | 6 ++++++ src/pdu.rs | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/database/sending.rs b/src/database/sending.rs index 187fd57..77998e7 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -36,6 +36,12 @@ impl Sending { let pdu_id = parts.next().ok_or_else(|| Error::bad_database("Invalid serverpduid in db."))?; let mut pdu_json = rooms.get_pdu_json_from_id(&pdu_id.into())?.ok_or_else(|| Error::bad_database("Event in serverpduids not found in db."))?; + if let Some(unsigned) = pdu_json + .as_object_mut() + .expect("json is object") + .get_mut("unsigned") { + unsigned.as_object_mut().expect("unsigned is object").remove("transaction_id"); + } pdu_json .as_object_mut() .expect("json is object") diff --git a/src/pdu.rs b/src/pdu.rs index 6d78092..957d9e0 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -201,6 +201,9 @@ impl PduEvent { } pub fn to_outgoing_federation_event(&self) -> Raw { + let mut unsigned = self.unsigned.clone(); + unsigned.remove("transaction_id"); + let mut json = json!({ "room_id": self.room_id, "sender": self.sender, @@ -210,7 +213,7 @@ impl PduEvent { "prev_events": self.prev_events, "depth": self.depth, "auth_events": self.auth_events, - "unsigned": self.unsigned, + "unsigned": unsigned, "hashes": self.hashes, "signatures": self.signatures, }); From 005e00e9b18459d569cca1993138a85d10dfc271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 17:16:55 +0200 Subject: [PATCH 33/48] fix: remove well-known --- src/main.rs | 1 - src/server_server.rs | 5 ----- 2 files changed, 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2817ab9..f81c7f4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -119,7 +119,6 @@ fn setup_rocket() -> rocket::Rocket { client_server::get_pushers_route, client_server::set_pushers_route, client_server::upgrade_room_route, - server_server::well_known_server, server_server::get_server_version, server_server::get_server_keys, server_server::get_server_keys_deprecated, diff --git a/src/server_server.rs b/src/server_server.rs index 40ad654..106f60e 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -163,11 +163,6 @@ where } } -#[cfg_attr(feature = "conduit_bin", get("/.well-known/matrix/server"))] -pub fn well_known_server() -> Json { - rocket::response::content::Json(json!({ "m.server": "pc.koesters.xyz:59003"}).to_string()) -} - #[cfg_attr(feature = "conduit_bin", get("/_matrix/federation/v1/version"))] pub fn get_server_version() -> ConduitResult { Ok(get_server_version::Response { From dd749b8aee7c09ca8084059f91cd922e95fb6424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 21:46:10 +0200 Subject: [PATCH 34/48] fix: server keys and destination resolution when server name contains port --- src/database/globals.rs | 45 ++++++++++++++++++++++++++++++++--------- src/server_server.rs | 9 +++++++-- src/utils.rs | 9 +++++++-- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/database/globals.rs b/src/database/globals.rs index 8951425..8ce9c01 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -1,4 +1,5 @@ use crate::{utils, Error, Result}; +use log::error; use ruma::ServerName; use std::{convert::TryInto, sync::Arc}; @@ -17,19 +18,43 @@ pub struct Globals { impl Globals { pub fn load(globals: sled::Tree, config: &rocket::Config) -> Result { - let keypair = Arc::new( - ruma::signatures::Ed25519KeyPair::new( - &*globals - .update_and_fetch("keypair", utils::generate_keypair)? - .expect("utils::generate_keypair always returns Some"), - "key1".to_owned(), - ) - .map_err(|_| Error::bad_database("Private or public keys are invalid."))?, - ); + let bytes = &*globals + .update_and_fetch("keypair", utils::generate_keypair)? + .expect("utils::generate_keypair always returns Some"); + + let mut parts = bytes.splitn(2, |&b| b == 0xff); + + let keypair = utils::string_from_bytes( + // 1. version + parts + .next() + .expect("splitn always returns at least one element"), + ) + .map_err(|_| Error::bad_database("Invalid version bytes in keypair.")) + .and_then(|version| { + // 2. key + parts + .next() + .ok_or_else(|| Error::bad_database("Invalid keypair format in database.")) + .map(|key| (version, key)) + }) + .and_then(|(version, key)| { + ruma::signatures::Ed25519KeyPair::new(&key, version) + .map_err(|_| Error::bad_database("Private or public keys are invalid.")) + }); + + let keypair = match keypair { + Ok(k) => k, + Err(e) => { + error!("Keypair invalid. Deleting..."); + globals.remove("keypair")?; + return Err(e); + } + }; Ok(Self { globals, - keypair, + keypair: Arc::new(keypair), reqwest_client: reqwest::Client::new(), server_name: config .get_str("server_name") diff --git a/src/server_server.rs b/src/server_server.rs index 106f60e..f334d6b 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -17,7 +17,6 @@ use ruma::{ directory::{IncomingFilter, IncomingRoomNetwork}, EventId, ServerName, }; -use serde_json::json; use std::{ collections::BTreeMap, convert::TryFrom, @@ -58,7 +57,13 @@ where let actual_destination = "https://".to_owned() + &request_well_known(globals, &destination.as_str()) .await - .unwrap_or(destination.as_str().to_owned() + ":8448"); + .unwrap_or_else(|| { + let mut destination = destination.as_str().to_owned(); + if destination.find(':').is_none() { + destination += ":8448"; + } + destination + }); let mut http_request = request .try_into_http_request(&actual_destination, Some("")) diff --git a/src/utils.rs b/src/utils.rs index 8cf1b2c..452b7c5 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -29,8 +29,13 @@ pub fn increment(old: Option<&[u8]>) -> Option> { pub fn generate_keypair(old: Option<&[u8]>) -> Option> { Some(old.map(|s| s.to_vec()).unwrap_or_else(|| { - ruma::signatures::Ed25519KeyPair::generate() - .expect("Ed25519KeyPair generation always works (?)") + let mut value = random_string(8).as_bytes().to_vec(); + value.push(0xff); + value.extend_from_slice( + &ruma::signatures::Ed25519KeyPair::generate() + .expect("Ed25519KeyPair generation always works (?)"), + ); + value })) } From f4078a29eb2e4975bc5664aab718875ce67da6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 10:49:54 +0200 Subject: [PATCH 35/48] fix: synapse complains about missing origin --- src/database/rooms.rs | 6 ++++++ src/database/sending.rs | 1 + 2 files changed, 7 insertions(+) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 2246a61..8e68033 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -812,6 +812,12 @@ impl Rooms { .expect("json is object") .remove("event_id"); + // Add origin because synapse likes that (and it's required in the spec) + pdu_json + .as_object_mut() + .expect("json is object") + .insert("origin".to_owned(), globals.server_name().as_str().into()); + ruma::signatures::hash_and_sign_event( globals.server_name().as_str(), globals.keypair(), diff --git a/src/database/sending.rs b/src/database/sending.rs index 77998e7..a3f1574 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -42,6 +42,7 @@ impl Sending { .get_mut("unsigned") { unsigned.as_object_mut().expect("unsigned is object").remove("transaction_id"); } + pdu_json .as_object_mut() .expect("json is object") From a567cd81d5e849285e6ef14b4d7ac41dd436c8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 15:08:51 +0200 Subject: [PATCH 36/48] improvement: better logs on deserialization errors --- src/server_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server_server.rs b/src/server_server.rs index f334d6b..aef3991 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -160,7 +160,7 @@ where .expect("reqwest body is valid http body"), ); response.map_err(|e| { - warn!("{}", e); + warn!("Server returned bad response: {:?}", e); Error::BadServerResponse("Server returned bad response.") }) } From 4db6d7e4308b206e0f34e291f1d23e7a54ea254a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 18:10:17 +0200 Subject: [PATCH 37/48] fix: remove avatar url checks They are not in the spec and maubot relies on that --- src/client_server/profile.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 53893c0..686d4c3 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -119,18 +119,6 @@ pub async fn set_avatar_url_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - if let Some(avatar_url) = &body.avatar_url { - if !avatar_url.starts_with("mxc://") { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "avatar_url has to start with mxc://.", - )); - } - - // TODO in the future when we can handle media uploads make sure that this url is our own server - // TODO also make sure this is valid mxc:// format (not only starting with it) - } - db.users .set_avatar_url(&sender_id, body.avatar_url.clone())?; From 506c2a3146bb5314c95ad75ed069869a724ce628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 21:11:38 +0200 Subject: [PATCH 38/48] fix: can't find count from event in db --- src/client_server/sync.rs | 31 ++++++++++++++----------------- src/database/rooms.rs | 25 +++++++++++++------------ 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index eeeec00..6ece180 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -93,7 +93,7 @@ pub async fn sync_events_route( let mut limited = false; let mut state_pdus = Vec::new(); - for pdu in non_timeline_pdus { + for (_, pdu) in non_timeline_pdus { if pdu.state_key.is_some() { state_pdus.push(pdu); } @@ -113,7 +113,7 @@ pub async fn sync_events_route( .rooms .pdus_since(&sender_id, &room_id, since)? .filter_map(|r| r.ok()) - .filter_map(|pdu| Some((pdu.state_key.clone()?, pdu))) + .filter_map(|(_, pdu)| Some((pdu.state_key.clone()?, pdu))) { if pdu.kind == EventType::RoomMember { send_member_count = true; @@ -188,8 +188,8 @@ pub async fn sync_events_route( .rooms .all_pdus(&sender_id, &room_id)? .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus - .filter(|pdu| pdu.kind == EventType::RoomMember) - .map(|pdu| { + .filter(|(_, pdu)| pdu.kind == EventType::RoomMember) + .map(|(_, pdu)| { let content = serde_json::from_value::< Raw, >(pdu.content.clone()) @@ -244,7 +244,7 @@ pub async fn sync_events_route( (db.rooms .pdus_since(&sender_id, &room_id, last_read)? .filter_map(|pdu| pdu.ok()) // Filter out buggy events - .filter(|pdu| { + .filter(|(_, pdu)| { matches!( pdu.kind.clone(), EventType::RoomMessage | EventType::RoomEncrypted @@ -260,18 +260,15 @@ pub async fn sync_events_route( None }; - let prev_batch = timeline_pdus.first().map_or(Ok::<_, Error>(None), |e| { - Ok(Some( - db.rooms - .get_pdu_count(&e.event_id)? - .ok_or_else(|| Error::bad_database("Can't find count from event in db."))? - .to_string(), - )) - })?; + let prev_batch = timeline_pdus + .first() + .map_or(Ok::<_, Error>(None), |(pdu_id, _)| { + Ok(Some(db.rooms.pdu_count(pdu_id)?.to_string())) + })?; let room_events = timeline_pdus .into_iter() - .map(|pdu| pdu.to_sync_room_event()) + .map(|(_, pdu)| pdu.to_sync_room_event()) .collect::>(); let mut edus = db @@ -380,7 +377,7 @@ pub async fn sync_events_route( let pdus = db.rooms.pdus_since(&sender_id, &room_id, since)?; let room_events = pdus .filter_map(|pdu| pdu.ok()) // Filter out buggy events - .map(|pdu| pdu.to_sync_room_event()) + .map(|(_, pdu)| pdu.to_sync_room_event()) .collect(); let left_room = sync_events::LeftRoom { @@ -395,7 +392,7 @@ pub async fn sync_events_route( let mut left_since_last_sync = false; for pdu in db.rooms.pdus_since(&sender_id, &room_id, since)? { - let pdu = pdu?; + let (_, pdu) = pdu?; if pdu.kind == EventType::RoomMember && pdu.state_key == Some(sender_id.to_string()) { let content = serde_json::from_value::< Raw, @@ -438,7 +435,7 @@ pub async fn sync_events_route( let room_id = room_id?; let mut invited_since_last_sync = false; for pdu in db.rooms.pdus_since(&sender_id, &room_id, since)? { - let pdu = pdu?; + let (_, pdu) = pdu?; if pdu.kind == EventType::RoomMember && pdu.state_key == Some(sender_id.to_string()) { let content = serde_json::from_value::< Raw, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 8e68033..263f51b 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -355,18 +355,19 @@ impl Rooms { } } + /// Returns the `count` of this pdu's id. + pub fn pdu_count(&self, pdu_id: &[u8]) -> Result { + Ok( + utils::u64_from_bytes(&pdu_id[pdu_id.len() - mem::size_of::()..pdu_id.len()]) + .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?, + ) + } + /// Returns the `count` of this pdu's id. pub fn get_pdu_count(&self, event_id: &EventId) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? - .map_or(Ok(None), |pdu_id| { - Ok(Some( - utils::u64_from_bytes( - &pdu_id[pdu_id.len() - mem::size_of::()..pdu_id.len()], - ) - .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?, - )) - }) + .map_or(Ok(None), |pdu_id| self.pdu_count(&pdu_id).map(Some)) } /// Returns the json of a pdu. @@ -860,7 +861,7 @@ impl Rooms { &self, user_id: &UserId, room_id: &RoomId, - ) -> Result>> { + ) -> Result>> { self.pdus_since(user_id, room_id, 0) } @@ -871,7 +872,7 @@ impl Rooms { user_id: &UserId, room_id: &RoomId, since: u64, - ) -> Result>> { + ) -> Result>> { let mut prefix = room_id.to_string().as_bytes().to_vec(); prefix.push(0xff); @@ -887,13 +888,13 @@ impl Rooms { .pduid_pdu .range(first_pdu_id..last_pdu_id) .filter_map(|r| r.ok()) - .map(move |(_, v)| { + .map(move |(pdu_id, v)| { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { pdu.unsigned.remove("transaction_id"); } - Ok(pdu) + Ok((pdu_id, pdu)) })) } From ea3aaa6b5c06e01bef52a66b64fe45d74d5f60c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 17 Sep 2020 14:44:47 +0200 Subject: [PATCH 39/48] improvement: more efficient /sync with gaps --- Cargo.lock | 48 +++---- src/client_server/context.rs | 10 +- src/client_server/membership.rs | 11 +- src/client_server/message.rs | 12 ++ src/client_server/sync.rs | 152 +++++++++++++------- src/database/rooms.rs | 247 ++++++++++++-------------------- src/main.rs | 1 + src/pdu.rs | 6 +- src/server_server.rs | 4 +- src/stateres.rs | 59 -------- 10 files changed, 251 insertions(+), 299 deletions(-) delete mode 100644 src/stateres.rs diff --git a/Cargo.lock b/Cargo.lock index 30144ca..e142d72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,9 +134,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.59" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" +checksum = "ef611cc68ff783f18535d77ddd080185275713d852c4f5cbb6122c462a7a825c" [[package]] name = "cfg-if" @@ -213,7 +213,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ "percent-encoding", - "time 0.2.19", + "time 0.2.20", "version_check", ] @@ -342,9 +342,9 @@ checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" [[package]] name = "either" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" @@ -1370,7 +1370,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "state", - "time 0.2.19", + "time 0.2.20", "tokio", "toml", "version_check", @@ -1405,7 +1405,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.19", + "time 0.2.20", "tokio", "tokio-rustls", "unicode-xid", @@ -1414,7 +1414,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1430,7 +1430,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "http", "percent-encoding", @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1456,7 +1456,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-common", @@ -1469,7 +1469,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "assign", "http", @@ -1488,7 +1488,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1502,7 +1502,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-common", @@ -1517,7 +1517,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1528,7 +1528,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1543,7 +1543,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1555,7 +1555,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro2", "quote", @@ -1566,7 +1566,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "serde", "strum", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "form_urlencoded", "itoa", @@ -1587,7 +1587,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "base64", "ring", @@ -1831,7 +1831,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#1d01b6e65b6afd50e65085fb40f1e7d2782f519e" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", @@ -1981,9 +1981,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c1a1fd93112fc50b11c43a1def21f926be3c18884fad676ea879572da070a1" +checksum = "0d4953c513c9bf1b97e9cdd83f11d60c4b0a83462880a360d80d96953a953fee" dependencies = [ "const_fn", "libc", diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 9593726..4c9be20 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -49,7 +49,10 @@ pub fn get_context_route( .filter_map(|r| r.ok()) // Remove buggy events .collect::>(); - let start_token = events_before.last().map(|(count, _)| count.to_string()); + let start_token = events_before + .last() + .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) + .map(|count| count.to_string()); let events_before = events_before .into_iter() @@ -68,7 +71,10 @@ pub fn get_context_route( .filter_map(|r| r.ok()) // Remove buggy events .collect::>(); - let end_token = events_after.last().map(|(count, _)| count.to_string()); + let end_token = events_after + .last() + .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) + .map(|count| count.to_string()); let events_after = events_after .into_iter() diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index c4eed95..628045d 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -601,8 +601,7 @@ async fn join_room_by_id_helper( .cloned() .collect::>(); - let power_level = - resolved_control_events.get(&(EventType::RoomPowerLevels, Some("".into()))); + let power_level = resolved_control_events.get(&(EventType::RoomPowerLevels, "".into())); // Sort the remaining non control events let sorted_event_ids = state_res::StateResolution::mainline_sort( room_id, @@ -644,13 +643,7 @@ async fn join_room_by_id_helper( )?; if state_events.contains(ev_id) { - state.insert( - ( - pdu.kind(), - pdu.state_key().expect("State events have a state key"), - ), - pdu_id, - ); + state.insert((pdu.kind(), pdu.state_key()), pdu_id); } } diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 3944d5b..5a4488f 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -117,6 +117,12 @@ pub fn get_message_events_route( .pdus_after(&sender_id, &body.room_id, from) .take(limit) .filter_map(|r| r.ok()) // Filter out buggy events + .filter_map(|(pdu_id, pdu)| { + db.rooms + .pdu_count(&pdu_id) + .map(|pdu_count| (pdu_count, pdu)) + .ok() + }) .take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to` .collect::>(); @@ -141,6 +147,12 @@ pub fn get_message_events_route( .pdus_until(&sender_id, &body.room_id, from) .take(limit) .filter_map(|r| r.ok()) // Filter out buggy events + .filter_map(|(pdu_id, pdu)| { + db.rooms + .pdu_count(&pdu_id) + .map(|pdu_count| (pdu_count, pdu)) + .ok() + }) .take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to` .collect::>(); diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 6ece180..0e40bfb 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -105,50 +105,92 @@ pub async fn sync_events_route( .room_state_get(&room_id, &EventType::RoomEncryption, "")? .is_some(); - // TODO: optimize this? - let mut send_member_count = false; - let mut joined_since_last_sync = false; - let mut new_encrypted_room = false; - for (state_key, pdu) in db + // Database queries: + let since_state_hash = db .rooms - .pdus_since(&sender_id, &room_id, since)? - .filter_map(|r| r.ok()) - .filter_map(|(_, pdu)| Some((pdu.state_key.clone()?, pdu))) - { - if pdu.kind == EventType::RoomMember { - send_member_count = true; + .pdus_until(sender_id, &room_id, since) + .next() + .and_then(|pdu| pdu.ok()) + .and_then(|pdu| db.rooms.pdu_state_hash(&pdu.0).ok()?); - let content = serde_json::from_value::< - Raw, - >(pdu.content.clone()) + let since_members = since_state_hash + .as_ref() + .and_then(|state_hash| db.rooms.state_type(state_hash, &EventType::RoomMember).ok()); + + let since_encryption = since_state_hash.as_ref().and_then(|state_hash| { + db.rooms + .state_get(&state_hash, &EventType::RoomEncryption, "") + .ok() + }); + + let current_members = db.rooms.room_state_type(&room_id, &EventType::RoomMember)?; + + // Calculations: + let new_encrypted_room = encrypted_room && since_encryption.is_none(); + + let send_member_count = since_members.as_ref().map_or(true, |since_members| { + current_members.len() != since_members.len() + }); + + let since_sender_member = since_members.as_ref().and_then(|members| { + members.get(sender_id.as_str()).and_then(|pdu| { + serde_json::from_value::>( + pdu.content.clone(), + ) .expect("Raw::from_value always works") .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database."))?; + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }) + }); - if pdu.state_key == Some(sender_id.to_string()) - && content.membership == MembershipState::Join - { - joined_since_last_sync = true; - } else if encrypted_room && content.membership == MembershipState::Join { - // A new user joined an encrypted room - let user_id = UserId::try_from(state_key) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; - // Add encryption update if we didn't share an encrypted room already - if !share_encrypted_room(&db, &sender_id, &user_id, &room_id) { - device_list_updates.insert(user_id); + if encrypted_room { + for (user_id, current_member) in current_members { + let current_membership = serde_json::from_value::< + Raw, + >(current_member.content.clone()) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database."))? + .membership; + + let since_membership = since_members + .as_ref() + .and_then(|members| { + members.get(&user_id).and_then(|since_member| { + serde_json::from_value::< + Raw, + >(since_member.content.clone()) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }) + }) + .map_or(MembershipState::Leave, |member| member.membership); + + let user_id = UserId::try_from(user_id) + .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; + + match (since_membership, current_membership) { + (MembershipState::Leave, MembershipState::Join) => { + // A new user joined an encrypted room + if !share_encrypted_room(&db, &sender_id, &user_id, &room_id) { + device_list_updates.insert(user_id); + } } - } else if encrypted_room && content.membership == MembershipState::Leave { - // Write down users that have left encrypted rooms we are in - left_encrypted_users.insert( - UserId::try_from(state_key) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?, - ); + (MembershipState::Join, MembershipState::Leave) => { + // Write down users that have left encrypted rooms we are in + left_encrypted_users.insert(user_id); + } + _ => {} } - } else if pdu.kind == EventType::RoomEncryption { - new_encrypted_room = true; } } + let joined_since_last_sync = + since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + if joined_since_last_sync && encrypted_room || new_encrypted_room { // If the user is in a new encrypted room, give them all joined users device_list_updates.extend( @@ -390,23 +432,37 @@ pub async fn sync_events_route( state: sync_events::State { events: Vec::new() }, }; - let mut left_since_last_sync = false; - for pdu in db.rooms.pdus_since(&sender_id, &room_id, since)? { - let (_, pdu) = pdu?; - if pdu.kind == EventType::RoomMember && pdu.state_key == Some(sender_id.to_string()) { - let content = serde_json::from_value::< - Raw, - >(pdu.content.clone()) + let since_member = db + .rooms + .pdus_until(sender_id, &room_id, since) + .next() + .and_then(|pdu| pdu.ok()) + .and_then(|pdu| { + db.rooms + .pdu_state_hash(&pdu.0) + .ok()? + .ok_or_else(|| Error::bad_database("Pdu in db doesn't have a state hash.")) + .ok() + }) + .and_then(|state_hash| { + db.rooms + .state_get(&state_hash, &EventType::RoomMember, sender_id.as_str()) + .ok()? + .ok_or_else(|| Error::bad_database("State hash in db doesn't have a state.")) + .ok() + }) + .and_then(|pdu| { + serde_json::from_value::>( + pdu.content.clone(), + ) .expect("Raw::from_value always works") .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database."))?; + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }); - if content.membership == MembershipState::Leave { - left_since_last_sync = true; - break; - } - } - } + let left_since_last_sync = + since_member.map_or(false, |member| member.membership == MembershipState::Join); if left_since_last_sync { device_list_left.extend( diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 263f51b..5958626 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -31,7 +31,7 @@ use std::{ /// /// This is created when a state group is added to the database by /// hashing the entire state. -pub type StateHashId = Vec; +pub type StateHashId = IVec; #[derive(Clone)] pub struct Rooms { @@ -100,7 +100,7 @@ impl StateStore for Rooms { impl Rooms { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. - pub fn state_full(&self, state_hash: StateHashId) -> Result> { + pub fn state_full(&self, state_hash: &StateHashId) -> Result> { self.stateid_pduid .scan_prefix(&state_hash) .values() @@ -115,61 +115,87 @@ impl Rooms { }) .map(|pdu| { let pdu = pdu?; - Ok(((pdu.kind, pdu.state_key), pdu.event_id)) + Ok(( + ( + pdu.kind.clone(), + pdu.state_key + .as_ref() + .ok_or_else(|| Error::bad_database("State event has no state key."))? + .clone(), + ), + pdu, + )) }) .collect::>>() } - // TODO make this return Result - /// Fetches the previous StateHash ID to `current`. - pub fn prev_state_hash(&self, current: StateHashId) -> Option { - let mut found = false; - for pair in self.pduid_statehash.iter().rev() { - let prev = pair.ok()?.1; - if current == prev.as_ref() { - found = true; - } - if current != prev.as_ref() && found { - return Some(prev.to_vec()); - } + /// Returns all state entries for this type. + pub fn state_type( + &self, + state_hash: &StateHashId, + event_type: &EventType, + ) -> Result> { + let mut prefix = state_hash.to_vec(); + prefix.push(0xff); + prefix.extend_from_slice(&event_type.to_string().as_bytes()); + prefix.push(0xff); + + let mut hashmap = HashMap::new(); + for pdu in self + .stateid_pduid + .scan_prefix(&prefix) + .values() + .map(|pdu_id| { + Ok::<_, Error>( + serde_json::from_slice::(&self.pduid_pdu.get(pdu_id?)?.ok_or_else( + || Error::bad_database("PDU in state not found in database."), + )?) + .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?, + ) + }) + { + let pdu = pdu?; + let state_key = pdu.state_key.clone().ok_or_else(|| { + Error::bad_database("Room state contains event without state_key.") + })?; + hashmap.insert(state_key, pdu); } - None + Ok(hashmap) + } + + /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). + pub fn state_get( + &self, + state_hash: &StateHashId, + event_type: &EventType, + state_key: &str, + ) -> Result> { + let mut key = state_hash.to_vec(); + key.push(0xff); + key.extend_from_slice(&event_type.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(&state_key.as_bytes()); + + self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| { + Ok::<_, Error>(Some( + serde_json::from_slice::( + &self.pduid_pdu.get(pdu_id)?.ok_or_else(|| { + Error::bad_database("PDU in state not found in database.") + })?, + ) + .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?, + )) + }) + } + + /// Returns the last state hash key added to the db. + pub fn pdu_state_hash(&self, pdu_id: &[u8]) -> Result> { + Ok(self.pduid_statehash.get(pdu_id)?) } /// Returns the last state hash key added to the db. pub fn current_state_hash(&self, room_id: &RoomId) -> Result> { - Ok(self - .roomid_statehash - .get(room_id.as_bytes())? - .map(|bytes| bytes.to_vec())) - } - - /// This fetches auth event_ids from the current state using the - /// full `roomstateid_pdu` tree. - pub fn get_auth_event_ids( - &self, - room_id: &RoomId, - kind: &EventType, - sender: &UserId, - state_key: Option<&str>, - content: serde_json::Value, - ) -> Result> { - let auth_events = state_res::auth_types_for_event( - kind.clone(), - sender, - state_key.map(|s| s.to_string()), - content, - ); - - let mut events = vec![]; - for (event_type, state_key) in auth_events { - if let Some(state_key) = state_key.as_ref() { - if let Some(id) = self.room_state_get(room_id, &event_type, state_key)? { - events.push(id.event_id); - } - } - } - Ok(events) + Ok(self.roomid_statehash.get(room_id.as_bytes())?) } /// This fetches auth events from the current state. @@ -190,10 +216,8 @@ impl Rooms { let mut events = StateMap::new(); for (event_type, state_key) in auth_events { - if let Some(s_key) = state_key.as_ref() { - if let Some(pdu) = self.room_state_get(room_id, &event_type, s_key)? { - events.insert((event_type, state_key), pdu); - } + if let Some(pdu) = self.room_state_get(room_id, &event_type, &state_key)? { + events.insert((event_type, state_key), pdu); } } Ok(events) @@ -206,7 +230,7 @@ impl Rooms { // We only hash the pdu's event ids, not the whole pdu let bytes = pdu_id_bytes.join(&0xff); let hash = digest::digest(&digest::SHA256, &bytes); - Ok(hash.as_ref().to_vec()) + Ok(hash.as_ref().into()) } /// Checks if a room exists. @@ -230,7 +254,7 @@ impl Rooms { ) -> Result<()> { let state_hash = self.calculate_hash(&state.values().map(|pdu_id| &**pdu_id).collect::>())?; - let mut prefix = state_hash.clone(); + let mut prefix = state_hash.to_vec(); prefix.push(0xff); for ((event_type, state_key), pdu_id) in state { @@ -248,41 +272,11 @@ impl Rooms { } /// Returns the full room state. - pub fn room_state_full( - &self, - room_id: &RoomId, - ) -> Result> { + pub fn room_state_full(&self, room_id: &RoomId) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - let mut prefix = current_state_hash; - prefix.push(0xff); - - let mut hashmap = HashMap::new(); - for pdu in self - .stateid_pduid - .scan_prefix(prefix) - .values() - .map(|pdu_id| { - Ok::<_, Error>( - serde_json::from_slice::( - &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { - Error::bad_database("PDU in state not found in database.") - })?, - ) - .map_err(|_| { - Error::bad_database("Invalid PDU bytes in current room state.") - })?, - ) - }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert((pdu.kind.clone(), state_key), pdu); - } - Ok(hashmap) + self.state_full(¤t_state_hash) } else { - Ok(HashMap::new()) + Ok(BTreeMap::new()) } } @@ -293,36 +287,7 @@ impl Rooms { event_type: &EventType, ) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - let mut prefix = current_state_hash; - prefix.push(0xff); - prefix.extend_from_slice(&event_type.to_string().as_bytes()); - prefix.push(0xff); - - let mut hashmap = HashMap::new(); - for pdu in self - .stateid_pduid - .scan_prefix(&prefix) - .values() - .map(|pdu_id| { - Ok::<_, Error>( - serde_json::from_slice::( - &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { - Error::bad_database("PDU in state not found in database.") - })?, - ) - .map_err(|_| { - Error::bad_database("Invalid PDU bytes in current room state.") - })?, - ) - }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert(state_key, pdu); - } - Ok(hashmap) + self.state_type(¤t_state_hash, event_type) } else { Ok(HashMap::new()) } @@ -336,20 +301,7 @@ impl Rooms { state_key: &str, ) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - let mut key = current_state_hash; - key.push(0xff); - key.extend_from_slice(&event_type.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(&state_key.as_bytes()); - - self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| { - Ok::<_, Error>(Some( - serde_json::from_slice::(&self.pduid_pdu.get(pdu_id)?.ok_or_else( - || Error::bad_database("PDU in state not found in database."), - )?) - .map_err(|_| Error::bad_database("Invalid PDU bytes in current room state."))?, - )) - }) + self.state_get(¤t_state_hash, event_type, state_key) } else { Ok(None) } @@ -562,14 +514,15 @@ impl Rooms { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. /// The incoming event is the `pdu_id` passed to this method. - fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result { + pub fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result { let old_state = if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? { // Store state for event. The state does not include the event itself. // Instead it's the state before the pdu, so the room's old state. - self.pduid_statehash.insert(new_pdu_id, &old_state_hash)?; + self.pduid_statehash + .insert(dbg!(new_pdu_id), &old_state_hash)?; if new_pdu.state_key.is_none() { - return Ok(old_state_hash.to_vec()); + return Ok(old_state_hash); } let mut prefix = old_state_hash.to_vec(); @@ -841,9 +794,7 @@ impl Rooms { let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data)?; - if pdu.state_key.is_some() { - self.append_to_state(&pdu_id, &pdu)?; - } + self.append_to_state(&pdu_id, &pdu)?; for server in self .room_servers(room_id) @@ -905,7 +856,7 @@ impl Rooms { user_id: &UserId, room_id: &RoomId, until: u64, - ) -> impl Iterator> { + ) -> impl Iterator> { // Create the first part of the full pdu id let mut prefix = room_id.to_string().as_bytes().to_vec(); prefix.push(0xff); @@ -916,23 +867,18 @@ impl Rooms { let current: &[u8] = ¤t; let user_id = user_id.clone(); - let prefixlen = prefix.len(); self.pduid_pdu .range(..current) .rev() .filter_map(|r| r.ok()) .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(move |(k, v)| { + .map(move |(pdu_id, v)| { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { pdu.unsigned.remove("transaction_id"); } - Ok(( - utils::u64_from_bytes(&k[prefixlen..]) - .map_err(|_| Error::bad_database("Invalid pdu id in db."))?, - pdu, - )) + Ok((pdu_id, pdu)) }) } @@ -943,7 +889,7 @@ impl Rooms { user_id: &UserId, room_id: &RoomId, from: u64, - ) -> impl Iterator> { + ) -> impl Iterator> { // Create the first part of the full pdu id let mut prefix = room_id.to_string().as_bytes().to_vec(); prefix.push(0xff); @@ -954,22 +900,17 @@ impl Rooms { let current: &[u8] = ¤t; let user_id = user_id.clone(); - let prefixlen = prefix.len(); self.pduid_pdu .range(current..) .filter_map(|r| r.ok()) .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(move |(k, v)| { + .map(move |(pdu_id, v)| { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { pdu.unsigned.remove("transaction_id"); } - Ok(( - utils::u64_from_bytes(&k[prefixlen..]) - .map_err(|_| Error::bad_database("Invalid pdu id in db."))?, - pdu, - )) + Ok((pdu_id, pdu)) }) } diff --git a/src/main.rs b/src/main.rs index f81c7f4..06fda59 100644 --- a/src/main.rs +++ b/src/main.rs @@ -123,6 +123,7 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_server_keys, server_server::get_server_keys_deprecated, server_server::get_public_rooms_route, + server_server::get_public_rooms_filtered_route, server_server::send_transaction_message_route, ], ) diff --git a/src/pdu.rs b/src/pdu.rs index 957d9e0..d5b5415 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,4 +1,4 @@ -use crate::{Error, Result}; +use crate::Error; use js_int::UInt; use ruma::{ events::pdu::PduStub, @@ -35,7 +35,7 @@ pub struct PduEvent { } impl PduEvent { - pub fn redact(&mut self, reason: &PduEvent) -> Result<()> { + pub fn redact(&mut self, reason: &PduEvent) -> crate::Result<()> { self.unsigned.clear(); let allowed: &[&str] = match self.kind { @@ -244,7 +244,7 @@ impl From<&state_res::StateEvent> for PduEvent { .expect("time is valid"), kind: pdu.kind(), content: pdu.content().clone(), - state_key: pdu.state_key(), + state_key: Some(pdu.state_key()), prev_events: pdu.prev_event_ids(), depth: pdu.depth().clone(), auth_events: pdu.auth_events(), diff --git a/src/server_server.rs b/src/server_server.rs index aef3991..6f2b179 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -329,8 +329,10 @@ pub fn send_transaction_message_route<'a>( let pdu = serde_json::from_value::(value.clone()) .expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - db.rooms + let pdu_id = db + .rooms .append_pdu(&pdu, &value, &db.globals, &db.account_data)?; + db.rooms.append_to_state(&pdu_id, &pdu)?; } } Ok(send_transaction_message::v1::Response { diff --git a/src/stateres.rs b/src/stateres.rs deleted file mode 100644 index ee47099..0000000 --- a/src/stateres.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::collections::HashMap; - -fn stateres(state_a: HashMap, state_b: HashMap) { - let mut unconflicted = todo!("state at fork event"); - - let mut conflicted: HashMap = state_a - .iter() - .filter(|(key_a, value_a)| match state_b.remove(key_a) { - Some(value_b) if value_a == value_b => unconflicted.insert(key_a, value_a), - _ => false, - }) - .collect(); - - // We removed unconflicted from state_b, now we can easily insert all events that are only in fork b - conflicted.extend(state_b); - - let partial_state = unconflicted.clone(); - - let full_conflicted = conflicted.clone(); // TODO: auth events - - let output_rev = Vec::new(); - let event_map = HashMap::new(); - let incoming_edges = HashMap::new(); - - for event in full_conflicted { - event_map.insert(event.event_id, event); - incoming_edges.insert(event.event_id, 0); - } - - for e in conflicted_control_events { - for a in e.auth_events { - incoming_edges[a.event_id] += 1; - } - } - - while incoming_edges.len() > 0 { - let mut count_0 = incoming_edges - .iter() - .filter(|(_, c)| c == 0) - .collect::>(); - - count_0.sort_by(|(x, _), (y, _)| { - x.power_level - .cmp(&a.power_level) - .then_with(|| x.origin_server.ts.cmp(&y.origin_server_ts)) - .then_with(|| x.event_id.cmp(&y.event_id)) - }); - - for (id, count) in count_0 { - output_rev.push(event_map[id]); - - for auth_event in event_map[id].auth_events { - incoming_edges[auth_event.event_id] -= 1; - } - - incoming_edges.remove(id); - } - } -} From 8bcfff276652e20fee14e13109d80a514e1a107d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 17 Sep 2020 19:58:19 +0200 Subject: [PATCH 40/48] fix: no notification counts for fast /syncs --- src/database/rooms.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 5958626..18881dd 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -447,6 +447,11 @@ impl Rooms { // This is also the next_batch/since value let index = globals.next_count()?; + // Mark as read first so the sending client doesn't get a notification even if appending + // fails + self.edus + .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; + let mut pdu_id = pdu.room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&index.to_be_bytes()); @@ -503,9 +508,6 @@ impl Rooms { _ => {} } - self.edus - .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; - Ok(pdu_id) } @@ -520,7 +522,7 @@ impl Rooms { // Store state for event. The state does not include the event itself. // Instead it's the state before the pdu, so the room's old state. self.pduid_statehash - .insert(dbg!(new_pdu_id), &old_state_hash)?; + .insert(new_pdu_id, &old_state_hash)?; if new_pdu.state_key.is_none() { return Ok(old_state_hash); } From 267c7216162932c9f6e35d69be4c6303d5134952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 17 Sep 2020 22:41:43 +0200 Subject: [PATCH 41/48] fix: encryption and sync spam --- src/client_server/sync.rs | 106 +++++++++++++++++++++++--------------- src/database/rooms.rs | 3 +- src/database/users.rs | 2 +- 3 files changed, 67 insertions(+), 44 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 0e40bfb..2f2c8ea 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -100,47 +100,61 @@ pub async fn sync_events_route( limited = true; } + // Database queries: let encrypted_room = db .rooms .room_state_get(&room_id, &EventType::RoomEncryption, "")? .is_some(); - // Database queries: + // These type is Option>. The outer Option is None when there is no event between + // since and the current room state, meaning there should be no updates. + // The inner Option is None when there is an event, but there is no state hash associated + // with it. This can happen for the RoomCreate event, so all updates should arrive. let since_state_hash = db .rooms - .pdus_until(sender_id, &room_id, since) + .pdus_after(sender_id, &room_id, since) // - 1 So we can get the event at since .next() - .and_then(|pdu| pdu.ok()) - .and_then(|pdu| db.rooms.pdu_state_hash(&pdu.0).ok()?); + .map(|pdu| db.rooms.pdu_state_hash(&pdu.ok()?.0).ok()?); - let since_members = since_state_hash - .as_ref() - .and_then(|state_hash| db.rooms.state_type(state_hash, &EventType::RoomMember).ok()); + let since_members = since_state_hash.as_ref().map(|state_hash| { + state_hash.as_ref().and_then(|state_hash| { + db.rooms + .state_type(&state_hash, &EventType::RoomMember) + .ok() + }) + }); - let since_encryption = since_state_hash.as_ref().and_then(|state_hash| { - db.rooms - .state_get(&state_hash, &EventType::RoomEncryption, "") - .ok() + let since_encryption = since_state_hash.as_ref().map(|state_hash| { + state_hash.as_ref().and_then(|state_hash| { + db.rooms + .state_get(&state_hash, &EventType::RoomEncryption, "") + .ok() + }) }); let current_members = db.rooms.room_state_type(&room_id, &EventType::RoomMember)?; // Calculations: - let new_encrypted_room = encrypted_room && since_encryption.is_none(); + let new_encrypted_room = + encrypted_room && since_encryption.map_or(false, |encryption| encryption.is_none()); - let send_member_count = since_members.as_ref().map_or(true, |since_members| { - current_members.len() != since_members.len() + let send_member_count = since_members.as_ref().map_or(false, |since_members| { + since_members.as_ref().map_or(true, |since_members| { + current_members.len() != since_members.len() + }) }); - let since_sender_member = since_members.as_ref().and_then(|members| { - members.get(sender_id.as_str()).and_then(|pdu| { - serde_json::from_value::>( - pdu.content.clone(), - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database.")) - .ok() + let since_sender_member = since_members.as_ref().map(|since_members| { + since_members.as_ref().and_then(|members| { + members.get(sender_id.as_str()).and_then(|pdu| { + serde_json::from_value::>( + pdu.content.clone(), + ) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }) }) }); @@ -154,20 +168,29 @@ pub async fn sync_events_route( .map_err(|_| Error::bad_database("Invalid PDU in database."))? .membership; - let since_membership = since_members - .as_ref() - .and_then(|members| { - members.get(&user_id).and_then(|since_member| { - serde_json::from_value::< - Raw, - >(since_member.content.clone()) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database.")) - .ok() - }) - }) - .map_or(MembershipState::Leave, |member| member.membership); + let since_membership = + since_members + .as_ref() + .map_or(MembershipState::Join, |members| { + members + .as_ref() + .and_then(|members| { + members.get(&user_id).and_then(|since_member| { + serde_json::from_value::< + Raw, + >( + since_member.content.clone() + ) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| { + Error::bad_database("Invalid PDU in database.") + }) + .ok() + }) + }) + .map_or(MembershipState::Leave, |member| member.membership) + }); let user_id = UserId::try_from(user_id) .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; @@ -188,8 +211,9 @@ pub async fn sync_events_route( } } - let joined_since_last_sync = - since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + let joined_since_last_sync = since_sender_member.map_or(false, |member| { + member.map_or(true, |member| member.membership != MembershipState::Join) + }); if joined_since_last_sync && encrypted_room || new_encrypted_room { // If the user is in a new encrypted room, give them all joined users @@ -434,7 +458,7 @@ pub async fn sync_events_route( let since_member = db .rooms - .pdus_until(sender_id, &room_id, since) + .pdus_after(sender_id, &room_id, since) .next() .and_then(|pdu| pdu.ok()) .and_then(|pdu| { @@ -581,7 +605,7 @@ pub async fn sync_events_route( changed: device_list_updates.into_iter().collect(), left: device_list_left.into_iter().collect(), }, - device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since { + device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since || since == 0 { db.users.count_one_time_keys(sender_id, device_id)? } else { BTreeMap::new() diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 18881dd..108edb5 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -521,8 +521,7 @@ impl Rooms { if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? { // Store state for event. The state does not include the event itself. // Instead it's the state before the pdu, so the room's old state. - self.pduid_statehash - .insert(new_pdu_id, &old_state_hash)?; + self.pduid_statehash.insert(new_pdu_id, &old_state_hash)?; if new_pdu.state_key.is_none() { return Ok(old_state_hash); } diff --git a/src/database/users.rs b/src/database/users.rs index 10e1ef3..2e26c1e 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -603,7 +603,7 @@ impl Users { .room_state_get(&room_id, &EventType::RoomEncryption, "")? .is_none() { - return Ok(()); + continue; } let mut key = room_id.to_string().as_bytes().to_vec(); From e08dfd982b87b632590bf125cdc48ee83bad9ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 23 Sep 2020 12:03:08 +0200 Subject: [PATCH 42/48] improvement: look at SRV record when sending requests --- Cargo.lock | 239 +++++++++++++++++++++++++++++++++----- Cargo.toml | 2 + src/client_server/sync.rs | 4 +- src/server_server.rs | 49 ++++++-- 4 files changed, 253 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e142d72..4cce666 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,20 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "addr2line" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + [[package]] name = "adler32" version = "1.2.0" @@ -73,6 +88,20 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "backtrace" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide 0.4.2", + "object", + "rustc-demangle", +] + [[package]] name = "base-x" version = "0.2.6" @@ -192,6 +221,7 @@ dependencies = [ "state-res", "thiserror", "tokio", + "trust-dns-resolver", ] [[package]] @@ -213,7 +243,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ "percent-encoding", - "time 0.2.20", + "time 0.2.21", "version_check", ] @@ -355,6 +385,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fnv" version = "1.0.7" @@ -537,6 +579,12 @@ dependencies = [ "lzw", ] +[[package]] +name = "gimli" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" + [[package]] name = "glob" version = "0.3.0" @@ -586,6 +634,17 @@ dependencies = [ "libc", ] +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.9", +] + [[package]] name = "http" version = "0.2.1" @@ -614,10 +673,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" [[package]] -name = "hyper" -version = "0.13.7" +name = "httpdate" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e68a8dd9716185d9e64ea473ea6ef63529252e3e27623295a0378a19665d5eb" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "hyper" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" dependencies = [ "bytes", "futures-channel", @@ -627,10 +692,10 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project", "socket2", - "time 0.1.44", "tokio", "tower-service", "tracing", @@ -695,9 +760,12 @@ checksum = "cb6ee2a7da03bfc3b66ca47c92c2e392fcc053ea040a85561749b026f7aad09a" [[package]] name = "instant" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" +checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" +dependencies = [ + "cfg-if", +] [[package]] name = "iovec" @@ -708,6 +776,18 @@ dependencies = [ "libc", ] +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2", + "widestring", + "winapi 0.3.9", + "winreg 0.6.2", +] + [[package]] name = "ipnet" version = "2.3.0" @@ -778,6 +858,12 @@ version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" +[[package]] +name = "linked-hash-map" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" + [[package]] name = "lock_api" version = "0.4.1" @@ -796,6 +882,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lzw" version = "0.10.0" @@ -808,6 +903,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + [[package]] name = "matchers" version = "0.0.1" @@ -837,9 +938,9 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "memoffset" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" dependencies = [ "autocfg", ] @@ -869,6 +970,16 @@ dependencies = [ "adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.6.22" @@ -991,6 +1102,12 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" + [[package]] name = "once_cell" version = "1.4.1" @@ -1130,7 +1247,7 @@ dependencies = [ "bitflags", "crc32fast", "deflate", - "miniz_oxide", + "miniz_oxide 0.3.7", ] [[package]] @@ -1181,6 +1298,12 @@ dependencies = [ "yansi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.7" @@ -1334,7 +1457,17 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg", + "winreg 0.7.0", +] + +[[package]] +name = "resolv-conf" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11834e137f3b14e309437a8276714eed3a80d1ef894869e510f2c0c0b98b9f4a" +dependencies = [ + "hostname", + "quick-error", ] [[package]] @@ -1370,7 +1503,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "state", - "time 0.2.20", + "time 0.2.21", "tokio", "toml", "version_check", @@ -1405,7 +1538,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.20", + "time 0.2.21", "tokio", "tokio-rustls", "unicode-xid", @@ -1414,7 +1547,6 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1430,7 +1562,6 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "http", "percent-encoding", @@ -1445,7 +1576,6 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1456,7 +1586,6 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-common", @@ -1469,7 +1598,6 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "assign", "http", @@ -1488,7 +1616,6 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1502,7 +1629,6 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-common", @@ -1517,7 +1643,6 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1528,7 +1653,6 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1543,7 +1667,6 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1555,7 +1678,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro2", "quote", @@ -1566,7 +1688,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "serde", "strum", @@ -1575,7 +1696,6 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "form_urlencoded", "itoa", @@ -1587,7 +1707,6 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "base64", "ring", @@ -1607,6 +1726,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + [[package]] name = "rustc_version" version = "0.2.3" @@ -1831,7 +1956,6 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", @@ -1981,9 +2105,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4953c513c9bf1b97e9cdd83f11d60c4b0a83462880a360d80d96953a953fee" +checksum = "2c2e31fb28e2a9f01f5ed6901b066c1ba2333c04b64dc61254142bafcb3feb2c" dependencies = [ "const_fn", "libc", @@ -2182,6 +2306,46 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "trust-dns-proto" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd7061ba6f4d4d9721afedffbfd403f20f39a4301fee1b70d6fcd09cca69f28" +dependencies = [ + "async-trait", + "backtrace", + "enum-as-inner", + "futures", + "idna", + "lazy_static", + "log", + "rand", + "smallvec", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f23cdfdc3d8300b3c50c9e84302d3bd6d860fb9529af84ace6cf9665f181b77" +dependencies = [ + "backtrace", + "cfg-if", + "futures", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "trust-dns-proto", +] + [[package]] name = "try-lock" version = "0.2.3" @@ -2366,6 +2530,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "widestring" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a763e303c0e0f23b0da40888724762e802a8ffefbc22de4127ef42493c2ea68c" + [[package]] name = "winapi" version = "0.2.8" @@ -2400,6 +2570,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "winreg" version = "0.7.0" diff --git a/Cargo.toml b/Cargo.toml index 60296a2..2126e42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,8 @@ image = { version = "0.23.9", default-features = false, features = ["jpeg", "png base64 = "0.12.3" # Used when hashing the state ring = "0.16.15" +# Used when querying the SRV record of other servers +trust-dns-resolver = "0.19.5" [features] default = ["conduit_bin"] diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 2f2c8ea..aec03af 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -605,7 +605,9 @@ pub async fn sync_events_route( changed: device_list_updates.into_iter().collect(), left: device_list_left.into_iter().collect(), }, - device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since || since == 0 { + device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since + || since == 0 + { db.users.count_one_time_keys(sender_id, device_id)? } else { BTreeMap::new() diff --git a/src/server_server.rs b/src/server_server.rs index 6f2b179..3ebbeac 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,5 +1,5 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; -use http::header::{HeaderValue, AUTHORIZATION}; +use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::warn; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ @@ -23,6 +23,7 @@ use std::{ fmt::Debug, time::{Duration, SystemTime}, }; +use trust_dns_resolver::AsyncResolver; pub async fn request_well_known( globals: &crate::database::globals::Globals, @@ -54,16 +55,36 @@ pub async fn send_request( where T: Debug, { + let resolver = AsyncResolver::tokio_from_system_conf() + .await + .map_err(|_| Error::BadConfig("Failed to set up trust dns resolver with system config."))?; + + let mut host = None; + let actual_destination = "https://".to_owned() - + &request_well_known(globals, &destination.as_str()) - .await - .unwrap_or_else(|| { - let mut destination = destination.as_str().to_owned(); - if destination.find(':').is_none() { - destination += ":8448"; + + &if let Some(mut delegated_hostname) = + request_well_known(globals, &destination.as_str()).await + { + if let Ok(Some(srv)) = resolver + .srv_lookup(format!("_matrix._tcp.{}", delegated_hostname)) + .await + .map(|srv| srv.iter().next().map(|result| result.target().to_string())) + { + host = Some(delegated_hostname); + srv.trim_end_matches('.').to_owned() + } else { + if delegated_hostname.find(':').is_none() { + delegated_hostname += ":8448"; } - destination - }); + delegated_hostname + } + } else { + let mut destination = destination.as_str().to_owned(); + if destination.find(':').is_none() { + destination += ":8448"; + } + destination + }; let mut http_request = request .try_into_http_request(&actual_destination, Some("")) @@ -129,9 +150,17 @@ where } } - let reqwest_request = reqwest::Request::try_from(http_request) + if let Some(host) = host { + http_request + .headers_mut() + .insert(HOST, HeaderValue::from_str(&host).unwrap()); + } + + let mut reqwest_request = reqwest::Request::try_from(http_request) .expect("all http requests are valid reqwest requests"); + *reqwest_request.timeout_mut() = Some(Duration::from_secs(30)); + let reqwest_response = globals.reqwest_client().execute(reqwest_request).await; // Because reqwest::Response -> http::Response is complicated: From ab332363ce3af716694cde7bf35cf421e91707d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 23 Sep 2020 15:23:29 +0200 Subject: [PATCH 43/48] fix: don't send new requests to servers if we are already waiting --- Cargo.lock | 15 ++++ src/database/sending.rs | 149 ++++++++++++++++++++++++++-------------- src/server_server.rs | 6 +- 3 files changed, 117 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4cce666..5f6b21c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1547,6 +1547,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1562,6 +1563,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "http", "percent-encoding", @@ -1576,6 +1578,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1586,6 +1589,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "ruma-api", "ruma-common", @@ -1598,6 +1602,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "assign", "http", @@ -1616,6 +1621,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "js_int", "ruma-api", @@ -1629,6 +1635,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "js_int", "ruma-common", @@ -1643,6 +1650,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1653,6 +1661,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "js_int", "ruma-api", @@ -1667,6 +1676,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1678,6 +1688,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "proc-macro2", "quote", @@ -1688,6 +1699,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "serde", "strum", @@ -1696,6 +1708,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "form_urlencoded", "itoa", @@ -1707,6 +1720,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "base64", "ring", @@ -1956,6 +1970,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", diff --git a/src/database/sending.rs b/src/database/sending.rs index a3f1574..d3c7fc6 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -1,8 +1,11 @@ -use std::{convert::TryFrom, time::SystemTime}; +use std::{collections::HashSet, convert::TryFrom, time::SystemTime}; use crate::{server_server, utils, Error, Result}; +use federation::transactions::send_transaction_message; +use log::warn; use rocket::futures::stream::{FuturesUnordered, StreamExt}; use ruma::{api::federation, Raw, ServerName}; +use sled::IVec; use tokio::select; pub struct Sending { @@ -18,66 +21,49 @@ impl Sending { tokio::spawn(async move { let mut futures = FuturesUnordered::new(); + let mut waiting_servers = HashSet::new(); + let mut subscriber = serverpduids.watch_prefix(b""); loop { select! { - Some(_) = futures.next() => {}, - Some(event) = &mut subscriber => { - let serverpduid = if let sled::Event::Insert {key, ..} = event { - key - } else - { return Err::<(), Error>(Error::bad_database("")); }; - let mut parts = serverpduid.splitn(2, |&b| b == 0xff); - let server = Box::::try_from( - utils::string_from_bytes(parts.next().expect("splitn will always return 1 or more elements")) - .map_err(|_| Error::bad_database("ServerName in serverpduid bytes are invalid."))? - ).map_err(|_| Error::bad_database("ServerName in serverpduid is invalid."))?; - - let pdu_id = parts.next().ok_or_else(|| Error::bad_database("Invalid serverpduid in db."))?; - let mut pdu_json = rooms.get_pdu_json_from_id(&pdu_id.into())?.ok_or_else(|| Error::bad_database("Event in serverpduids not found in db."))?; - - if let Some(unsigned) = pdu_json - .as_object_mut() - .expect("json is object") - .get_mut("unsigned") { - unsigned.as_object_mut().expect("unsigned is object").remove("transaction_id"); - } - - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); - - let raw_json = - serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - - let globals = &globals; - - futures.push( - async move { - let pdus = vec![raw_json]; - let transaction_id = utils::random_string(16); - - server_server::send_request( - &globals, - server, - federation::transactions::send_transaction_message::v1::Request { - origin: globals.server_name(), - pdus: &pdus, - edus: &[], - origin_server_ts: SystemTime::now(), - transaction_id: &transaction_id, - }, - ).await + Some(server) = futures.next() => { + warn!("response: {:?}", &server); + match server { + Ok((server, _response)) => { + waiting_servers.remove(&server) } - ); + Err((server, _e)) => { + waiting_servers.remove(&server) + } + }; }, + Some(event) = &mut subscriber => { + if let sled::Event::Insert { key, .. } = event { + let serverpduid = key.clone(); + let mut parts = serverpduid.splitn(2, |&b| b == 0xff); + + if let Some((server, pdu_id)) = utils::string_from_bytes( + parts + .next() + .expect("splitn will always return 1 or more elements"), + ) + .map_err(|_| Error::bad_database("ServerName in serverpduid bytes are invalid.")) + .and_then(|server_str|Box::::try_from(server_str) + .map_err(|_| Error::bad_database("ServerName in serverpduid is invalid."))) + .ok() + .filter(|server| waiting_servers.insert(server.clone())) + .and_then(|server| parts + .next() + .ok_or_else(|| Error::bad_database("Invalid serverpduid in db.")).ok().map(|pdu_id| (server, pdu_id))) + { + futures.push(Self::handle_event(server, pdu_id.into(), &globals, &rooms)); + } + } + } } } }); } - /* - */ pub fn send_pdu(&self, server: Box, pdu_id: &[u8]) -> Result<()> { let mut key = server.as_bytes().to_vec(); @@ -87,4 +73,63 @@ impl Sending { Ok(()) } + + async fn handle_event( + server: Box, + pdu_id: IVec, + globals: &super::globals::Globals, + rooms: &super::rooms::Rooms, + ) -> std::result::Result< + (Box, send_transaction_message::v1::Response), + (Box, Error), + > { + let mut pdu_json = rooms + .get_pdu_json_from_id(&pdu_id) + .map_err(|e| (server.clone(), e))? + .ok_or_else(|| { + ( + server.clone(), + Error::bad_database("Event in serverpduids not found in db."), + ) + })?; + + if let Some(unsigned) = pdu_json + .as_object_mut() + .expect("json is object") + .get_mut("unsigned") + { + unsigned + .as_object_mut() + .expect("unsigned is object") + .remove("transaction_id"); + } + + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + let raw_json = + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); + + let globals = &globals; + + let pdus = vec![raw_json]; + let transaction_id = utils::random_string(16); + + server_server::send_request( + &globals, + server.clone(), + send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus: &pdus, + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &transaction_id, + }, + ) + .await + .map(|response| (server.clone(), response)) + .map_err(|e| (server, e)) + } } diff --git a/src/server_server.rs b/src/server_server.rs index 3ebbeac..d67b0b6 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -161,6 +161,7 @@ where *reqwest_request.timeout_mut() = Some(Duration::from_secs(30)); + let url = reqwest_request.url().clone(); let reqwest_response = globals.reqwest_client().execute(reqwest_request).await; // Because reqwest::Response -> http::Response is complicated: @@ -189,7 +190,10 @@ where .expect("reqwest body is valid http body"), ); response.map_err(|e| { - warn!("Server returned bad response: {:?}", e); + warn!( + "Server returned bad response {} ({}): {:?}", + destination, url, e + ); Error::BadServerResponse("Server returned bad response.") }) } From 0d6159c2dafc87685dc2e6fd79b034e4b3f21c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 25 Sep 2020 12:26:29 +0200 Subject: [PATCH 44/48] improvement: get_missing_events route and cleanup --- Cargo.lock | 32 ++++++++++++------------- src/database/sending.rs | 52 ++++++++++++----------------------------- src/main.rs | 1 + src/pdu.rs | 43 ++++++++++++++-------------------- src/server_server.rs | 47 +++++++++++++++++++++++++++++++++++-- 5 files changed, 95 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f6b21c..ed20ca3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -627,9 +627,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" +checksum = "4c30f6d0bc6b00693347368a67d41b58f2fb851215ff1da49e90fe2c5c667151" dependencies = [ "libc", ] @@ -1547,7 +1547,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1563,7 +1563,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "http", "percent-encoding", @@ -1578,7 +1578,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1589,7 +1589,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-common", @@ -1602,7 +1602,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "assign", "http", @@ -1621,7 +1621,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1635,7 +1635,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-common", @@ -1650,7 +1650,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1661,7 +1661,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1688,7 +1688,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro2", "quote", @@ -1699,7 +1699,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "serde", "strum", @@ -1708,7 +1708,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "form_urlencoded", "itoa", @@ -1720,7 +1720,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "base64", "ring", diff --git a/src/database/sending.rs b/src/database/sending.rs index d3c7fc6..1ed94cc 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -1,10 +1,10 @@ use std::{collections::HashSet, convert::TryFrom, time::SystemTime}; -use crate::{server_server, utils, Error, Result}; +use crate::{server_server, utils, Error, PduEvent, Result}; use federation::transactions::send_transaction_message; use log::warn; use rocket::futures::stream::{FuturesUnordered, StreamExt}; -use ruma::{api::federation, Raw, ServerName}; +use ruma::{api::federation, ServerName}; use sled::IVec; use tokio::select; @@ -83,49 +83,27 @@ impl Sending { (Box, send_transaction_message::v1::Response), (Box, Error), > { - let mut pdu_json = rooms - .get_pdu_json_from_id(&pdu_id) - .map_err(|e| (server.clone(), e))? - .ok_or_else(|| { - ( - server.clone(), - Error::bad_database("Event in serverpduids not found in db."), - ) - })?; - - if let Some(unsigned) = pdu_json - .as_object_mut() - .expect("json is object") - .get_mut("unsigned") - { - unsigned - .as_object_mut() - .expect("unsigned is object") - .remove("transaction_id"); - } - - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); - - let raw_json = - serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - - let globals = &globals; - - let pdus = vec![raw_json]; - let transaction_id = utils::random_string(16); + let pdu_json = PduEvent::to_outgoing_federation_event( + rooms + .get_pdu_json_from_id(&pdu_id) + .map_err(|e| (server.clone(), e))? + .ok_or_else(|| { + ( + server.clone(), + Error::bad_database("Event in serverpduids not found in db."), + ) + })?, + ); server_server::send_request( &globals, server.clone(), send_transaction_message::v1::Request { origin: globals.server_name(), - pdus: &pdus, + pdus: &[pdu_json], edus: &[], origin_server_ts: SystemTime::now(), - transaction_id: &transaction_id, + transaction_id: &utils::random_string(16), }, ) .await diff --git a/src/main.rs b/src/main.rs index 06fda59..fa1cc5c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -125,6 +125,7 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_public_rooms_route, server_server::get_public_rooms_filtered_route, server_server::send_transaction_message_route, + server_server::get_missing_events_route, ], ) .attach(AdHoc::on_attach("Config", |mut rocket| async { diff --git a/src/pdu.rs b/src/pdu.rs index d5b5415..4b1df4b 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,7 +1,6 @@ use crate::Error; use js_int::UInt; use ruma::{ - events::pdu::PduStub, events::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, @@ -200,32 +199,26 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - pub fn to_outgoing_federation_event(&self) -> Raw { - let mut unsigned = self.unsigned.clone(); - unsigned.remove("transaction_id"); - - let mut json = json!({ - "room_id": self.room_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "type": self.kind, - "content": self.content, - "prev_events": self.prev_events, - "depth": self.depth, - "auth_events": self.auth_events, - "unsigned": unsigned, - "hashes": self.hashes, - "signatures": self.signatures, - }); - - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &self.redacts { - json["redacts"] = json!(redacts); + pub fn to_outgoing_federation_event( + mut pdu_json: serde_json::Value, + ) -> Raw { + if let Some(unsigned) = pdu_json + .as_object_mut() + .expect("json is object") + .get_mut("unsigned") + { + unsigned + .as_object_mut() + .expect("unsigned is object") + .remove("transaction_id"); } - serde_json::from_value(json).expect("Raw::from_value always works") + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works") } } diff --git a/src/server_server.rs b/src/server_server.rs index d67b0b6..2d52c4a 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -3,13 +3,13 @@ use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::warn; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ - api::federation::directory::get_public_rooms_filtered, api::{ federation::{ - directory::get_public_rooms, + directory::{get_public_rooms, get_public_rooms_filtered}, discovery::{ get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, }, + event::get_missing_events, transactions::send_transaction_message, }, OutgoingRequest, @@ -373,3 +373,46 @@ pub fn send_transaction_message_route<'a>( } .into()) } + +#[cfg_attr( + feature = "conduit_bin", + post("/_matrix/federation/v1/get_missing_events/<_>", data = "") +)] +pub fn get_missing_events_route<'a>( + db: State<'a, Database>, + body: Ruma>, +) -> ConduitResult { + let mut queued_events = body.latest_events.clone(); + let mut events = Vec::new(); + + let mut i = 0; + while i < queued_events.len() && events.len() < u64::from(body.limit) as usize { + if let Some(pdu) = db.rooms.get_pdu_json(&queued_events[i])? { + if body.earliest_events.contains( + &serde_json::from_value( + pdu.get("event_id") + .cloned() + .ok_or_else(|| Error::bad_database("Event in db has no event_id field."))?, + ) + .map_err(|_| Error::bad_database("Invalid event_id field in pdu in db."))?, + ) { + i += 1; + continue; + } + queued_events.extend_from_slice( + &serde_json::from_value::>( + pdu.get("prev_events").cloned().ok_or_else(|| { + Error::bad_database("Invalid prev_events field of pdu in db.") + })?, + ) + .map_err(|_| Error::bad_database("Invalid prev_events content in pdu in db."))?, + ); + events.push(PduEvent::to_outgoing_federation_event(pdu)); + } + i += 1; + } + + dbg!(&events); + + Ok(get_missing_events::v1::Response { events }.into()) +} From bcd1fe18561cbda0f26f53464325fccd177e42ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 5 Oct 2020 22:19:22 +0200 Subject: [PATCH 45/48] feat: admin room --- Cargo.lock | 15 - src/client_server/account.rs | 301 +++++++++++++++-- src/client_server/membership.rs | 196 ++++++----- src/client_server/message.rs | 43 ++- src/client_server/profile.rs | 140 ++++---- src/client_server/redact.rs | 37 +-- src/client_server/room.rs | 556 +++++++++++++++----------------- src/client_server/state.rs | 31 +- src/database/rooms.rs | 49 ++- src/database/users.rs | 5 + src/main.rs | 1 + src/server_server.rs | 64 +++- 12 files changed, 864 insertions(+), 574 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed20ca3..6571f7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1547,7 +1547,6 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1563,7 +1562,6 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "http", "percent-encoding", @@ -1578,7 +1576,6 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1589,7 +1586,6 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-common", @@ -1602,7 +1598,6 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "assign", "http", @@ -1621,7 +1616,6 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1635,7 +1629,6 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-common", @@ -1650,7 +1643,6 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1661,7 +1653,6 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1676,7 +1667,6 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1688,7 +1678,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro2", "quote", @@ -1699,7 +1688,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "serde", "strum", @@ -1708,7 +1696,6 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "form_urlencoded", "itoa", @@ -1720,7 +1707,6 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "base64", "ring", @@ -1970,7 +1956,6 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 7e0f942..66b4a62 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -1,3 +1,5 @@ +use std::{collections::BTreeMap, convert::TryInto}; + use super::{State, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; use crate::{pdu::PduBuilder, utils, ConduitResult, Database, Error, Ruma}; use ruma::{ @@ -11,8 +13,11 @@ use ruma::{ uiaa::{AuthFlow, UiaaInfo}, }, }, - events::{room::member, EventType}, - UserId, + events::{ + room::canonical_alias, room::guest_access, room::history_visibility, room::join_rules, + room::member, room::name, room::topic, EventType, + }, + RoomAliasId, RoomId, RoomVersionId, UserId, }; use register::RegistrationKind; @@ -73,7 +78,7 @@ pub fn get_register_available_route( feature = "conduit_bin", post("/_matrix/client/r0/register", data = "") )] -pub fn register_route( +pub async fn register_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -202,6 +207,265 @@ pub fn register_route( body.initial_device_display_name.clone(), )?; + // If this is the first user on this server, create the admins room + if db.users.count() == 1 { + // Create a user for the server + let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name()) + .expect("@conduit:server_name is valid"); + + db.users.create(&conduit_user, "")?; + + let room_id = RoomId::new(db.globals.server_name()); + + let mut content = ruma::events::room::create::CreateEventContent::new(conduit_user.clone()); + content.federate = true; + content.predecessor = None; + content.room_version = RoomVersionId::Version6; + + // 1. The room create event + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 2. Make conduit bot join + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(conduit_user.to_string()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 3. Power levels + let mut users = BTreeMap::new(); + users.insert(conduit_user.clone(), 100.into()); + users.insert(user_id.clone(), 100.into()); + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: serde_json::to_value( + ruma::events::room::power_levels::PowerLevelsEventContent { + ban: 50.into(), + events: BTreeMap::new(), + events_default: 0.into(), + invite: 50.into(), + kick: 50.into(), + redact: 50.into(), + state_default: 50.into(), + users, + users_default: 0.into(), + notifications: ruma::events::room::power_levels::NotificationPowerLevels { + room: 50.into(), + }, + }, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.1 Join Rules + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomJoinRules, + content: serde_json::to_value(join_rules::JoinRulesEventContent::new( + join_rules::JoinRule::Invite, + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.2 History Visibility + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomHistoryVisibility, + content: serde_json::to_value( + history_visibility::HistoryVisibilityEventContent::new( + history_visibility::HistoryVisibility::Shared, + ), + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.3 Guest Access + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomGuestAccess, + content: serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::Forbidden, + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 6. Events implied by name and topic + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomName, + content: serde_json::to_value( + name::NameEventContent::new("Admin Room".to_owned()).map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") + })?, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTopic, + content: serde_json::to_value(topic::TopicEventContent { + topic: format!("Manage {}", db.globals.server_name()), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // Room alias + let alias: RoomAliasId = format!("#admins:{}", db.globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid alias name"); + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCanonicalAlias, + content: serde_json::to_value(canonical_alias::CanonicalAliasEventContent { + alias: Some(alias.clone()), + alt_aliases: Vec::new(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?; + + // Invite and join the real user + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &user_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + } + Ok(register::Response { access_token: Some(token), user_id, @@ -354,23 +618,20 @@ pub async fn deactivate_route( third_party_invite: None, }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } // Remove devices and mark account as deactivated diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 628045d..526e82f 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -108,22 +108,20 @@ pub async fn leave_room_route( event.membership = member::MembershipState::Leave; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(leave_room::Response::new().into()) } @@ -139,29 +137,27 @@ pub async fn invite_user_route( let sender_id = body.sender_id.as_ref().expect("user is authenticated"); if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user_id)?, - avatar_url: db.users.avatar_url(&user_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user_id)?, + avatar_url: db.users.avatar_url(&user_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(invite_user::Response.into()) } else { @@ -199,22 +195,20 @@ pub async fn kick_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; // TODO: reason - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(kick_user::Response::new().into()) } @@ -257,22 +251,20 @@ pub async fn ban_user_route( }, )?; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(ban_user::Response::new().into()) } @@ -306,22 +298,20 @@ pub async fn unban_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(unban_user::Response::new().into()) } @@ -640,6 +630,7 @@ async fn join_room_by_id_helper( &serde_json::to_value(&**pdu).expect("PDU is valid value"), &db.globals, &db.account_data, + &db.sending, )?; if state_events.contains(ev_id) { @@ -657,23 +648,20 @@ async fn join_room_by_id_helper( third_party_invite: None, }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } Ok(join_room_by_id::Response::new(room_id.clone()).into()) diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 5a4488f..c32bd68 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -49,29 +49,26 @@ pub async fn send_message_event_route( let mut unsigned = serde_json::Map::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into()); - let event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: body.content.event_type().into(), - content: serde_json::from_str( - body.json_body - .as_ref() - .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? - .get(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, - unsigned: Some(unsigned), - state_key: None, - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: body.content.event_type().into(), + content: serde_json::from_str( + body.json_body + .as_ref() + .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? + .get(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, + unsigned: Some(unsigned), + state_key: None, + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; db.transaction_ids .add_txnid(sender_id, device_id, &body.txn_id, event_id.as_bytes())?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 686d4c3..9c6bd51 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -31,43 +31,41 @@ pub async fn set_displayname_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - displayname: body.displayname.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + displayname: body.displayname.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( "Tried to send displayname update for user not in the room.", ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Presence update db.rooms.edus.update_presence( @@ -125,43 +123,41 @@ pub async fn set_avatar_url_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - avatar_url: body.avatar_url.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( - "Tried to send avatar url update for user not in the room.", - ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + avatar_url: body.avatar_url.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( + "Tried to send avatar url update for user not in the room.", + ) + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Presence update db.rooms.edus.update_presence( diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 24df8dd..b13cd80 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -18,26 +18,23 @@ pub async fn redact_event_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - let event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomRedaction, - content: serde_json::to_value(redaction::RedactionEventContent { - reason: body.reason.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: Some(body.event_id.clone()), - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomRedaction, + content: serde_json::to_value(redaction::RedactionEventContent { + reason: body.reason.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: Some(body.event_id.clone()), + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(redact_event::Response { event_id }.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index d21148b..28d30e2 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -53,47 +53,43 @@ pub async fn create_room_route( content.room_version = RoomVersionId::Version6; // 1. The room create event - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 2. Let the room creator join - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 3. Power levels let mut users = BTreeMap::new(); @@ -123,22 +119,20 @@ pub async fn create_room_route( }) .expect("event is valid, we just created it") }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomPowerLevels, - content: power_levels_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: power_levels_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 4. Events set by preset @@ -149,84 +143,76 @@ pub async fn create_room_route( }); // 4.1 Join Rules - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomJoinRules, - content: match preset { - create_room::RoomPreset::PublicChat => serde_json::to_value( - join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), - ) - .expect("event is valid, we just created it"), - // according to spec "invite" is the default - _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( - join_rules::JoinRule::Invite, - )) - .expect("event is valid, we just created it"), - }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; - - // 4.2 History Visibility - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomHistoryVisibility, - content: serde_json::to_value( - history_visibility::HistoryVisibilityEventContent::new( - history_visibility::HistoryVisibility::Shared, - ), + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomJoinRules, + content: match preset { + create_room::RoomPreset::PublicChat => serde_json::to_value( + join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), ) .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, + // according to spec "invite" is the default + _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( + join_rules::JoinRule::Invite, + )) + .expect("event is valid, we just created it"), }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.2 History Visibility + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomHistoryVisibility, + content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( + history_visibility::HistoryVisibility::Shared, + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 4.3 Guest Access - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomGuestAccess, - content: match preset { - create_room::RoomPreset::PublicChat => { - serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::Forbidden, - )) - .expect("event is valid, we just created it") - } - _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::CanJoin, + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomGuestAccess, + content: match preset { + create_room::RoomPreset::PublicChat => { + serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::Forbidden, )) - .expect("event is valid, we just created it"), - }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, + .expect("event is valid, we just created it") + } + _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::CanJoin, + )) + .expect("event is valid, we just created it"), }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 5. Events listed in initial_state for event in &body.initial_state { @@ -240,90 +226,82 @@ pub async fn create_room_route( continue; } - db.rooms - .build_and_append_pdu( - pdu_builder, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + pdu_builder, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } // 6. Events implied by name and topic if let Some(name) = &body.name { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomName, - content: serde_json::to_value( - name::NameEventContent::new(name.clone()).map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") - })?, - ) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomName, + content: serde_json::to_value( + name::NameEventContent::new(name.clone()).map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") + })?, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } if let Some(topic) = &body.topic { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTopic, - content: serde_json::to_value(topic::TopicEventContent { - topic: topic.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTopic, + content: serde_json::to_value(topic::TopicEventContent { + topic: topic.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } // 7. Events implied by invite (and TODO: invite_3pid) for user in &body.invite { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user)?, - avatar_url: db.users.avatar_url(&user)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user)?, + avatar_url: db.users.avatar_url(&user)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } // Homeserver specific stuff @@ -395,29 +373,24 @@ pub async fn upgrade_room_route( // Send a m.room.tombstone event to the old room to indicate that it is not intended to be used any further // Fail if the sender does not have the required permissions - let tombstone_event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTombstone, - content: serde_json::to_value( - ruma::events::room::tombstone::TombstoneEventContent { - body: "This room has been replaced".to_string(), - replacement_room: replacement_room.clone(), - }, - ) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let tombstone_event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTombstone, + content: serde_json::to_value(ruma::events::room::tombstone::TombstoneEventContent { + body: "This room has been replaced".to_string(), + replacement_room: replacement_room.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Get the old room federations status let federate = serde_json::from_value::>( @@ -444,48 +417,44 @@ pub async fn upgrade_room_route( create_event_content.room_version = new_version; create_event_content.predecessor = predecessor; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(create_event_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(create_event_content) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + )?; // Join the new room - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + )?; // Recommended transferable state events list from the specs let transferable_state_events = vec![ @@ -507,22 +476,20 @@ pub async fn upgrade_room_route( None => continue, // Skipping missing events. }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type, - content: event_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type, + content: event_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + )?; } // Moves any local aliases to the new room @@ -552,24 +519,21 @@ pub async fn upgrade_room_route( power_levels_event_content.invite = new_level; // Modify the power levels in the old room to prevent sending of events and inviting new users - let _ = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomPowerLevels, - content: serde_json::to_value(power_levels_event_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await; + let _ = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: serde_json::to_value(power_levels_event_content) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Return the replacement room id Ok(upgrade_room::Response { replacement_room }.into()) diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 46182a1..1e13b42 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -213,23 +213,20 @@ pub async fn send_state_event_for_key_helper( } } - let event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: content.event_type().into(), - content: json, - unsigned: None, - state_key, - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: content.event_type().into(), + content: json, + unsigned: None, + state_key, + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(event_id) } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 108edb5..ab05b39 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -10,7 +10,7 @@ use ruma::{ events::{ ignored_user_list, room::{ - member, + member, message, power_levels::{self, PowerLevelsEventContent}, }, EventType, @@ -440,6 +440,7 @@ impl Rooms { pdu_json: &serde_json::Value, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, + sending: &super::sending::Sending, ) -> Result> { self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?; @@ -452,7 +453,8 @@ impl Rooms { self.edus .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; - let mut pdu_id = pdu.room_id.as_bytes().to_vec(); + let room_id = pdu.room_id.clone(); + let mut pdu_id = room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&index.to_be_bytes()); @@ -503,6 +505,45 @@ impl Rooms { key.extend_from_slice(&pdu_id); self.tokenids.insert(key, &[])?; } + + if body.starts_with(&format!("@conduit:{}: ", globals.server_name())) + && self + .id_from_alias( + &format!("#admins:{}", globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid room alias"), + )? + .as_ref() + == Some(&pdu.room_id) + { + let mut parts = body.split_whitespace().skip(1); + if let Some(command) = parts.next() { + let args = parts.collect::>(); + + self.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMessage, + content: serde_json::to_value( + message::TextMessageEventContent { + body: format!("Command: {}, Args: {:?}", command, args), + formatted: None, + relates_to: None, + }, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: None, + }, + &UserId::try_from(format!("@conduit:{}", globals.server_name())) + .expect("@conduit:server_name is valid"), + &room_id, + &globals, + &sending, + &account_data, + )?; + } + } } } _ => {} @@ -570,7 +611,7 @@ impl Rooms { } /// Creates a new persisted data unit and adds it to a room. - pub async fn build_and_append_pdu( + pub fn build_and_append_pdu( &self, pdu_builder: PduBuilder, sender: &UserId, @@ -793,7 +834,7 @@ impl Rooms { .expect("json is object") .insert("event_id".to_owned(), pdu.event_id.to_string().into()); - let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data)?; + let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data, sending)?; self.append_to_state(&pdu_id, &pdu)?; diff --git a/src/database/users.rs b/src/database/users.rs index 2e26c1e..0d35e36 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -57,6 +57,11 @@ impl Users { Ok(()) } + /// Returns the number of users registered on this server. + pub fn count(&self) -> usize { + self.userid_password.iter().count() + } + /// Find out which user an access token belongs to. pub fn find_from_token(&self, token: &str) -> Result> { self.token_userdeviceid diff --git a/src/main.rs b/src/main.rs index fa1cc5c..8fb5fda 100644 --- a/src/main.rs +++ b/src/main.rs @@ -126,6 +126,7 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_public_rooms_filtered_route, server_server::send_transaction_message_route, server_server::get_missing_events_route, + server_server::get_profile_information_route, ], ) .attach(AdHoc::on_attach("Config", |mut rocket| async { diff --git a/src/server_server.rs b/src/server_server.rs index 2d52c4a..0c175bf 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,4 +1,5 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; +use get_profile_information::v1::ProfileField; use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::warn; use rocket::{get, post, put, response::content::Json, State}; @@ -10,6 +11,7 @@ use ruma::{ get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, }, event::get_missing_events, + query::get_profile_information, transactions::send_transaction_message, }, OutgoingRequest, @@ -362,9 +364,9 @@ pub fn send_transaction_message_route<'a>( let pdu = serde_json::from_value::(value.clone()) .expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - let pdu_id = db - .rooms - .append_pdu(&pdu, &value, &db.globals, &db.account_data)?; + let pdu_id = + db.rooms + .append_pdu(&pdu, &value, &db.globals, &db.account_data, &db.sending)?; db.rooms.append_to_state(&pdu_id, &pdu)?; } } @@ -416,3 +418,59 @@ pub fn get_missing_events_route<'a>( Ok(get_missing_events::v1::Response { events }.into()) } + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/federation/v1/query/profile", data = "") +)] +pub fn get_profile_information_route<'a>( + db: State<'a, Database>, + body: Ruma>, +) -> ConduitResult { + let mut displayname = None; + let mut avatar_url = None; + + match body.field { + Some(ProfileField::DisplayName) => displayname = db.users.displayname(&body.user_id)?, + Some(ProfileField::AvatarUrl) => avatar_url = db.users.avatar_url(&body.user_id)?, + None => { + displayname = db.users.displayname(&body.user_id)?; + avatar_url = db.users.avatar_url(&body.user_id)?; + } + } + + Ok(get_profile_information::v1::Response { + displayname, + avatar_url, + } + .into()) +} + +/* +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/federation/v2/invite/<_>/<_>", data = "") +)] +pub fn get_user_devices_route<'a>( + db: State<'a, Database>, + body: Ruma>, +) -> ConduitResult { + let mut displayname = None; + let mut avatar_url = None; + + match body.field { + Some(ProfileField::DisplayName) => displayname = db.users.displayname(&body.user_id)?, + Some(ProfileField::AvatarUrl) => avatar_url = db.users.avatar_url(&body.user_id)?, + None => { + displayname = db.users.displayname(&body.user_id)?; + avatar_url = db.users.avatar_url(&body.user_id)?; + } + } + + Ok(get_profile_information::v1::Response { + displayname, + avatar_url, + } + .into()) +} +*/ From c15ae3c126020c3826f15e5f50ca70a669fc1402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 6 Oct 2020 20:43:35 +0200 Subject: [PATCH 46/48] fix: invalid typing bytes because of 0xff in numbers --- src/database/rooms/edus.rs | 9 ++++++--- src/database/sending.rs | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index a794c69..29f5407 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -11,6 +11,7 @@ use ruma::{ use std::{ collections::HashMap, convert::{TryFrom, TryInto}, + mem, }; #[derive(Clone)] @@ -228,9 +229,11 @@ impl RoomEdus { let key = key?; Ok::<_, Error>(( key.clone(), - utils::u64_from_bytes(key.split(|&b| b == 0xff).nth(1).ok_or_else(|| { - Error::bad_database("RoomTyping has invalid timestamp or delimiters.") - })?) + utils::u64_from_bytes( + &key.splitn(2, |&b| b == 0xff).nth(1).ok_or_else(|| { + Error::bad_database("RoomTyping has invalid timestamp or delimiters.") + })?[0..mem::size_of::()], + ) .map_err(|_| Error::bad_database("RoomTyping has invalid timestamp bytes."))?, )) }) diff --git a/src/database/sending.rs b/src/database/sending.rs index 1ed94cc..c818cbf 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -28,6 +28,7 @@ impl Sending { select! { Some(server) = futures.next() => { warn!("response: {:?}", &server); + warn!("futures left: {}", &futures.len()); match server { Ok((server, _response)) => { waiting_servers.remove(&server) From 6afc4c9b3e066f2d071e8420c9e4111d0dc65d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 6 Oct 2020 21:04:51 +0200 Subject: [PATCH 47/48] feat: federation disabled by default It can be enable in the Rocket.toml config or using ROCKET_FEDERATION_ENABLED=true --- DEPLOY_FROM_SOURCE.md | 3 +++ Rocket-example.toml | 2 ++ docker-compose.yml | 1 + src/database/globals.rs | 6 ++++++ src/server_server.rs | 39 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/DEPLOY_FROM_SOURCE.md b/DEPLOY_FROM_SOURCE.md index 4d685f6..456fe6e 100644 --- a/DEPLOY_FROM_SOURCE.md +++ b/DEPLOY_FROM_SOURCE.md @@ -27,7 +27,10 @@ Environment="ROCKET_SERVER_NAME=YOURSERVERNAME.HERE" # EDIT THIS Environment="ROCKET_PORT=14004" # Reverse proxy port +#Environment="ROCKET_MAX_REQUEST_SIZE=20000000" # in bytes #Environment="ROCKET_REGISTRATION_DISABLED=true" +#Environment="ROCKET_ENCRYPTION_DISABLED=true" +#Environment="ROCKET_FEDERATION_ENABLED=true" #Environment="ROCKET_LOG=normal" # Detailed logging Environment="ROCKET_ENV=production" diff --git a/Rocket-example.toml b/Rocket-example.toml index 41b36d3..8eb48e9 100644 --- a/Rocket-example.toml +++ b/Rocket-example.toml @@ -16,6 +16,8 @@ port = 14004 # Note: existing rooms will continue to work #encryption_disabled = true +#federation_enabled = true + # Default path is in this user's data #database_path = "/home/timo/MyConduitServer" diff --git a/docker-compose.yml b/docker-compose.yml index f06eaca..7d19762 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: # ROCKET_PORT: 8000 # ROCKET_REGISTRATION_DISABLED: 'true' # ROCKET_ENCRYPTION_DISABLED: 'true' + # ROCKET_FEDERATION_ENABLED: 'true' # ROCKET_DATABASE_PATH: /srv/conduit/.local/share/conduit # ROCKET_WORKERS: 10 # ROCKET_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB diff --git a/src/database/globals.rs b/src/database/globals.rs index 8ce9c01..37f10ee 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -14,6 +14,7 @@ pub struct Globals { max_request_size: u32, registration_disabled: bool, encryption_disabled: bool, + federation_enabled: bool, } impl Globals { @@ -69,6 +70,7 @@ impl Globals { .map_err(|_| Error::BadConfig("Invalid max_request_size."))?, registration_disabled: config.get_bool("registration_disabled").unwrap_or(false), encryption_disabled: config.get_bool("encryption_disabled").unwrap_or(false), + federation_enabled: config.get_bool("federation_enabled").unwrap_or(false), }) } @@ -114,4 +116,8 @@ impl Globals { pub fn encryption_disabled(&self) -> bool { self.encryption_disabled } + + pub fn federation_enabled(&self) -> bool { + self.federation_enabled + } } diff --git a/src/server_server.rs b/src/server_server.rs index 0c175bf..79976c0 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -57,6 +57,10 @@ pub async fn send_request( where T: Debug, { + if !globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let resolver = AsyncResolver::tokio_from_system_conf() .await .map_err(|_| Error::BadConfig("Failed to set up trust dns resolver with system config."))?; @@ -204,7 +208,11 @@ where } #[cfg_attr(feature = "conduit_bin", get("/_matrix/federation/v1/version"))] -pub fn get_server_version() -> ConduitResult { +pub fn get_server_version(db: State<'_, Database>) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + Ok(get_server_version::Response { server: Some(get_server_version::Server { name: Some("Conduit".to_owned()), @@ -216,6 +224,11 @@ pub fn get_server_version() -> ConduitResult { #[cfg_attr(feature = "conduit_bin", get("/_matrix/key/v2/server"))] pub fn get_server_keys(db: State<'_, Database>) -> Json { + if !db.globals.federation_enabled() { + // TODO: Use proper types + return Json("Federation is disabled.".to_owned()); + } + let mut verify_keys = BTreeMap::new(); verify_keys.insert( format!("ed25519:{}", db.globals.keypair().version()), @@ -259,6 +272,10 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let response = client_server::get_public_rooms_filtered_helper( &db, None, @@ -302,6 +319,10 @@ pub async fn get_public_rooms_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let response = client_server::get_public_rooms_filtered_helper( &db, None, @@ -345,6 +366,10 @@ pub fn send_transaction_message_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + //dbg!(&*body); for pdu in &body.pdus { let mut value = serde_json::from_str(pdu.json().get()) @@ -384,6 +409,10 @@ pub fn get_missing_events_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let mut queued_events = body.latest_events.clone(); let mut events = Vec::new(); @@ -427,6 +456,10 @@ pub fn get_profile_information_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let mut displayname = None; let mut avatar_url = None; @@ -455,6 +488,10 @@ pub fn get_user_devices_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let mut displayname = None; let mut avatar_url = None; From 304c53c4f53397a251e567a891c54ece7835173f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 7 Oct 2020 12:29:19 +0200 Subject: [PATCH 48/48] style: make clippy happier --- src/client_server/room.rs | 8 ++------ src/client_server/sync.rs | 2 +- src/database/sending.rs | 2 +- src/pdu.rs | 4 ++-- src/server_server.rs | 2 +- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 28d30e2..744d949 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -354,12 +354,8 @@ pub async fn upgrade_room_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - // Validate the room version requested - let new_version = - RoomVersionId::try_from(body.new_version.clone()).expect("invalid room version id"); - if !matches!( - new_version, + body.new_version, RoomVersionId::Version5 | RoomVersionId::Version6 ) { return Err(Error::BadRequest( @@ -414,7 +410,7 @@ pub async fn upgrade_room_route( let mut create_event_content = ruma::events::room::create::CreateEventContent::new(sender_id.clone()); create_event_content.federate = federate; - create_event_content.room_version = new_version; + create_event_content.room_version = body.new_version.clone(); create_event_content.predecessor = predecessor; db.rooms.build_and_append_pdu( diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index aec03af..6f41160 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -477,7 +477,7 @@ pub async fn sync_events_route( }) .and_then(|pdu| { serde_json::from_value::>( - pdu.content.clone(), + pdu.content, ) .expect("Raw::from_value always works") .deserialize() diff --git a/src/database/sending.rs b/src/database/sending.rs index c818cbf..24a783b 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -84,7 +84,7 @@ impl Sending { (Box, send_transaction_message::v1::Response), (Box, Error), > { - let pdu_json = PduEvent::to_outgoing_federation_event( + let pdu_json = PduEvent::convert_to_outgoing_federation_event( rooms .get_pdu_json_from_id(&pdu_id) .map_err(|e| (server.clone(), e))? diff --git a/src/pdu.rs b/src/pdu.rs index 4b1df4b..7118bfc 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -199,7 +199,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - pub fn to_outgoing_federation_event( + pub fn convert_to_outgoing_federation_event( mut pdu_json: serde_json::Value, ) -> Raw { if let Some(unsigned) = pdu_json @@ -239,7 +239,7 @@ impl From<&state_res::StateEvent> for PduEvent { content: pdu.content().clone(), state_key: Some(pdu.state_key()), prev_events: pdu.prev_event_ids(), - depth: pdu.depth().clone(), + depth: *pdu.depth(), auth_events: pdu.auth_events(), redacts: pdu.redacts().cloned(), unsigned: pdu.unsigned().clone().into_iter().collect(), diff --git a/src/server_server.rs b/src/server_server.rs index 79976c0..462d636 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -438,7 +438,7 @@ pub fn get_missing_events_route<'a>( ) .map_err(|_| Error::bad_database("Invalid prev_events content in pdu in db."))?, ); - events.push(PduEvent::to_outgoing_federation_event(pdu)); + events.push(PduEvent::convert_to_outgoing_federation_event(pdu)); } i += 1; }