From 32da76b9a21ca41a39f521419d3f29d6e672e5c4 Mon Sep 17 00:00:00 2001 From: timokoesters Date: Thu, 4 Jun 2020 18:27:57 +0200 Subject: [PATCH 1/3] feat: heroes, don't send notifications every time --- src/client_server.rs | 109 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 91 insertions(+), 18 deletions(-) diff --git a/src/client_server.rs b/src/client_server.rs index 07f1911..b20e7af 100644 --- a/src/client_server.rs +++ b/src/client_server.rs @@ -24,8 +24,8 @@ use ruma_client_api::{ keys::{self, claim_keys, get_keys, upload_keys}, media::{create_content, get_content, get_content_thumbnail, get_media_config}, membership::{ - forget_room, get_member_events, invite_user, join_room_by_id, join_room_by_id_or_alias, - kick_user, leave_room, ban_user, unban_user, + ban_user, forget_room, get_member_events, invite_user, join_room_by_id, + join_room_by_id_or_alias, kick_user, leave_room, unban_user, }, message::{create_message_event, get_message_events}, presence::set_presence, @@ -2151,7 +2151,9 @@ pub fn sync_route( let mut send_member_count = false; let mut send_full_state = false; + let mut send_notification_counts = false; for pdu in &pdus { + send_notification_counts = true; if pdu.kind == EventType::RoomMember { send_member_count = true; if !send_full_state && pdu.state_key == Some(user_id.to_string()) { @@ -2171,7 +2173,85 @@ pub fn sync_route( } } - let notification_count = + let state = db.rooms.room_state(&room_id).unwrap(); + + let (joined_member_count, invited_member_count, heroes) = if send_member_count { + let joined_member_count = db.rooms.room_members(&room_id).count(); + let invited_member_count = db.rooms.room_members_invited(&room_id).count(); + + // Recalculate heroes (first 5 members) + let mut heroes = Vec::new(); + + if joined_member_count + invited_member_count <= 5 { + // Go through all PDUs and for each member event, check if the user is still joined or + // invited until we have 5 or we reach the end + + for hero in db + .rooms + .all_pdus(&room_id) + .unwrap() + .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus + .filter(|pdu| pdu.kind == EventType::RoomMember) + .filter_map(|pdu| { + let content = serde_json::from_value::< + EventJson, + >(pdu.content.clone()) + .unwrap() + .deserialize() + .unwrap(); + + let current_content = serde_json::from_value::< + EventJson, + >( + state + .get(&( + EventType::RoomMember, + pdu.state_key.clone().expect( + "TODO: error handling. Is it really a state event?", + ), + )) + .expect("a user that joined once will always have a member event") + .content + .clone(), + ) + .unwrap() + .deserialize() + .unwrap(); + + // The membership was and still is invite or join + if matches!( + content.membership, + ruma_events::room::member::MembershipState::Join + | ruma_events::room::member::MembershipState::Invite + ) && matches!( + current_content.membership, + ruma_events::room::member::MembershipState::Join + | ruma_events::room::member::MembershipState::Invite + ) { + Some(pdu.state_key.unwrap()) + } else { + None + } + }) + { + if heroes.contains(&hero) || hero == user_id.to_string() { + continue; + } + + heroes.push(hero); + } + } + + ( + Some(joined_member_count), + Some(invited_member_count), + heroes, + ) + } else { + (None, None, Vec::new()) + }; + + let notification_count = if send_notification_counts { if let Some(last_read) = db.rooms.edus.room_read_get(&room_id, &user_id).unwrap() { Some( (db.rooms @@ -2188,7 +2268,10 @@ pub fn sync_route( ) } else { None - }; + } + } else { + None + }; // They /sync response doesn't always return all messages, so we say the output is // limited unless there are enough events @@ -2247,17 +2330,9 @@ pub fn sync_route( .collect(), }), summary: sync_events::RoomSummary { - heroes: Vec::new(), - joined_member_count: if send_member_count { - Some((db.rooms.room_members(&room_id).count() as u32).into()) - } else { - None - }, - invited_member_count: if send_member_count { - Some((db.rooms.room_members_invited(&room_id).count() as u32).into()) - } else { - None - }, + heroes, + joined_member_count: joined_member_count.map(|n| (n as u32).into()), + invited_member_count: invited_member_count.map(|n| (n as u32).into()), }, unread_notifications: sync_events::UnreadNotificationsCount { highlight_count: None, @@ -2271,9 +2346,7 @@ pub fn sync_route( // TODO: state before timeline state: sync_events::State { events: if send_full_state { - db.rooms - .room_state(&room_id) - .unwrap() + state .into_iter() .map(|(_, pdu)| pdu.to_state_event()) .collect() From d404f902bf3bbd190a55c9ec2523cda3149b5e59 Mon Sep 17 00:00:00 2001 From: timokoesters Date: Thu, 4 Jun 2020 22:36:48 +0200 Subject: [PATCH 2/3] fix: send presence too often --- src/client_server.rs | 47 ++++++++++++++----------------------- src/database.rs | 3 +-- src/database/global_edus.rs | 46 +++++++++++++----------------------- src/database/rooms/edus.rs | 14 +---------- 4 files changed, 36 insertions(+), 74 deletions(-) diff --git a/src/client_server.rs b/src/client_server.rs index b20e7af..a7f8093 100644 --- a/src/client_server.rs +++ b/src/client_server.rs @@ -548,9 +548,8 @@ pub fn set_displayname_route( // Presence update db.global_edus - .update_globallatest( - &user_id, - EduEvent::Presence(ruma_events::presence::PresenceEvent { + .update_presence( + ruma_events::presence::PresenceEvent { content: ruma_events::presence::PresenceEventContent { avatar_url: db.users.avatar_url(&user_id).unwrap(), currently_active: None, @@ -560,7 +559,7 @@ pub fn set_displayname_route( status_msg: None, }, sender: user_id.clone(), - }), + }, &db.globals, ) .unwrap(); @@ -640,9 +639,8 @@ pub fn set_avatar_url_route( // Presence update db.global_edus - .update_globallatest( - &user_id, - EduEvent::Presence(ruma_events::presence::PresenceEvent { + .update_presence( + ruma_events::presence::PresenceEvent { content: ruma_events::presence::PresenceEventContent { avatar_url: db.users.avatar_url(&user_id).unwrap(), currently_active: None, @@ -652,7 +650,7 @@ pub fn set_avatar_url_route( status_msg: None, }, sender: user_id.clone(), - }), + }, &db.globals, ) .unwrap(); @@ -707,9 +705,8 @@ pub fn set_presence_route( let user_id = body.user_id.as_ref().expect("user is authenticated"); db.global_edus - .update_globallatest( - &user_id, - EduEvent::Presence(ruma_events::presence::PresenceEvent { + .update_presence( + ruma_events::presence::PresenceEvent { content: ruma_events::presence::PresenceEventContent { avatar_url: db.users.avatar_url(&user_id).unwrap(), currently_active: None, @@ -719,7 +716,7 @@ pub fn set_presence_route( status_msg: body.status_msg.clone(), }, sender: user_id.clone(), - }), + }, &db.globals, ) .unwrap(); @@ -2435,24 +2432,16 @@ pub fn sync_route( presence: sync_events::Presence { events: db .global_edus - .globallatests_since(since) + .presence_since(since) .unwrap() - .filter_map(|edu| { - // Only look for presence events - if let Ok(mut edu) = EventJson::::from( - edu.unwrap().into_json(), - ) - .deserialize() - { - let timestamp = edu.content.last_active_ago.unwrap(); - edu.content.last_active_ago = Some( - js_int::UInt::try_from(utils::millis_since_unix_epoch()).unwrap() - - timestamp, - ); - Some(edu.into()) - } else { - None - } + .map(|edu| { + let mut edu = edu.unwrap().deserialize().unwrap(); + let timestamp = edu.content.last_active_ago.unwrap(); + let last_active_ago = js_int::UInt::try_from(utils::millis_since_unix_epoch()) + .unwrap() + - timestamp; + edu.content.last_active_ago = Some(last_active_ago); + edu.into() }) .collect(), }, diff --git a/src/database.rs b/src/database.rs index 7be0dc7..dc78ba9 100644 --- a/src/database.rs +++ b/src/database.rs @@ -94,8 +94,7 @@ impl Database { roomuserdataid_accountdata: db.open_tree("roomuserdataid_accountdata").unwrap(), }, global_edus: global_edus::GlobalEdus { - //globalallid_globalall: db.open_tree("globalallid_globalall").unwrap(), - globallatestid_globallatest: db.open_tree("globallatestid_globallatest").unwrap(), // Presence + presenceid_presence: db.open_tree("presenceid_presence").unwrap(), // Presence }, media: media::Media { mediaid_file: db.open_tree("mediaid_file").unwrap(), diff --git a/src/database/global_edus.rs b/src/database/global_edus.rs index f665260..5f7491b 100644 --- a/src/database/global_edus.rs +++ b/src/database/global_edus.rs @@ -1,67 +1,53 @@ use crate::Result; -use ruma_events::{collections::only::Event as EduEvent, EventJson}; -use ruma_identifiers::UserId; +use ruma_events::EventJson; pub struct GlobalEdus { //pub globalallid_globalall: sled::Tree, // ToDevice, GlobalAllId = UserId + Count - pub(super) globallatestid_globallatest: sled::Tree, // Presence, GlobalLatestId = Count + UserId + pub(super) presenceid_presence: sled::Tree, // Presence, PresenceId = Count + UserId } impl GlobalEdus { /// Adds a global event which will be saved until a new event replaces it (e.g. presence updates). - pub fn update_globallatest( + pub fn update_presence( &self, - user_id: &UserId, - event: EduEvent, + presence: ruma_events::presence::PresenceEvent, globals: &super::globals::Globals, ) -> Result<()> { // Remove old entry if let Some(old) = self - .globallatestid_globallatest + .presenceid_presence .iter() .keys() .rev() .filter_map(|r| r.ok()) .find(|key| { - key.rsplit(|&b| b == 0xff).next().unwrap() == user_id.to_string().as_bytes() + key.rsplit(|&b| b == 0xff).next().unwrap() == presence.sender.to_string().as_bytes() }) { // This is the old global_latest - self.globallatestid_globallatest.remove(old)?; + self.presenceid_presence.remove(old)?; } - let mut global_latest_id = globals.next_count()?.to_be_bytes().to_vec(); - global_latest_id.push(0xff); - global_latest_id.extend_from_slice(&user_id.to_string().as_bytes()); + let mut presence_id = globals.next_count()?.to_be_bytes().to_vec(); + presence_id.push(0xff); + presence_id.extend_from_slice(&presence.sender.to_string().as_bytes()); - self.globallatestid_globallatest - .insert(global_latest_id, &*serde_json::to_string(&event)?)?; + self.presenceid_presence + .insert(presence_id, &*serde_json::to_string(&presence)?)?; Ok(()) } /// Returns an iterator over the most recent presence updates that happened after the event with id `since`. - pub fn globallatests_since( + pub fn presence_since( &self, since: u64, - ) -> Result>>> { - let first_possible_edu = since.to_be_bytes().to_vec(); + ) -> Result>>> { + let first_possible_edu = (since + 1).to_be_bytes().to_vec(); // +1 so we don't send the event at since Ok(self - .globallatestid_globallatest + .presenceid_presence .range(&*first_possible_edu..) - // Skip the first pdu if it's exactly at since, because we sent that last time - .skip( - if self - .globallatestid_globallatest - .get(first_possible_edu)? - .is_some() - { - 1 - } else { - 0 - }, - ) .filter_map(|r| r.ok()) .map(|(_, v)| Ok(serde_json::from_slice(&v)?))) } diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index 0519b43..a3fa2bc 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -59,23 +59,11 @@ impl RoomEdus { prefix.push(0xff); let mut first_possible_edu = prefix.clone(); - first_possible_edu.extend_from_slice(&since.to_be_bytes()); + first_possible_edu.extend_from_slice(&(since + 1).to_be_bytes()); // +1 so we don't send the event at since Ok(self .roomlatestid_roomlatest .range(&*first_possible_edu..) - // Skip the first pdu if it's exactly at since, because we sent that last time - .skip( - if self - .roomlatestid_roomlatest - .get(first_possible_edu)? - .is_some() - { - 1 - } else { - 0 - }, - ) .filter_map(|r| r.ok()) .take_while(move |(k, _)| k.starts_with(&prefix)) .map(|(_, v)| Ok(serde_json::from_slice(&v)?))) From 08d19c3e0bcdb4ca4cd6972501fac92a5fdca398 Mon Sep 17 00:00:00 2001 From: timokoesters Date: Fri, 5 Jun 2020 09:04:51 +0200 Subject: [PATCH 3/3] fix: broken dependencies --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- Cargo.toml | 13 +++++++------ 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 365781d..bd8af81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,9 +26,9 @@ checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" [[package]] name = "async-trait" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb7f9ad01405feb3c1dac82463038945cf88eea4569acaf3ad662233496dd96" +checksum = "8f1c13101a3224fb178860ae372a031ce350bbd92d39968518f016744dde0bf7" dependencies = [ "proc-macro2 1.0.18", "quote 1.0.6", @@ -163,7 +163,7 @@ dependencies = [ "ruma-api", "ruma-client-api", "ruma-common", - "ruma-events 0.21.3 (git+https://github.com/ruma/ruma-events?rev=7395f94)", + "ruma-events 0.21.3 (git+https://github.com/ruma/ruma-events?rev=c1ee72d)", "ruma-federation-api", "ruma-identifiers", "ruma-signatures", @@ -989,18 +989,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc93aeee735e60ecb40cf740eb319ff23eab1c5748abfdb5c180e4ce49f7791" +checksum = "ba3a1acf4a3e70849f8a673497ef984f043f95d2d8252dcdf74d54e6a1e47e8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e58db2081ba5b4c93bd6be09c40fd36cb9193a8336c384f3b40012e531aa7e40" +checksum = "194e88048b71a3e02eb4ee36a6995fed9b8236c11a7bb9f7247a9d9835b3f265" dependencies = [ "proc-macro2 1.0.18", "quote 1.0.6", @@ -1009,9 +1009,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9df32da11d84f3a7d70205549562966279adb900e080fad3dccd8e64afccf0ad" +checksum = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715" [[package]] name = "pin-utils" @@ -1160,11 +1160,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.4" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b81e49ddec5109a9dcfc5f2a317ff53377c915e9ae9d4f2fb50914b85614e2" +checksum = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" dependencies = [ - "base64 0.11.0", + "base64 0.12.1", "bytes", "encoding_rs", "futures-core", @@ -1183,7 +1183,6 @@ dependencies = [ "pin-project-lite", "serde", "serde_urlencoded", - "time", "tokio", "tokio-tls", "url", @@ -1297,13 +1296,13 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.9.0" -source = "git+https://github.com/ruma/ruma-client-api.git?rev=c2c5a3cea01b0544e5adb40f7ddae828627afd2c#c2c5a3cea01b0544e5adb40f7ddae828627afd2c" +source = "git+https://github.com/ruma/ruma-client-api.git?rev=632eb9d520028816c5fb7224bd0aca8d1e3793f1#632eb9d520028816c5fb7224bd0aca8d1e3793f1" dependencies = [ "http", "js_int", "ruma-api", "ruma-common", - "ruma-events 0.21.3 (git+https://github.com/ruma/ruma-events?rev=7395f94)", + "ruma-events 0.21.3 (git+https://github.com/ruma/ruma-events?rev=c1ee72d)", "ruma-identifiers", "ruma-serde", "serde", @@ -1313,24 +1312,25 @@ dependencies = [ [[package]] name = "ruma-common" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "253416d67b4bde281f2781424232a58a946a4f1c451d5f857a8d0705d58eaf2a" +checksum = "6cb49e83277e82c69cc258cedc7e68b3d72ba378f1cb6105cbfcc8831e422b4d" dependencies = [ "matches", "ruma-serde", "serde", "serde_json", + "strum", ] [[package]] name = "ruma-events" version = "0.21.3" -source = "git+https://github.com/ruma/ruma-events?rev=7395f94#7395f940a7cf70c1598223570fb2b731a6a41707" +source = "git+https://github.com/ruma/ruma-events?rev=c1ee72d#c1ee72db0f3107a97f6a4273a0ea3fed5c4c30e2" dependencies = [ "js_int", "ruma-common", - "ruma-events-macros 0.21.3 (git+https://github.com/ruma/ruma-events?rev=7395f94)", + "ruma-events-macros 0.21.3 (git+https://github.com/ruma/ruma-events?rev=c1ee72d)", "ruma-identifiers", "ruma-serde", "serde", @@ -1356,7 +1356,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.21.3" -source = "git+https://github.com/ruma/ruma-events?rev=7395f94#7395f940a7cf70c1598223570fb2b731a6a41707" +source = "git+https://github.com/ruma/ruma-events?rev=c1ee72d#c1ee72db0f3107a97f6a4273a0ea3fed5c4c30e2" dependencies = [ "proc-macro2 1.0.18", "quote 1.0.6", @@ -2019,9 +2019,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winreg" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ "winapi 0.3.8", ] diff --git a/Cargo.toml b/Cargo.toml index 7f7ba5b..1bca0a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,14 +12,16 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -rocket = { git = "https://github.com/SergioBenitez/Rocket.git", branch = "async", features = ["tls"] } -http = "0.2.1" -ruma-client-api = { git = "https://github.com/ruma/ruma-client-api.git", rev = "c2c5a3cea01b0544e5adb40f7ddae828627afd2c" } +ruma-client-api = { git = "https://github.com/ruma/ruma-client-api.git", rev = "632eb9d520028816c5fb7224bd0aca8d1e3793f1" } ruma-identifiers = { version = "0.16.2", features = ["rand"] } ruma-api = "0.16.1" -ruma-events = { git = "https://github.com/ruma/ruma-events.git", rev = "7395f94" } +ruma-events = { git = "https://github.com/ruma/ruma-events.git", rev = "c1ee72d" } ruma-signatures = { git = "https://github.com/ruma/ruma-signatures.git", rev = "1ca545cba8dfd43e0fc8e3c18e1311fb73390a97" } ruma-federation-api = { git = "https://github.com/ruma/ruma-federation-api.git", rev = "4cf4aa6ef74b25ad8c14d99d7774129f023df163" } +ruma-common = "0.1.3" + +rocket = { git = "https://github.com/SergioBenitez/Rocket.git", branch = "async", features = ["tls"] } +http = "0.2.1" log = "0.4.8" sled = "0.31.0" directories = "2.0.2" @@ -29,8 +31,7 @@ serde = "1.0.111" tokio = { version = "0.2.21", features = ["macros"] } rand = "0.7.3" rust-argon2 = "0.8.2" -reqwest = "=0.10.4" +reqwest = "0.10.6" base64 = "0.12.1" thiserror = "1.0.19" -ruma-common = "0.1.2" image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] }