From 7889da2b302cb43f9fa845e7b28ae3e2d809047a Mon Sep 17 00:00:00 2001 From: Devin R Date: Thu, 16 Apr 2020 10:02:59 -0400 Subject: [PATCH] state_store: very rough draft of json store --- Cargo.toml | 8 ++- design.md | 4 +- src/error.rs | 9 +++ src/models/room.rs | 8 +-- src/models/room_member.rs | 19 +++++- src/session.rs | 4 +- src/state/mod.rs | 139 +++++++++++++++++++++++++++----------- src/state/state_store.rs | 139 ++++++++++++++++++++++++++++++++++++-- 8 files changed, 273 insertions(+), 57 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0d76e75a..c160a20d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,15 +12,18 @@ version = "0.1.0" [features] default = [] -encryption = ["olm-rs", "serde/derive", "serde_json", "cjson", "zeroize"] +encryption = ["olm-rs", "serde/derive", "cjson", "zeroize"] sqlite-cryptostore = ["sqlx", "zeroize"] [dependencies] +dirs = "2.0.2" futures = "0.3.4" reqwest = "0.10.4" http = "0.2.1" url = "2.1.1" async-trait = "0.1.30" +serde = "1.0.106" +serde_json = "1.0.51" # Ruma dependencies js_int = "0.1.4" @@ -32,8 +35,6 @@ uuid = { version = "0.8.1", features = ["v4"] } # Dependencies for the encryption support olm-rs = { git = "https://gitlab.gnome.org/poljar/olm-rs", optional = true, features = ["serde"]} -serde = { version = "1.0.106", optional = true, features = ["derive"] } -serde_json = { version = "1.0.51", optional = true } cjson = { version = "0.1.0", optional = true } zeroize = { version = "1.1.0", optional = true, features = ["zeroize_derive"] } @@ -65,3 +66,4 @@ serde_json = "1.0.51" tracing-subscriber = "0.2.4" tempfile = "3.1.0" mockito = "0.25.1" +lazy_static = "1.4.0" diff --git a/design.md b/design.md index 7b7dae99..43e9d7d8 100644 --- a/design.md +++ b/design.md @@ -11,7 +11,7 @@ The highest level structure that ties the other pieces of functionality together - make raw Http requests #### Base Client/Client State Machine -In addition to Http the `AsyncClient` passes along methods from the `BaseClient` that deal with `Room`s and `RoomMember`s. This allows the client to keep track of more complicated information that needs to be calculated in some way. +In addition to Http, the `AsyncClient` passes along methods from the `BaseClient` that deal with `Room`s and `RoomMember`s. This allows the client to keep track of more complicated information that needs to be calculated in some way. - human readable room names - power level? - ignored list? @@ -87,7 +87,7 @@ pub struct RoomMember { ``` #### State Store -The `BaseClient` also has access to a `dyn StateStore` this is an abstraction around a "database" to keep client state without requesting a full sync from the server on start up. A default implementation that serializes/deserializes json to files in a specified directory can be used. The user can also implement `StateStore` to fit any storage solution they choose. +The `BaseClient` also has access to a `dyn StateStore` this is an abstraction around a "database" to keep the client state without requesting a full sync from the server on startup. A default implementation that serializes/deserializes JSON to files in a specified directory can be used. The user can also implement `StateStore` to fit any storage solution they choose. - load - store/save - update ?? diff --git a/src/error.rs b/src/error.rs index 817baae9..33e119a6 100644 --- a/src/error.rs +++ b/src/error.rs @@ -15,10 +15,13 @@ //! Error conditions. +use std::io::Error as IoError; + use reqwest::Error as ReqwestError; use ruma_api::error::FromHttpResponseError as RumaResponseError; use ruma_api::error::IntoHttpError as RumaIntoHttpError; use ruma_client_api::Error as RumaClientError; +use serde_json::Error as JsonError; use thiserror::Error; use url::ParseError; @@ -46,6 +49,12 @@ pub enum Error { /// An error converting between ruma_client_api types and Hyper types. #[error("can't convert between ruma_client_api and hyper types.")] IntoHttp(RumaIntoHttpError), + /// An error de/serializing type for the `StateStore` + #[error(transparent)] + SerdeJson(#[from] JsonError), + /// An error de/serializing type for the `StateStore` + #[error(transparent)] + IoError(#[from] IoError), #[cfg(feature = "encryption")] /// An error occured durring a E2EE operation. #[error(transparent)] diff --git a/src/models/room.rs b/src/models/room.rs index 9701d04a..21ce143b 100644 --- a/src/models/room.rs +++ b/src/models/room.rs @@ -32,8 +32,8 @@ use crate::events::EventType; use crate::identifiers::{RoomAliasId, RoomId, UserId}; use js_int::{Int, UInt}; - -#[derive(Debug, Default)] +use serde::{Deserialize, Serialize}; +#[derive(Debug, Default, PartialEq, Serialize, Deserialize)] /// `RoomName` allows the calculation of a text room name. pub struct RoomName { /// The displayed name of the room. @@ -44,7 +44,7 @@ pub struct RoomName { aliases: Vec, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct PowerLevels { /// The level required to ban a user. pub ban: Int, @@ -70,7 +70,7 @@ pub struct PowerLevels { pub notifications: Int, } -#[derive(Debug)] +#[derive(Debug, PartialEq, Serialize, Deserialize)] /// A Matrix rooom. pub struct Room { /// The unique id of the room. diff --git a/src/models/room_member.rs b/src/models/room_member.rs index e1211112..fb784653 100644 --- a/src/models/room_member.rs +++ b/src/models/room_member.rs @@ -24,10 +24,10 @@ use crate::events::room::{ use crate::identifiers::UserId; use js_int::{Int, UInt}; - +use serde::{Deserialize, Serialize}; // Notes: if Alice invites Bob into a room we will get an event with the sender as Alice and the state key as Bob. -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] /// A Matrix room member. /// pub struct RoomMember { @@ -58,11 +58,26 @@ pub struct RoomMember { /// The human readable name of this room member. pub name: String, /// The events that created the state of this room member. + #[serde(skip)] pub events: Vec, /// The `PresenceEvent`s connected to this user. + #[serde(skip)] pub presence_events: Vec, } +impl PartialEq for RoomMember { + fn eq(&self, other: &RoomMember) -> bool { + // TODO check everything but events and presence_events they dont impl PartialEq + self.room_id == other.room_id + && self.user_id == other.user_id + && self.name == other.name + && self.display_name == other.display_name + && self.avatar_url == other.avatar_url + && self.last_active_ago == other.last_active_ago + && self.membership == other.membership + } +} + impl RoomMember { pub fn new(event: &MemberEvent) -> Self { Self { diff --git a/src/session.rs b/src/session.rs index 6df86eff..eeee864f 100644 --- a/src/session.rs +++ b/src/session.rs @@ -16,9 +16,9 @@ //! User sessions. use ruma_identifiers::UserId; - +use serde::{Deserialize, Serialize}; /// A user session, containing an access token and information about the associated user account. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] pub struct Session { /// The access token used for this session. pub access_token: String, diff --git a/src/state/mod.rs b/src/state/mod.rs index 0a95bb88..ae81fc82 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -16,53 +16,112 @@ pub mod state_store; pub use state_store::JsonStore; -use crate::api; -use crate::events; -use api::r0::message::create_message_event; -use api::r0::session::login; -use api::r0::sync::sync_events; -use events::collections::all::{Event as NonRoomEvent, RoomEvent, StateEvent}; +use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::convert::{TryFrom, TryInto}; -use std::result::Result as StdResult; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use uuid::Uuid; - -use futures::future::Future; -use tokio::sync::RwLock; -use tokio::time::delay_for as sleep; -#[cfg(feature = "encryption")] -use tracing::debug; -use tracing::{info, instrument, trace}; - -use http::Method as HttpMethod; -use http::Response as HttpResponse; -use reqwest::header::{HeaderValue, InvalidHeaderValue}; -use url::Url; - -use ruma_api::{Endpoint, Outgoing}; -use ruma_events::room::message::MessageEventContent; -use ruma_events::EventResult; -pub use ruma_events::EventType; -use ruma_identifiers::RoomId; - -use crate::base_client::Client as BaseClient; +use crate::events::push_rules::Ruleset; +use crate::identifiers::{RoomId, UserId}; use crate::models::Room; use crate::session::Session; -use crate::VERSION; -use crate::{Error, EventEmitter, Result}; +use crate::{base_client::Token, Result}; + +#[derive(Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ClientState { + /// The current client session containing our user id, device id and access + /// token. + pub session: Option, + /// The current sync token that should be used for the next sync call. + pub sync_token: Option, + /// A list of ignored users. + pub ignored_users: Vec, + /// The push ruleset for the logged in user. + pub push_ruleset: Option, +} + /// Abstraction around the data store to avoid unnecessary request on client initialization. -/// pub trait StateStore { /// - fn load_state(&self) -> sync_events::IncomingResponse; + fn load_client_state(&self) -> Result; /// - fn save_state_events(&mut self, events: Vec) -> Result<()>; + fn load_room_state(&self, room_id: &RoomId) -> Result; /// - fn save_room_events(&mut self, events: Vec) -> Result<()>; + fn store_client_state(&self, _: ClientState) -> Result<()>; /// - fn save_non_room_events(&mut self, events: Vec) -> Result<()>; + fn store_room_state(&self, _: &Room) -> Result<()>; +} + +#[cfg(test)] +mod test { + use super::*; + + use std::collections::HashMap; + use std::convert::TryFrom; + + use crate::identifiers::{RoomId, UserId}; + + #[test] + fn serialize() { + let id = RoomId::try_from("!roomid:example.com").unwrap(); + let user = UserId::try_from("@example:example.com").unwrap(); + + let room = Room::new(&id, &user); + + let state = ClientState { + session: None, + sync_token: Some("hello".into()), + ignored_users: vec![user], + push_ruleset: None, + }; + assert_eq!( + r#"{"session":null,"sync_token":"hello","ignored_users":["@example:example.com"],"push_ruleset":null}"#, + serde_json::to_string(&state).unwrap() + ); + + let mut joined_rooms = HashMap::new(); + joined_rooms.insert(id, room); + assert_eq!( + r#"{ + "!roomid:example.com": { + "room_id": "!roomid:example.com", + "room_name": { + "name": null, + "canonical_alias": null, + "aliases": [] + }, + "own_user_id": "@example:example.com", + "creator": null, + "members": {}, + "typing_users": [], + "power_levels": null, + "encrypted": false, + "unread_highlight": null, + "unread_notifications": null + } +}"#, + serde_json::to_string_pretty(&joined_rooms).unwrap() + ); + } + + #[test] + fn deserialize() { + let id = RoomId::try_from("!roomid:example.com").unwrap(); + let user = UserId::try_from("@example:example.com").unwrap(); + + let room = Room::new(&id, &user); + + let state = ClientState { + session: None, + sync_token: Some("hello".into()), + ignored_users: vec![user], + push_ruleset: None, + }; + let json = serde_json::to_string(&state).unwrap(); + + assert_eq!(state, serde_json::from_str(&json).unwrap()); + + let mut joined_rooms = HashMap::new(); + joined_rooms.insert(id, room); + let json = serde_json::to_string(&joined_rooms).unwrap(); + + assert_eq!(joined_rooms, serde_json::from_str(&json).unwrap()); + } } diff --git a/src/state/state_store.rs b/src/state/state_store.rs index 2ebb4052..ea78cee1 100644 --- a/src/state/state_store.rs +++ b/src/state/state_store.rs @@ -1,9 +1,140 @@ -use super::StateStore; +use std::fs::OpenOptions; +use std::io::{BufReader, BufWriter, Write}; +use std::path::Path; +use super::{ClientState, StateStore}; +use crate::identifiers::RoomId; +use crate::{Error, Result, Room}; /// A default `StateStore` implementation that serializes state as json /// and saves it to disk. -pub struct JsonStore {} +pub struct JsonStore; -// impl StateStore for JsonStore { +impl StateStore for JsonStore { + fn load_client_state(&self) -> Result { + if let Some(mut path) = dirs::home_dir() { + path.push(".matrix_store/client.json"); + let file = OpenOptions::new().read(true).open(path)?; + let reader = BufReader::new(file); + serde_json::from_reader(reader).map_err(Error::from) + } else { + todo!("Error maybe") + } + } -// } + fn load_room_state(&self, room_id: &RoomId) -> Result { + if let Some(mut path) = dirs::home_dir() { + path.push(&format!(".matrix_store/rooms/{}.json", room_id)); + + let file = OpenOptions::new().read(true).open(path)?; + let reader = BufReader::new(file); + serde_json::from_reader(reader).map_err(Error::from) + } else { + todo!("Error maybe") + } + } + + fn store_client_state(&self, state: ClientState) -> Result<()> { + if let Some(mut path) = dirs::home_dir() { + path.push(".matrix_store/client.json"); + + if !Path::new(&path).exists() { + let mut dir = path.clone(); + dir.pop(); + std::fs::create_dir_all(dir)?; + } + + let json = serde_json::to_string(&state).map_err(Error::from)?; + + let file = OpenOptions::new().write(true).create(true).open(path)?; + let mut writer = BufWriter::new(file); + writer.write_all(json.as_bytes())?; + + Ok(()) + } else { + todo!("Error maybe") + } + } + + fn store_room_state(&self, room: &Room) -> Result<()> { + if let Some(mut path) = dirs::home_dir() { + path.push(&format!(".matrix_store/rooms/{}.json", room.room_id)); + + if !Path::new(&path).exists() { + let mut dir = path.clone(); + dir.pop(); + std::fs::create_dir_all(dir)?; + } + + let json = serde_json::to_string(&room).map_err(Error::from)?; + + let file = OpenOptions::new().write(true).create(true).open(path)?; + let mut writer = BufWriter::new(file); + writer.write_all(json.as_bytes())?; + + Ok(()) + } else { + todo!("Error maybe") + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + use std::convert::TryFrom; + use std::fs; + use std::sync::Mutex; + + use lazy_static::lazy_static; + + use crate::identifiers::{RoomId, UserId}; + + lazy_static! { + /// Limit io tests to one thread at a time. + pub static ref MTX: Mutex<()> = Mutex::new(()); + } + + fn run_and_cleanup(test: fn()) { + let _lock = MTX.lock(); + + test(); + + let mut path = dirs::home_dir().unwrap(); + path.push(".matrix_store"); + + if path.exists() { + fs::remove_dir_all(path).unwrap(); + } + } + + fn test_store_client_state() { + let store = JsonStore; + let state = ClientState::default(); + store.store_client_state(state).unwrap(); + let loaded = store.load_client_state().unwrap(); + assert_eq!(loaded, ClientState::default()); + } + + #[test] + fn store_client_state() { + run_and_cleanup(test_store_client_state); + } + + fn test_store_room_state() { + let store = JsonStore; + + let id = RoomId::try_from("!roomid:example.com").unwrap(); + let user = UserId::try_from("@example:example.com").unwrap(); + + let room = Room::new(&id, &user); + store.store_room_state(&room).unwrap(); + let loaded = store.load_room_state(&id).unwrap(); + assert_eq!(loaded, Room::new(&id, &user)); + } + + #[test] + fn store_room_state() { + run_and_cleanup(test_store_room_state); + } +}