char
/
conduit
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'speed' into 'master' 

improvement: more efficient auth chain cache

See merge request famedly/conduit!164
next
Timo Kösters 2021-08-24 20:12:10 +00:00
parent 30b309b708 dd87066546
commit 5aa56b92ee
4 changed files with 51 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
Cargo.lock generated
View File

@ -2045,7 +2045,7 @@ dependencies = [
[[package]]
name = "ruma"
version = "0.3.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"assign",
"js_int",
@ -2066,7 +2066,7 @@ dependencies = [
[[package]]
name = "ruma-api"
version = "0.18.3"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"bytes",
"http",
@ -2082,7 +2082,7 @@ dependencies = [
[[package]]
name = "ruma-api-macros"
version = "0.18.3"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"proc-macro-crate",
"proc-macro2",
@ -2093,7 +2093,7 @@ dependencies = [
[[package]]
name = "ruma-appservice-api"
version = "0.4.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"ruma-api",
"ruma-common",
@ -2107,7 +2107,7 @@ dependencies = [
[[package]]
name = "ruma-client-api"
version = "0.12.2"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"assign",
"bytes",
@ -2127,7 +2127,7 @@ dependencies = [
[[package]]
name = "ruma-common"
version = "0.6.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"indexmap",
"js_int",
@ -2142,7 +2142,7 @@ dependencies = [
[[package]]
name = "ruma-events"
version = "0.24.4"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"indoc",
"js_int",
@ -2158,7 +2158,7 @@ dependencies = [
[[package]]
name = "ruma-events-macros"
version = "0.24.4"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"proc-macro-crate",
"proc-macro2",
@ -2169,7 +2169,7 @@ dependencies = [
[[package]]
name = "ruma-federation-api"
version = "0.3.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"js_int",
"ruma-api",
@ -2184,7 +2184,7 @@ dependencies = [
[[package]]
name = "ruma-identifiers"
version = "0.20.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"paste",
"rand 0.8.4",
@ -2198,7 +2198,7 @@ dependencies = [
[[package]]
name = "ruma-identifiers-macros"
version = "0.20.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"quote",
"ruma-identifiers-validation",
@ -2208,7 +2208,7 @@ dependencies = [
[[package]]
name = "ruma-identifiers-validation"
version = "0.5.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"thiserror",
]
@ -2216,7 +2216,7 @@ dependencies = [
[[package]]
name = "ruma-identity-service-api"
version = "0.3.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"js_int",
"ruma-api",
@ -2229,7 +2229,7 @@ dependencies = [
[[package]]
name = "ruma-push-gateway-api"
version = "0.3.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"js_int",
"ruma-api",
@ -2244,7 +2244,7 @@ dependencies = [
[[package]]
name = "ruma-serde"
version = "0.5.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"bytes",
"form_urlencoded",
@ -2258,7 +2258,7 @@ dependencies = [
[[package]]
name = "ruma-serde-macros"
version = "0.5.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"proc-macro-crate",
"proc-macro2",
@ -2269,7 +2269,7 @@ dependencies = [
[[package]]
name = "ruma-signatures"
version = "0.9.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"base64 0.13.0",
"ed25519-dalek",
@ -2286,7 +2286,7 @@ dependencies = [
[[package]]
name = "ruma-state-res"
version = "0.3.0"
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"itertools 0.10.1",
"js_int",

2
Cargo.toml
View File

@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request
# Used for matrix spec type definitions and helpers
#ruma = { git = "https://github.com/ruma/ruma", rev = "f5ab038e22421ed338396ece977b6b2844772ced", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
ruma = { git = "https://github.com/DevinR528/ruma", rev = "2215049b60a1c3358f5a52215adf1e7bb88619a1", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
ruma = { git = "https://github.com/timokoesters/ruma", rev = "2215049b60a1c3358f5a52215adf1e7bb88619a1", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
#ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
# Used for long polling and federation sender, should be the same as rocket::tokio

6
src/database/rooms.rs
View File

@ -19,7 +19,7 @@ use ruma::{
},
push::{self, Action, Tweak},
serde::{CanonicalJsonObject, CanonicalJsonValue, Raw},
state_res::{self, Event, RoomVersion, StateMap},
state_res::{self, RoomVersion, StateMap},
uint, EventId, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId,
};
use std::{
@ -91,7 +91,7 @@ pub struct Rooms {
pub(super) referencedevents: Arc<dyn Tree>,
pub(super) pdu_cache: Mutex<LruCache<EventId, Arc<PduEvent>>>,
pub(super) auth_chain_cache: Mutex<LruCache<u64, HashSet<u64>>>,
pub(super) auth_chain_cache: Mutex<LruCache<Vec<u64>, HashSet<u64>>>,
pub(super) shorteventid_cache: Mutex<LruCache<u64, EventId>>,
pub(super) eventidshort_cache: Mutex<LruCache<EventId, u64>>,
pub(super) statekeyshort_cache: Mutex<LruCache<(EventType, String), u64>>,
@ -3166,7 +3166,7 @@ impl Rooms {
}
#[tracing::instrument(skip(self))]
pub fn auth_chain_cache(&self) -> std::sync::MutexGuard<'_, LruCache<u64, HashSet<u64>>> {
pub fn auth_chain_cache(&self) -> std::sync::MutexGuard<'_, LruCache<Vec<u64>, HashSet<u64>>> {
self.auth_chain_cache.lock().unwrap()
}
}

46
src/server_server.rs
View File

@ -1976,27 +1976,39 @@ fn get_auth_chain(
) -> Result<impl Iterator<Item = EventId> + '_> {
let mut full_auth_chain = HashSet::new();
let starting_events = starting_events
.iter()
.map(|id| {
db.rooms
.get_or_create_shorteventid(id, &db.globals)
.map(|s| (s, id))
})
.collect::<Result<Vec<_>>>()?;
const NUM_BUCKETS: usize = 100;
let mut buckets = vec![HashSet::new(); NUM_BUCKETS];
for id in starting_events {
let short = db.rooms.get_or_create_shorteventid(&id, &db.globals)?;
let bucket_id = (short % NUM_BUCKETS as u64) as usize;
buckets[bucket_id].insert((short, id));
}
let mut cache = db.rooms.auth_chain_cache();
for (sevent_id, event_id) in starting_events {
if let Some(cached) = cache.get_mut(&sevent_id) {
for chunk in buckets {
let chunk_key = chunk.iter().map(|(short, _)| short).copied().collect();
if let Some(cached) = cache.get_mut(&chunk_key) {
full_auth_chain.extend(cached.iter().cloned());
} else {
drop(cache);
let auth_chain = get_auth_chain_inner(&event_id, db)?;
cache = db.rooms.auth_chain_cache();
cache.insert(sevent_id, auth_chain.clone());
full_auth_chain.extend(auth_chain);
};
continue;
}
let mut chunk_cache = HashSet::new();
for (sevent_id, event_id) in chunk {
if let Some(cached) = cache.get_mut(&[sevent_id][..]) {
chunk_cache.extend(cached.iter().cloned());
} else {
drop(cache);
let auth_chain = get_auth_chain_inner(&event_id, db)?;
cache = db.rooms.auth_chain_cache();
cache.insert(vec![sevent_id], auth_chain.clone());
chunk_cache.extend(auth_chain);
};
}
cache.insert(chunk_key, chunk_cache.clone());
full_auth_chain.extend(chunk_cache);
}
drop(cache);