Merge branch 'speed' into 'master'
improvement: more efficient auth chain cache See merge request famedly/conduit!164
This commit is contained in:
Timo Kösters
commit
5aa56b92ee
4 changed files with 51 additions and 39 deletions
36
Cargo.lock
generated
36
Cargo.lock
generated
|
|
@ -2045,7 +2045,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma"
|
||||
version = "0.3.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"assign",
|
||||
"js_int",
|
||||
|
|
@ -2066,7 +2066,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-api"
|
||||
version = "0.18.3"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http",
|
||||
|
|
@ -2082,7 +2082,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-api-macros"
|
||||
version = "0.18.3"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
|
|
@ -2093,7 +2093,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-appservice-api"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"ruma-api",
|
||||
"ruma-common",
|
||||
|
|
@ -2107,7 +2107,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-client-api"
|
||||
version = "0.12.2"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"assign",
|
||||
"bytes",
|
||||
|
|
@ -2127,7 +2127,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-common"
|
||||
version = "0.6.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"js_int",
|
||||
|
|
@ -2142,7 +2142,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-events"
|
||||
version = "0.24.4"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"indoc",
|
||||
"js_int",
|
||||
|
|
@ -2158,7 +2158,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-events-macros"
|
||||
version = "0.24.4"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
|
|
@ -2169,7 +2169,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-federation-api"
|
||||
version = "0.3.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"js_int",
|
||||
"ruma-api",
|
||||
|
|
@ -2184,7 +2184,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-identifiers"
|
||||
version = "0.20.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"paste",
|
||||
"rand 0.8.4",
|
||||
|
|
@ -2198,7 +2198,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-identifiers-macros"
|
||||
version = "0.20.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"ruma-identifiers-validation",
|
||||
|
|
@ -2208,7 +2208,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-identifiers-validation"
|
||||
version = "0.5.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"thiserror",
|
||||
]
|
||||
|
|
@ -2216,7 +2216,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-identity-service-api"
|
||||
version = "0.3.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"js_int",
|
||||
"ruma-api",
|
||||
|
|
@ -2229,7 +2229,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-push-gateway-api"
|
||||
version = "0.3.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"js_int",
|
||||
"ruma-api",
|
||||
|
|
@ -2244,7 +2244,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-serde"
|
||||
version = "0.5.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
|
|
@ -2258,7 +2258,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-serde-macros"
|
||||
version = "0.5.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
|
|
@ -2269,7 +2269,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-signatures"
|
||||
version = "0.9.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"base64 0.13.0",
|
||||
"ed25519-dalek",
|
||||
|
|
@ -2286,7 +2286,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "ruma-state-res"
|
||||
version = "0.3.0"
|
||||
source = "git+https://github.com/DevinR528/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
|
||||
dependencies = [
|
||||
"itertools 0.10.1",
|
||||
"js_int",
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request
|
|||
|
||||
# Used for matrix spec type definitions and helpers
|
||||
#ruma = { git = "https://github.com/ruma/ruma", rev = "f5ab038e22421ed338396ece977b6b2844772ced", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
|
||||
ruma = { git = "https://github.com/DevinR528/ruma", rev = "2215049b60a1c3358f5a52215adf1e7bb88619a1", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
|
||||
ruma = { git = "https://github.com/timokoesters/ruma", rev = "2215049b60a1c3358f5a52215adf1e7bb88619a1", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
|
||||
#ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
|
||||
|
||||
# Used for long polling and federation sender, should be the same as rocket::tokio
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ use ruma::{
|
|||
},
|
||||
push::{self, Action, Tweak},
|
||||
serde::{CanonicalJsonObject, CanonicalJsonValue, Raw},
|
||||
state_res::{self, Event, RoomVersion, StateMap},
|
||||
state_res::{self, RoomVersion, StateMap},
|
||||
uint, EventId, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId,
|
||||
};
|
||||
use std::{
|
||||
|
|
@ -91,7 +91,7 @@ pub struct Rooms {
|
|||
pub(super) referencedevents: Arc<dyn Tree>,
|
||||
|
||||
pub(super) pdu_cache: Mutex<LruCache<EventId, Arc<PduEvent>>>,
|
||||
pub(super) auth_chain_cache: Mutex<LruCache<u64, HashSet<u64>>>,
|
||||
pub(super) auth_chain_cache: Mutex<LruCache<Vec<u64>, HashSet<u64>>>,
|
||||
pub(super) shorteventid_cache: Mutex<LruCache<u64, EventId>>,
|
||||
pub(super) eventidshort_cache: Mutex<LruCache<EventId, u64>>,
|
||||
pub(super) statekeyshort_cache: Mutex<LruCache<(EventType, String), u64>>,
|
||||
|
|
@ -3166,7 +3166,7 @@ impl Rooms {
|
|||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub fn auth_chain_cache(&self) -> std::sync::MutexGuard<'_, LruCache<u64, HashSet<u64>>> {
|
||||
pub fn auth_chain_cache(&self) -> std::sync::MutexGuard<'_, LruCache<Vec<u64>, HashSet<u64>>> {
|
||||
self.auth_chain_cache.lock().unwrap()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1976,28 +1976,40 @@ fn get_auth_chain(
|
|||
) -> Result<impl Iterator<Item = EventId> + '_> {
|
||||
let mut full_auth_chain = HashSet::new();
|
||||
|
||||
let starting_events = starting_events
|
||||
.iter()
|
||||
.map(|id| {
|
||||
db.rooms
|
||||
.get_or_create_shorteventid(id, &db.globals)
|
||||
.map(|s| (s, id))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
const NUM_BUCKETS: usize = 100;
|
||||
|
||||
let mut buckets = vec![HashSet::new(); NUM_BUCKETS];
|
||||
|
||||
for id in starting_events {
|
||||
let short = db.rooms.get_or_create_shorteventid(&id, &db.globals)?;
|
||||
let bucket_id = (short % NUM_BUCKETS as u64) as usize;
|
||||
buckets[bucket_id].insert((short, id));
|
||||
}
|
||||
|
||||
let mut cache = db.rooms.auth_chain_cache();
|
||||
|
||||
for (sevent_id, event_id) in starting_events {
|
||||
if let Some(cached) = cache.get_mut(&sevent_id) {
|
||||
for chunk in buckets {
|
||||
let chunk_key = chunk.iter().map(|(short, _)| short).copied().collect();
|
||||
if let Some(cached) = cache.get_mut(&chunk_key) {
|
||||
full_auth_chain.extend(cached.iter().cloned());
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut chunk_cache = HashSet::new();
|
||||
for (sevent_id, event_id) in chunk {
|
||||
if let Some(cached) = cache.get_mut(&[sevent_id][..]) {
|
||||
chunk_cache.extend(cached.iter().cloned());
|
||||
} else {
|
||||
drop(cache);
|
||||
let auth_chain = get_auth_chain_inner(&event_id, db)?;
|
||||
cache = db.rooms.auth_chain_cache();
|
||||
cache.insert(sevent_id, auth_chain.clone());
|
||||
full_auth_chain.extend(auth_chain);
|
||||
cache.insert(vec![sevent_id], auth_chain.clone());
|
||||
chunk_cache.extend(auth_chain);
|
||||
};
|
||||
}
|
||||
cache.insert(chunk_key, chunk_cache.clone());
|
||||
full_auth_chain.extend(chunk_cache);
|
||||
}
|
||||
|
||||
drop(cache);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue