char/conduit
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

improvement: batched inserts for tokenids

Browse source
This commit is contained in:
Timo Kösters 2021-08-04 12:54:26 +02:00
parent fce22362d4
commit 9bb4c3cd01
No known key found for this signature in database
GPG key ID: 24DA7517711A2BA4
3 changed files with 24 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/database.rs
View file

@ -271,8 +271,8 @@ impl Database {
eventid_outlierpdu: builder.open_tree("eventid_outlierpdu")?,
referencedevents: builder.open_tree("referencedevents")?,
pdu_cache: Mutex::new(LruCache::new(100_000)),
auth_chain_cache: Mutex::new(LruCache::new(100_000)),
pdu_cache: Mutex::new(LruCache::new(1_000_000)),
auth_chain_cache: Mutex::new(LruCache::new(1_000_000)),
},
account_data: account_data::AccountData {
roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,

20
src/database/rooms.rs
View file

@ -905,18 +905,20 @@ impl Rooms {
}
EventType::RoomMessage => {
if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) {
for word in body
let mut batch = body
.split_terminator(|c: char| !c.is_alphanumeric())
.filter(|word| word.len() <= 50)
.map(str::to_lowercase)
{
let mut key = pdu.room_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(word.as_bytes());
key.push(0xff);
key.extend_from_slice(&pdu_id);
self.tokenids.insert(&key, &[])?;
}
.map(|word| {
let mut key = pdu.room_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(word.as_bytes());
key.push(0xff);
key.extend_from_slice(&pdu_id);
(key, Vec::new())
});
self.tokenids.insert_batch(&mut batch)?;
if body.starts_with(&format!("@conduit:{}: ", db.globals.server_name()))
&& self

13
src/server_server.rs
View file

@ -1,6 +1,6 @@
use crate::{
client_server::{self, claim_keys_helper, get_keys_helper},
database::DatabaseGuard,
database::{abstraction::sqlite::MILLI, DatabaseGuard},
utils, ConduitResult, Database, Error, PduEvent, Result, Ruma,
};
use get_profile_information::v1::ProfileField;
@ -1732,7 +1732,12 @@ fn get_auth_chain(starting_events: Vec<EventId>, db: &Database) -> Result<HashSe
cached.clone()
} else {
drop(cache);
let start = Instant::now();
let auth_chain = get_auth_chain_recursive(&event_id, HashSet::new(), db)?;
let elapsed = start.elapsed();
if elapsed > MILLI {
println!("auth chain for {} took {:?}", &event_id, elapsed)
}
cache = db.rooms.auth_chain_cache();
@ -1747,7 +1752,11 @@ fn get_auth_chain(starting_events: Vec<EventId>, db: &Database) -> Result<HashSe
Ok(full_auth_chain)
}
fn get_auth_chain_recursive(event_id: &EventId, mut found: HashSet<EventId>, db: &Database) -> Result<HashSet<EventId>> {
fn get_auth_chain_recursive(
event_id: &EventId,
mut found: HashSet<EventId>,
db: &Database,
) -> Result<HashSet<EventId>> {
if let Some(pdu) = db.rooms.get_pdu(&event_id)? {
for auth_event in &pdu.auth_events {
if !found.contains(auth_event) {