char
/
conduit
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

improvement: batch inserts for stateids

next
Timo Kösters 2021-08-03 16:14:07 +02:00
parent 49ade0cfbd
commit 41ec7cf5d0
No known key found for this signature in database
GPG Key ID: 24DA7517711A2BA4
3 changed files with 91 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/database/abstraction/sqlite.rs
View File

@ -243,7 +243,10 @@ impl Tree for SqliteTable {
let statement = Box::leak(Box::new( let statement = Box::leak(Box::new(
guard guard
.prepare(&format!("SELECT key, value FROM {} ORDER BY key ASC", &self.name)) .prepare(&format!(
"SELECT key, value FROM {} ORDER BY key ASC",
&self.name
))
.unwrap(), .unwrap(),
)); ));

69
src/database/rooms.rs
View File

@ -313,31 +313,37 @@ impl Rooms {
let new_state = if !already_existed { let new_state = if !already_existed {
let mut new_state = HashSet::new(); let mut new_state = HashSet::new();
for ((event_type, state_key), eventid) in state { let batch = state
.iter()
.filter_map(|((event_type, state_key), eventid)| {
new_state.insert(eventid.clone()); new_state.insert(eventid.clone());
let mut statekey = event_type.as_ref().as_bytes().to_vec(); let mut statekey = event_type.as_ref().as_bytes().to_vec();
statekey.push(0xff); statekey.push(0xff);
statekey.extend_from_slice(&state_key.as_bytes()); statekey.extend_from_slice(&state_key.as_bytes());
let shortstatekey = match self.statekey_shortstatekey.get(&statekey)? { let shortstatekey = match self.statekey_shortstatekey.get(&statekey).ok()? {
Some(shortstatekey) => shortstatekey.to_vec(), Some(shortstatekey) => shortstatekey.to_vec(),
None => { None => {
let shortstatekey = db.globals.next_count()?; let shortstatekey = db.globals.next_count().ok()?;
self.statekey_shortstatekey self.statekey_shortstatekey
.insert(&statekey, &shortstatekey.to_be_bytes())?; .insert(&statekey, &shortstatekey.to_be_bytes())
.ok()?;
shortstatekey.to_be_bytes().to_vec() shortstatekey.to_be_bytes().to_vec()
} }
}; };
let shorteventid = match self.eventid_shorteventid.get(eventid.as_bytes())? { let shorteventid =
match self.eventid_shorteventid.get(eventid.as_bytes()).ok()? {
Some(shorteventid) => shorteventid.to_vec(), Some(shorteventid) => shorteventid.to_vec(),
None => { None => {
let shorteventid = db.globals.next_count()?; let shorteventid = db.globals.next_count().ok()?;
self.eventid_shorteventid self.eventid_shorteventid
.insert(eventid.as_bytes(), &shorteventid.to_be_bytes())?; .insert(eventid.as_bytes(), &shorteventid.to_be_bytes())
.ok()?;
self.shorteventid_eventid self.shorteventid_eventid
.insert(&shorteventid.to_be_bytes(), eventid.as_bytes())?; .insert(&shorteventid.to_be_bytes(), eventid.as_bytes())
.ok()?;
shorteventid.to_be_bytes().to_vec() shorteventid.to_be_bytes().to_vec()
} }
}; };
@ -345,9 +351,12 @@ impl Rooms {
let mut state_id = shortstatehash.to_be_bytes().to_vec(); let mut state_id = shortstatehash.to_be_bytes().to_vec();
state_id.extend_from_slice(&shortstatekey); state_id.extend_from_slice(&shortstatekey);
Some((state_id, shorteventid))
})
.collect::<Vec<_>>();
self.stateid_shorteventid self.stateid_shorteventid
.insert(&state_id, &*shorteventid)?; .insert_batch(&mut batch.into_iter())?;
}
new_state new_state
} else { } else {
@ -1120,29 +1129,38 @@ impl Rooms {
} }
}; };
for ((event_type, state_key), pdu) in state { let batch = state
.iter()
.filter_map(|((event_type, state_key), pdu)| {
let mut statekey = event_type.as_ref().as_bytes().to_vec(); let mut statekey = event_type.as_ref().as_bytes().to_vec();
statekey.push(0xff); statekey.push(0xff);
statekey.extend_from_slice(&state_key.as_bytes()); statekey.extend_from_slice(&state_key.as_bytes());
let shortstatekey = match self.statekey_shortstatekey.get(&statekey)? { let shortstatekey = match self.statekey_shortstatekey.get(&statekey).ok()? {
Some(shortstatekey) => shortstatekey.to_vec(), Some(shortstatekey) => shortstatekey.to_vec(),
None => { None => {
let shortstatekey = globals.next_count()?; let shortstatekey = globals.next_count().ok()?;
self.statekey_shortstatekey self.statekey_shortstatekey
.insert(&statekey, &shortstatekey.to_be_bytes())?; .insert(&statekey, &shortstatekey.to_be_bytes())
.ok()?;
shortstatekey.to_be_bytes().to_vec() shortstatekey.to_be_bytes().to_vec()
} }
}; };
let shorteventid = match self.eventid_shorteventid.get(pdu.event_id.as_bytes())? { let shorteventid = match self
.eventid_shorteventid
.get(pdu.event_id.as_bytes())
.ok()?
{
Some(shorteventid) => shorteventid.to_vec(), Some(shorteventid) => shorteventid.to_vec(),
None => { None => {
let shorteventid = globals.next_count()?; let shorteventid = globals.next_count().ok()?;
self.eventid_shorteventid self.eventid_shorteventid
.insert(pdu.event_id.as_bytes(), &shorteventid.to_be_bytes())?; .insert(pdu.event_id.as_bytes(), &shorteventid.to_be_bytes())
.ok()?;
self.shorteventid_eventid self.shorteventid_eventid
.insert(&shorteventid.to_be_bytes(), pdu.event_id.as_bytes())?; .insert(&shorteventid.to_be_bytes(), pdu.event_id.as_bytes())
.ok()?;
shorteventid.to_be_bytes().to_vec() shorteventid.to_be_bytes().to_vec()
} }
}; };
@ -1150,9 +1168,12 @@ impl Rooms {
let mut state_id = shortstatehash.clone(); let mut state_id = shortstatehash.clone();
state_id.extend_from_slice(&shortstatekey); state_id.extend_from_slice(&shortstatekey);
Some((state_id, shorteventid))
})
.collect::<Vec<_>>();
self.stateid_shorteventid self.stateid_shorteventid
.insert(&*state_id, &*shorteventid)?; .insert_batch(&mut batch.into_iter())?;
}
self.shorteventid_shortstatehash self.shorteventid_shortstatehash
.insert(&shorteventid, &*shortstatehash)?; .insert(&shorteventid, &*shortstatehash)?;
@ -1257,11 +1278,13 @@ impl Rooms {
} }
}; };
for (shortstatekey, shorteventid) in new_state { let mut batch = new_state.into_iter().map(|(shortstatekey, shorteventid)| {
let mut state_id = shortstatehash.to_be_bytes().to_vec(); let mut state_id = shortstatehash.to_be_bytes().to_vec();
state_id.extend_from_slice(&shortstatekey); state_id.extend_from_slice(&shortstatekey);
self.stateid_shorteventid.insert(&state_id, &shorteventid)?; (state_id, shorteventid)
} });
self.stateid_shorteventid.insert_batch(&mut batch)?;
Ok(shortstatehash) Ok(shortstatehash)
} else { } else {

4
src/database/rooms/edus.rs
View File

@ -422,7 +422,7 @@ impl RoomEdus {
} }
/// Sets all users to offline who have been quiet for too long. /// Sets all users to offline who have been quiet for too long.
pub fn presence_maintain( fn presence_maintain(
&self, &self,
rooms: &super::Rooms, rooms: &super::Rooms,
globals: &super::super::globals::Globals, globals: &super::super::globals::Globals,
@ -497,7 +497,7 @@ impl RoomEdus {
rooms: &super::Rooms, rooms: &super::Rooms,
globals: &super::super::globals::Globals, globals: &super::super::globals::Globals,
) -> Result<HashMap<UserId, PresenceEvent>> { ) -> Result<HashMap<UserId, PresenceEvent>> {
self.presence_maintain(rooms, globals)?; //self.presence_maintain(rooms, globals)?;
let mut prefix = room_id.as_bytes().to_vec(); let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff); prefix.push(0xff);