char/conduit
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

improvement: batch inserts for stateids

Browse source
This commit is contained in:
Timo Kösters 2021-08-03 16:14:07 +02:00
parent 49ade0cfbd
commit 41ec7cf5d0
No known key found for this signature in database
GPG key ID: 24DA7517711A2BA4
3 changed files with 91 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/database/abstraction/sqlite.rs
View file

@ -243,7 +243,10 @@ impl Tree for SqliteTable {
let statement = Box::leak(Box::new( let statement = Box::leak(Box::new(
guard guard
.prepare(&format!("SELECT key, value FROM {} ORDER BY key ASC", &self.name)) .prepare(&format!(
"SELECT key, value FROM {} ORDER BY key ASC",
&self.name
))
.unwrap(), .unwrap(),
)); ));

147
src/database/rooms.rs
View file

@ -313,41 +313,50 @@ impl Rooms {
let new_state = if !already_existed { let new_state = if !already_existed {
let mut new_state = HashSet::new(); let mut new_state = HashSet::new();
for ((event_type, state_key), eventid) in state { let batch = state
new_state.insert(eventid.clone()); .iter()
.filter_map(|((event_type, state_key), eventid)| {
new_state.insert(eventid.clone());
let mut statekey = event_type.as_ref().as_bytes().to_vec(); let mut statekey = event_type.as_ref().as_bytes().to_vec();
statekey.push(0xff); statekey.push(0xff);
statekey.extend_from_slice(&state_key.as_bytes()); statekey.extend_from_slice(&state_key.as_bytes());
let shortstatekey = match self.statekey_shortstatekey.get(&statekey)? { let shortstatekey = match self.statekey_shortstatekey.get(&statekey).ok()? {
Some(shortstatekey) => shortstatekey.to_vec(), Some(shortstatekey) => shortstatekey.to_vec(),
None => { None => {
let shortstatekey = db.globals.next_count()?; let shortstatekey = db.globals.next_count().ok()?;
self.statekey_shortstatekey self.statekey_shortstatekey
.insert(&statekey, &shortstatekey.to_be_bytes())?; .insert(&statekey, &shortstatekey.to_be_bytes())
shortstatekey.to_be_bytes().to_vec() .ok()?;
} shortstatekey.to_be_bytes().to_vec()
}; }
};
let shorteventid = match self.eventid_shorteventid.get(eventid.as_bytes())? { let shorteventid =
Some(shorteventid) => shorteventid.to_vec(), match self.eventid_shorteventid.get(eventid.as_bytes()).ok()? {
None => { Some(shorteventid) => shorteventid.to_vec(),
let shorteventid = db.globals.next_count()?; None => {
self.eventid_shorteventid let shorteventid = db.globals.next_count().ok()?;
.insert(eventid.as_bytes(), &shorteventid.to_be_bytes())?; self.eventid_shorteventid
self.shorteventid_eventid .insert(eventid.as_bytes(), &shorteventid.to_be_bytes())
.insert(&shorteventid.to_be_bytes(), eventid.as_bytes())?; .ok()?;
shorteventid.to_be_bytes().to_vec() self.shorteventid_eventid
} .insert(&shorteventid.to_be_bytes(), eventid.as_bytes())
}; .ok()?;
shorteventid.to_be_bytes().to_vec()
}
};
let mut state_id = shortstatehash.to_be_bytes().to_vec(); let mut state_id = shortstatehash.to_be_bytes().to_vec();
state_id.extend_from_slice(&shortstatekey); state_id.extend_from_slice(&shortstatekey);
self.stateid_shorteventid Some((state_id, shorteventid))
.insert(&state_id, &*shorteventid)?; })
} .collect::<Vec<_>>();
self.stateid_shorteventid
.insert_batch(&mut batch.into_iter())?;
new_state new_state
} else { } else {
@ -1120,39 +1129,51 @@ impl Rooms {
} }
}; };
for ((event_type, state_key), pdu) in state { let batch = state
let mut statekey = event_type.as_ref().as_bytes().to_vec(); .iter()
statekey.push(0xff); .filter_map(|((event_type, state_key), pdu)| {
statekey.extend_from_slice(&state_key.as_bytes()); let mut statekey = event_type.as_ref().as_bytes().to_vec();
statekey.push(0xff);
statekey.extend_from_slice(&state_key.as_bytes());
let shortstatekey = match self.statekey_shortstatekey.get(&statekey)? { let shortstatekey = match self.statekey_shortstatekey.get(&statekey).ok()? {
Some(shortstatekey) => shortstatekey.to_vec(), Some(shortstatekey) => shortstatekey.to_vec(),
None => { None => {
let shortstatekey = globals.next_count()?; let shortstatekey = globals.next_count().ok()?;
self.statekey_shortstatekey self.statekey_shortstatekey
.insert(&statekey, &shortstatekey.to_be_bytes())?; .insert(&statekey, &shortstatekey.to_be_bytes())
shortstatekey.to_be_bytes().to_vec() .ok()?;
} shortstatekey.to_be_bytes().to_vec()
}; }
};
let shorteventid = match self.eventid_shorteventid.get(pdu.event_id.as_bytes())? { let shorteventid = match self
Some(shorteventid) => shorteventid.to_vec(), .eventid_shorteventid
None => { .get(pdu.event_id.as_bytes())
let shorteventid = globals.next_count()?; .ok()?
self.eventid_shorteventid {
.insert(pdu.event_id.as_bytes(), &shorteventid.to_be_bytes())?; Some(shorteventid) => shorteventid.to_vec(),
self.shorteventid_eventid None => {
.insert(&shorteventid.to_be_bytes(), pdu.event_id.as_bytes())?; let shorteventid = globals.next_count().ok()?;
shorteventid.to_be_bytes().to_vec() self.eventid_shorteventid
} .insert(pdu.event_id.as_bytes(), &shorteventid.to_be_bytes())
}; .ok()?;
self.shorteventid_eventid
.insert(&shorteventid.to_be_bytes(), pdu.event_id.as_bytes())
.ok()?;
shorteventid.to_be_bytes().to_vec()
}
};
let mut state_id = shortstatehash.clone(); let mut state_id = shortstatehash.clone();
state_id.extend_from_slice(&shortstatekey); state_id.extend_from_slice(&shortstatekey);
self.stateid_shorteventid Some((state_id, shorteventid))
.insert(&*state_id, &*shorteventid)?; })
} .collect::<Vec<_>>();
self.stateid_shorteventid
.insert_batch(&mut batch.into_iter())?;
self.shorteventid_shortstatehash self.shorteventid_shortstatehash
.insert(&shorteventid, &*shortstatehash)?; .insert(&shorteventid, &*shortstatehash)?;
@ -1257,11 +1278,13 @@ impl Rooms {
} }
}; };
for (shortstatekey, shorteventid) in new_state { let mut batch = new_state.into_iter().map(|(shortstatekey, shorteventid)| {
let mut state_id = shortstatehash.to_be_bytes().to_vec(); let mut state_id = shortstatehash.to_be_bytes().to_vec();
state_id.extend_from_slice(&shortstatekey); state_id.extend_from_slice(&shortstatekey);
self.stateid_shorteventid.insert(&state_id, &shorteventid)?; (state_id, shorteventid)
} });
self.stateid_shorteventid.insert_batch(&mut batch)?;
Ok(shortstatehash) Ok(shortstatehash)
} else { } else {

4
src/database/rooms/edus.rs
View file

@ -422,7 +422,7 @@ impl RoomEdus {
} }
/// Sets all users to offline who have been quiet for too long. /// Sets all users to offline who have been quiet for too long.
pub fn presence_maintain( fn presence_maintain(
&self, &self,
rooms: &super::Rooms, rooms: &super::Rooms,
globals: &super::super::globals::Globals, globals: &super::super::globals::Globals,
@ -497,7 +497,7 @@ impl RoomEdus {
rooms: &super::Rooms, rooms: &super::Rooms,
globals: &super::super::globals::Globals, globals: &super::super::globals::Globals,
) -> Result<HashMap<UserId, PresenceEvent>> { ) -> Result<HashMap<UserId, PresenceEvent>> {
self.presence_maintain(rooms, globals)?; //self.presence_maintain(rooms, globals)?;
let mut prefix = room_id.as_bytes().to_vec(); let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff); prefix.push(0xff);