char
/
conduit
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: heed db backend (LMDB)

next
Timo Kösters 2021-07-29 20:17:47 +02:00
parent c209775abd
commit 5c776e9ba7
No known key found for this signature in database
GPG Key ID: 24DA7517711A2BA4
9 changed files with 456 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
Cargo.lock generated
View File

@ -135,6 +135,15 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "bindgen" name = "bindgen"
version = "0.59.1" version = "0.59.1"
@ -234,6 +243,12 @@ dependencies = [
"nom", "nom",
] ]
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
version = "1.0.0" version = "1.0.0"
@ -278,6 +293,7 @@ dependencies = [
"bytes", "bytes",
"crossbeam", "crossbeam",
"directories", "directories",
"heed",
"http", "http",
"image", "image",
"jsonwebtoken", "jsonwebtoken",
@ -379,7 +395,7 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
] ]
[[package]] [[package]]
@ -388,12 +404,12 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"crossbeam-channel", "crossbeam-channel",
"crossbeam-deque", "crossbeam-deque",
"crossbeam-epoch", "crossbeam-epoch",
"crossbeam-queue", "crossbeam-queue 0.3.2",
"crossbeam-utils", "crossbeam-utils 0.8.5",
] ]
[[package]] [[package]]
@ -402,8 +418,8 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"crossbeam-utils", "crossbeam-utils 0.8.5",
] ]
[[package]] [[package]]
@ -412,9 +428,9 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"crossbeam-epoch", "crossbeam-epoch",
"crossbeam-utils", "crossbeam-utils 0.8.5",
] ]
[[package]] [[package]]
@ -423,21 +439,40 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"crossbeam-utils", "crossbeam-utils 0.8.5",
"lazy_static", "lazy_static",
"memoffset", "memoffset",
"scopeguard", "scopeguard",
] ]
[[package]]
name = "crossbeam-queue"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
dependencies = [
"crossbeam-utils 0.6.6",
]
[[package]] [[package]]
name = "crossbeam-queue" name = "crossbeam-queue"
version = "0.3.2" version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"crossbeam-utils", "crossbeam-utils 0.8.5",
]
[[package]]
name = "crossbeam-utils"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
dependencies = [
"cfg-if 0.1.10",
"lazy_static",
] ]
[[package]] [[package]]
@ -446,7 +481,7 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"lazy_static", "lazy_static",
] ]
@ -610,7 +645,7 @@ version = "0.8.28"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
] ]
[[package]] [[package]]
@ -828,7 +863,7 @@ version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"libc", "libc",
"wasi 0.9.0+wasi-snapshot-preview1", "wasi 0.9.0+wasi-snapshot-preview1",
] ]
@ -839,7 +874,7 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"libc", "libc",
"wasi 0.10.2+wasi-snapshot-preview1", "wasi 0.10.2+wasi-snapshot-preview1",
] ]
@ -906,6 +941,42 @@ dependencies = [
"unicode-segmentation", "unicode-segmentation",
] ]
[[package]]
name = "heed"
version = "0.10.6"
source = "git+https://github.com/Kerollmops/heed.git?rev=b235e9c3e9984737c967b5de1014b48f125dc28b#b235e9c3e9984737c967b5de1014b48f125dc28b"
dependencies = [
"bytemuck",
"byteorder",
"heed-traits",
"heed-types",
"libc",
"lmdb-rkv-sys",
"once_cell",
"page_size",
"serde",
"synchronoise",
"url",
]
[[package]]
name = "heed-traits"
version = "0.7.0"
source = "git+https://github.com/Kerollmops/heed.git?rev=b235e9c3e9984737c967b5de1014b48f125dc28b#b235e9c3e9984737c967b5de1014b48f125dc28b"
[[package]]
name = "heed-types"
version = "0.7.2"
source = "git+https://github.com/Kerollmops/heed.git?rev=b235e9c3e9984737c967b5de1014b48f125dc28b#b235e9c3e9984737c967b5de1014b48f125dc28b"
dependencies = [
"bincode",
"bytemuck",
"byteorder",
"heed-traits",
"serde",
"serde_json",
]
[[package]] [[package]]
name = "hermit-abi" name = "hermit-abi"
version = "0.1.19" version = "0.1.19"
@ -1068,7 +1139,7 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
] ]
[[package]] [[package]]
@ -1190,7 +1261,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f84d96438c15fcd6c3f244c8fce01d1e2b9c6b5623e9c711dc9286d8fc92d6a" checksum = "6f84d96438c15fcd6c3f244c8fce01d1e2b9c6b5623e9c711dc9286d8fc92d6a"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"winapi", "winapi",
] ]
@ -1223,6 +1294,17 @@ version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
[[package]]
name = "lmdb-rkv-sys"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b27470ac25167b3afdfb6af8fcd3bc1be67de50ffbdaf4073378cfded6ae24a5"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]] [[package]]
name = "lock_api" name = "lock_api"
version = "0.4.4" version = "0.4.4"
@ -1238,7 +1320,7 @@ version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
] ]
[[package]] [[package]]
@ -1247,7 +1329,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2111607c723d7857e0d8299d5ce7a0bf4b844d3e44f8de136b13da513eaf8fc4" checksum = "2111607c723d7857e0d8299d5ce7a0bf4b844d3e44f8de136b13da513eaf8fc4"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"generator", "generator",
"scoped-tls", "scoped-tls",
"serde", "serde",
@ -1503,6 +1585,16 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "page_size"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.11.1" version = "0.11.1"
@ -1520,7 +1612,7 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"instant", "instant",
"libc", "libc",
"redox_syscall", "redox_syscall",
@ -2320,7 +2412,7 @@ dependencies = [
"base64 0.13.0", "base64 0.13.0",
"blake2b_simd", "blake2b_simd",
"constant_time_eq", "constant_time_eq",
"crossbeam-utils", "crossbeam-utils 0.8.5",
] ]
[[package]] [[package]]
@ -2540,7 +2632,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12"
dependencies = [ dependencies = [
"block-buffer", "block-buffer",
"cfg-if", "cfg-if 1.0.0",
"cpufeatures", "cpufeatures",
"digest", "digest",
"opaque-debug", "opaque-debug",
@ -2601,7 +2693,7 @@ checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
dependencies = [ dependencies = [
"crc32fast", "crc32fast",
"crossbeam-epoch", "crossbeam-epoch",
"crossbeam-utils", "crossbeam-utils 0.8.5",
"fs2", "fs2",
"fxhash", "fxhash",
"libc", "libc",
@ -2622,7 +2714,7 @@ version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"libc", "libc",
"winapi", "winapi",
] ]
@ -2751,6 +2843,15 @@ dependencies = [
"unicode-xid", "unicode-xid",
] ]
[[package]]
name = "synchronoise"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb"
dependencies = [
"crossbeam-queue 0.1.2",
]
[[package]] [[package]]
name = "synstructure" name = "synstructure"
version = "0.12.5" version = "0.12.5"
@ -2775,7 +2876,7 @@ version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"libc", "libc",
"rand 0.8.4", "rand 0.8.4",
"redox_syscall", "redox_syscall",
@ -3005,7 +3106,7 @@ version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"pin-project-lite", "pin-project-lite",
"tracing-attributes", "tracing-attributes",
"tracing-core", "tracing-core",
@ -3105,7 +3206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad0d7f5db438199a6e2609debe3f69f808d074e0a2888ee0bccb45fe234d03f4" checksum = "ad0d7f5db438199a6e2609debe3f69f808d074e0a2888ee0bccb45fe234d03f4"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"cfg-if", "cfg-if 1.0.0",
"data-encoding", "data-encoding",
"enum-as-inner", "enum-as-inner",
"futures-channel", "futures-channel",
@ -3129,7 +3230,7 @@ version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6ad17b608a64bd0735e67bde16b0636f8aa8591f831a25d18443ed00a699770" checksum = "f6ad17b608a64bd0735e67bde16b0636f8aa8591f831a25d18443ed00a699770"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"futures-util", "futures-util",
"ipconfig", "ipconfig",
"lazy_static", "lazy_static",
@ -3290,7 +3391,7 @@ version = "0.2.74"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"serde", "serde",
"serde_json", "serde_json",
"wasm-bindgen-macro", "wasm-bindgen-macro",
@ -3317,7 +3418,7 @@ version = "0.4.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1"
dependencies = [ dependencies = [
"cfg-if", "cfg-if 1.0.0",
"js-sys", "js-sys",
"wasm-bindgen", "wasm-bindgen",
"web-sys", "web-sys",

4
Cargo.toml
View File

@ -79,12 +79,14 @@ parking_lot = { version = "0.11.1", optional = true }
crossbeam = { version = "0.8.1", optional = true } crossbeam = { version = "0.8.1", optional = true }
num_cpus = "1.13.0" num_cpus = "1.13.0"
threadpool = "1.8.1" threadpool = "1.8.1"
heed = { git = "https://github.com/Kerollmops/heed.git", rev = "b235e9c3e9984737c967b5de1014b48f125dc28b", optional = true }
[features] [features]
default = ["conduit_bin", "backend_sqlite"] default = ["conduit_bin", "backend_heed"]
backend_sled = ["sled"] backend_sled = ["sled"]
backend_rocksdb = ["rocksdb"] backend_rocksdb = ["rocksdb"]
backend_sqlite = ["sqlite"] backend_sqlite = ["sqlite"]
backend_heed = ["heed", "crossbeam"]
sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"] sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"]
conduit_bin = [] # TODO: add rocket to this when it is optional conduit_bin = [] # TODO: add rocket to this when it is optional

2
src/client_server/to_device.rs
View File

@ -47,7 +47,7 @@ pub async fn send_event_to_device_route(
db.sending.send_reliable_edu( db.sending.send_reliable_edu(
target_user_id.server_name(), target_user_id.server_name(),
&serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice( serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice(
DirectDeviceContent { DirectDeviceContent {
sender: sender_user.clone(), sender: sender_user.clone(),
ev_type: EventType::from(&body.event_type), ev_type: EventType::from(&body.event_type),

15
src/database.rs
View File

@ -156,6 +156,9 @@ pub type Engine = abstraction::rocksdb::Engine;
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
pub type Engine = abstraction::sqlite::Engine; pub type Engine = abstraction::sqlite::Engine;
#[cfg(feature = "heed")]
pub type Engine = abstraction::heed::Engine;
pub struct Database { pub struct Database {
_db: Arc<Engine>, _db: Arc<Engine>,
pub globals: globals::Globals, pub globals: globals::Globals,
@ -188,26 +191,22 @@ impl Database {
fn check_sled_or_sqlite_db(config: &Config) -> Result<()> { fn check_sled_or_sqlite_db(config: &Config) -> Result<()> {
let path = Path::new(&config.database_path); let path = Path::new(&config.database_path);
#[cfg(feature = "backend_sqlite")]
{
let sled_exists = path.join("db").exists(); let sled_exists = path.join("db").exists();
let sqlite_exists = path.join("conduit.db").exists(); let sqlite_exists = path.join("conduit.db").exists();
// TODO: heed
if sled_exists { if sled_exists {
if sqlite_exists { if sqlite_exists {
// most likely an in-place directory, only warn // most likely an in-place directory, only warn
warn!("Both sled and sqlite databases are detected in database directory"); warn!("Both sled and sqlite databases are detected in database directory");
warn!("Currently running from the sqlite database, but consider removing sled database files to free up space") warn!("Currently running from the sqlite database, but consider removing sled database files to free up space")
} else { } else {
error!( error!("Sled database detected, conduit now uses sqlite for database operations");
"Sled database detected, conduit now uses sqlite for database operations"
);
error!("This database must be converted to sqlite, go to https://github.com/ShadowJonathan/conduit_toolbox#conduit_sled_to_sqlite"); error!("This database must be converted to sqlite, go to https://github.com/ShadowJonathan/conduit_toolbox#conduit_sled_to_sqlite");
return Err(Error::bad_config( return Err(Error::bad_config(
"sled database detected, migrate to sqlite", "sled database detected, migrate to sqlite",
)); ));
} }
} }
}
Ok(()) Ok(())
} }
@ -313,8 +312,8 @@ impl Database {
}, },
sending: sending::Sending { sending: sending::Sending {
servername_educount: builder.open_tree("servername_educount")?, servername_educount: builder.open_tree("servername_educount")?,
servernamepduids: builder.open_tree("servernamepduids")?, servernameevent_data: builder.open_tree("servernameevent_data")?,
servercurrentevents: builder.open_tree("servercurrentevents")?, servercurrentevent_data: builder.open_tree("servercurrentevent_data")?,
maximum_requests: Arc::new(Semaphore::new(config.max_concurrent_requests as usize)), maximum_requests: Arc::new(Semaphore::new(config.max_concurrent_requests as usize)),
sender: sending_sender, sender: sending_sender,
}, },

3
src/database/abstraction.rs
View File

@ -12,6 +12,9 @@ pub mod sled;
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
pub mod sqlite; pub mod sqlite;
#[cfg(feature = "heed")]
pub mod heed;
pub trait DatabaseEngine: Sized { pub trait DatabaseEngine: Sized {
fn open(config: &Config) -> Result<Arc<Self>>; fn open(config: &Config) -> Result<Arc<Self>>;
fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>>; fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>>;

241
src/database/abstraction/heed.rs Normal file
View File

@ -0,0 +1,241 @@
use super::super::Config;
use crossbeam::channel::{bounded, Sender as ChannelSender};
use threadpool::ThreadPool;
use crate::{Error, Result};
use std::{
collections::HashMap,
future::Future,
pin::Pin,
sync::{Arc, Mutex, RwLock},
};
use tokio::sync::oneshot::Sender;
use super::{DatabaseEngine, Tree};
type TupleOfBytes = (Vec<u8>, Vec<u8>);
pub struct Engine {
env: heed::Env,
iter_pool: Mutex<ThreadPool>,
}
pub struct EngineTree {
engine: Arc<Engine>,
tree: Arc<heed::UntypedDatabase>,
watchers: RwLock<HashMap<Vec<u8>, Vec<Sender<()>>>>,
}
fn convert_error(error: heed::Error) -> Error {
panic!(error.to_string());
Error::HeedError {
error: error.to_string(),
}
}
impl DatabaseEngine for Engine {
fn open(config: &Config) -> Result<Arc<Self>> {
let mut env_builder = heed::EnvOpenOptions::new();
env_builder.map_size(1024 * 1024 * 1024 * 1024); // 1 Terabyte
env_builder.max_readers(126);
env_builder.max_dbs(128);
unsafe {
env_builder.flag(heed::flags::Flags::MdbNoSync);
env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
}
Ok(Arc::new(Engine {
env: env_builder
.open(&config.database_path)
.map_err(convert_error)?,
iter_pool: Mutex::new(ThreadPool::new(10)),
}))
}
fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>> {
// Creates the db if it doesn't exist already
Ok(Arc::new(EngineTree {
engine: Arc::clone(self),
tree: Arc::new(
self.env
.create_database(Some(name))
.map_err(convert_error)?,
),
watchers: RwLock::new(HashMap::new()),
}))
}
fn flush(self: &Arc<Self>) -> Result<()> {
self.env.force_sync().map_err(convert_error)?;
Ok(())
}
}
impl EngineTree {
#[tracing::instrument(skip(self, tree, from, backwards))]
fn iter_from_thread(
&self,
tree: Arc<heed::UntypedDatabase>,
from: Vec<u8>,
backwards: bool,
) -> Box<dyn Iterator<Item = TupleOfBytes> + Send + Sync> {
let (s, r) = bounded::<TupleOfBytes>(5);
let engine = Arc::clone(&self.engine);
let lock = self.engine.iter_pool.lock().unwrap();
if lock.active_count() < lock.max_count() {
lock.execute(move || {
iter_from_thread_work(tree, &engine.env.read_txn().unwrap(), from, backwards, &s);
});
} else {
std::thread::spawn(move || {
iter_from_thread_work(tree, &engine.env.read_txn().unwrap(), from, backwards, &s);
});
}
Box::new(r.into_iter())
}
}
#[tracing::instrument(skip(tree, txn, from, backwards))]
fn iter_from_thread_work(
tree: Arc<heed::UntypedDatabase>,
txn: &heed::RoTxn<'_>,
from: Vec<u8>,
backwards: bool,
s: &ChannelSender<(Vec<u8>, Vec<u8>)>,
) {
if backwards {
for (k, v) in tree.rev_range(txn, ..=&*from).unwrap().map(|r| r.unwrap()) {
if s.send((k.to_vec(), v.to_vec())).is_err() {
return;
}
}
} else {
if from.is_empty() {
for (k, v) in tree.iter(txn).unwrap().map(|r| r.unwrap()) {
if s.send((k.to_vec(), v.to_vec())).is_err() {
return;
}
}
} else {
for (k, v) in tree.range(txn, &*from..).unwrap().map(|r| r.unwrap()) {
if s.send((k.to_vec(), v.to_vec())).is_err() {
return;
}
}
}
}
}
impl Tree for EngineTree {
#[tracing::instrument(skip(self, key))]
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
let txn = self.engine.env.read_txn().map_err(convert_error)?;
Ok(self
.tree
.get(&txn, &key)
.map_err(convert_error)?
.map(|s| s.to_vec()))
}
#[tracing::instrument(skip(self, key, value))]
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
let mut txn = self.engine.env.write_txn().map_err(convert_error)?;
self.tree
.put(&mut txn, &key, &value)
.map_err(convert_error)?;
txn.commit().map_err(convert_error)?;
let watchers = self.watchers.read().unwrap();
let mut triggered = Vec::new();
for length in 0..=key.len() {
if watchers.contains_key(&key[..length]) {
triggered.push(&key[..length]);
}
}
drop(watchers);
if !triggered.is_empty() {
let mut watchers = self.watchers.write().unwrap();
for prefix in triggered {
if let Some(txs) = watchers.remove(prefix) {
for tx in txs {
let _ = tx.send(());
}
}
}
};
Ok(())
}
#[tracing::instrument(skip(self, key))]
fn remove(&self, key: &[u8]) -> Result<()> {
let mut txn = self.engine.env.write_txn().map_err(convert_error)?;
self.tree.delete(&mut txn, &key).map_err(convert_error)?;
txn.commit().map_err(convert_error)?;
Ok(())
}
#[tracing::instrument(skip(self))]
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> {
self.iter_from(&[], false)
}
#[tracing::instrument(skip(self, from, backwards))]
fn iter_from(
&self,
from: &[u8],
backwards: bool,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send> {
self.iter_from_thread(Arc::clone(&self.tree), from.to_vec(), backwards)
}
#[tracing::instrument(skip(self, key))]
fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
let mut txn = self.engine.env.write_txn().map_err(convert_error)?;
let old = self.tree.get(&txn, &key).map_err(convert_error)?;
let new =
crate::utils::increment(old.as_deref()).expect("utils::increment always returns Some");
self.tree
.put(&mut txn, &key, &&*new)
.map_err(convert_error)?;
txn.commit().map_err(convert_error)?;
Ok(new)
}
#[tracing::instrument(skip(self, prefix))]
fn scan_prefix<'a>(
&'a self,
prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> {
Box::new(
self.iter_from(&prefix, false)
.take_while(move |(key, _)| key.starts_with(&prefix)),
)
}
#[tracing::instrument(skip(self, prefix))]
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
let (tx, rx) = tokio::sync::oneshot::channel();
self.watchers
.write()
.unwrap()
.entry(prefix.to_vec())
.or_default()
.push(tx);
Box::pin(async move {
// Tx is never destroyed
rx.await.unwrap();
})
}
}

1
src/database/rooms.rs
View File

@ -863,6 +863,7 @@ impl Rooms {
if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) { if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) {
for word in body for word in body
.split_terminator(|c: char| !c.is_alphanumeric()) .split_terminator(|c: char| !c.is_alphanumeric())
.filter(|word| word.len() <= 50)
.map(str::to_lowercase) .map(str::to_lowercase)
{ {
let mut key = pdu.room_id.as_bytes().to_vec(); let mut key = pdu.room_id.as_bytes().to_vec();

87
src/database/sending.rs
View File

@ -81,10 +81,10 @@ pub enum SendingEventType {
pub struct Sending { pub struct Sending {
/// The state for a given state hash. /// The state for a given state hash.
pub(super) servername_educount: Arc<dyn Tree>, // EduCount: Count of last EDU sync pub(super) servername_educount: Arc<dyn Tree>, // EduCount: Count of last EDU sync
pub(super) servernamepduids: Arc<dyn Tree>, // ServernamePduId = (+ / $)SenderKey / ServerName / UserId + PduId pub(super) servernameevent_data: Arc<dyn Tree>, // ServernamEvent = (+ / $)SenderKey / ServerName / UserId + PduId / * (for edus), Data = EDU content
pub(super) servercurrentevents: Arc<dyn Tree>, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / (*)EduEvent pub(super) servercurrentevent_data: Arc<dyn Tree>, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / * (for edus), Data = EDU content
pub(super) maximum_requests: Arc<Semaphore>, pub(super) maximum_requests: Arc<Semaphore>,
pub sender: mpsc::UnboundedSender<Vec<u8>>, pub sender: mpsc::UnboundedSender<(Vec<u8>, Vec<u8>)>,
} }
enum TransactionStatus { enum TransactionStatus {
@ -97,7 +97,7 @@ impl Sending {
pub fn start_handler( pub fn start_handler(
&self, &self,
db: Arc<RwLock<Database>>, db: Arc<RwLock<Database>>,
mut receiver: mpsc::UnboundedReceiver<Vec<u8>>, mut receiver: mpsc::UnboundedReceiver<(Vec<u8>, Vec<u8>)>,
) { ) {
tokio::spawn(async move { tokio::spawn(async move {
let mut futures = FuturesUnordered::new(); let mut futures = FuturesUnordered::new();
@ -109,13 +109,12 @@ impl Sending {
let guard = db.read().await; let guard = db.read().await;
for (key, outgoing_kind, event) in for (key, outgoing_kind, event) in guard
guard
.sending .sending
.servercurrentevents .servercurrentevent_data
.iter() .iter()
.filter_map(|(key, _)| { .filter_map(|(key, v)| {
Self::parse_servercurrentevent(&key) Self::parse_servercurrentevent(&key, v)
.ok() .ok()
.map(|(k, e)| (key, k, e)) .map(|(k, e)| (key, k, e))
}) })
@ -129,7 +128,7 @@ impl Sending {
"Dropping some current events: {:?} {:?} {:?}", "Dropping some current events: {:?} {:?} {:?}",
key, outgoing_kind, event key, outgoing_kind, event
); );
guard.sending.servercurrentevents.remove(&key).unwrap(); guard.sending.servercurrentevent_data.remove(&key).unwrap();
continue; continue;
} }
@ -156,17 +155,17 @@ impl Sending {
let guard = db.read().await; let guard = db.read().await;
let prefix = outgoing_kind.get_prefix(); let prefix = outgoing_kind.get_prefix();
for (key, _) in guard.sending.servercurrentevents for (key, _) in guard.sending.servercurrentevent_data
.scan_prefix(prefix.clone()) .scan_prefix(prefix.clone())
{ {
guard.sending.servercurrentevents.remove(&key).unwrap(); guard.sending.servercurrentevent_data.remove(&key).unwrap();
} }
// Find events that have been added since starting the last request // Find events that have been added since starting the last request
let new_events = guard.sending.servernamepduids let new_events = guard.sending.servernameevent_data
.scan_prefix(prefix.clone()) .scan_prefix(prefix.clone())
.filter_map(|(k, _)| { .filter_map(|(k, v)| {
Self::parse_servercurrentevent(&k).ok().map(|ev| (ev, k)) Self::parse_servercurrentevent(&k, v).ok().map(|ev| (ev, k))
}) })
.take(30) .take(30)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
@ -175,9 +174,10 @@ impl Sending {
if !new_events.is_empty() { if !new_events.is_empty() {
// Insert pdus we found // Insert pdus we found
for (_, key) in &new_events { for (e, key) in &new_events {
guard.sending.servercurrentevents.insert(&key, &[]).unwrap(); let value = if let SendingEventType::Edu(value) = &e.1 { &**value } else { &[] };
guard.sending.servernamepduids.remove(&key).unwrap(); guard.sending.servercurrentevent_data.insert(&key, value).unwrap();
guard.sending.servernameevent_data.remove(&key).unwrap();
} }
drop(guard); drop(guard);
@ -205,8 +205,8 @@ impl Sending {
} }
}; };
}, },
Some(key) = receiver.next() => { Some((key, value)) = receiver.next() => {
if let Ok((outgoing_kind, event)) = Self::parse_servercurrentevent(&key) { if let Ok((outgoing_kind, event)) = Self::parse_servercurrentevent(&key, value) {
let guard = db.read().await; let guard = db.read().await;
if let Ok(Some(events)) = Self::select_events( if let Ok(Some(events)) = Self::select_events(
@ -267,18 +267,25 @@ impl Sending {
if retry { if retry {
// We retry the previous transaction // We retry the previous transaction
for (key, _) in db.sending.servercurrentevents.scan_prefix(prefix) { for (key, value) in db.sending.servercurrentevent_data.scan_prefix(prefix) {
if let Ok((_, e)) = Self::parse_servercurrentevent(&key) { if let Ok((_, e)) = Self::parse_servercurrentevent(&key, value) {
events.push(e); events.push(e);
} }
} }
} else { } else {
for (e, full_key) in new_events { for (e, full_key) in new_events {
db.sending.servercurrentevents.insert(&full_key, &[])?; let value = if let SendingEventType::Edu(value) = &e {
&**value
} else {
&[][..]
};
db.sending
.servercurrentevent_data
.insert(&full_key, value)?;
// If it was a PDU we have to unqueue it // If it was a PDU we have to unqueue it
// TODO: don't try to unqueue EDUs // TODO: don't try to unqueue EDUs
db.sending.servernamepduids.remove(&full_key)?; db.sending.servernameevent_data.remove(&full_key)?;
events.push(e); events.push(e);
} }
@ -380,8 +387,8 @@ impl Sending {
key.extend_from_slice(&senderkey); key.extend_from_slice(&senderkey);
key.push(0xff); key.push(0xff);
key.extend_from_slice(pdu_id); key.extend_from_slice(pdu_id);
self.servernamepduids.insert(&key, b"")?; self.servernameevent_data.insert(&key, &[])?;
self.sender.unbounded_send(key).unwrap(); self.sender.unbounded_send((key, vec![])).unwrap();
Ok(()) Ok(())
} }
@ -391,20 +398,19 @@ impl Sending {
let mut key = server.as_bytes().to_vec(); let mut key = server.as_bytes().to_vec();
key.push(0xff); key.push(0xff);
key.extend_from_slice(pdu_id); key.extend_from_slice(pdu_id);
self.servernamepduids.insert(&key, b"")?; self.servernameevent_data.insert(&key, &[])?;
self.sender.unbounded_send(key).unwrap(); self.sender.unbounded_send((key, vec![])).unwrap();
Ok(()) Ok(())
} }
#[tracing::instrument(skip(self, server, serialized))] #[tracing::instrument(skip(self, server, serialized))]
pub fn send_reliable_edu(&self, server: &ServerName, serialized: &[u8]) -> Result<()> { pub fn send_reliable_edu(&self, server: &ServerName, serialized: Vec<u8>) -> Result<()> {
let mut key = server.as_bytes().to_vec(); let mut key = server.as_bytes().to_vec();
key.push(0xff); key.push(0xff);
key.push(b'*'); key.push(b'*');
key.extend_from_slice(serialized); self.servernameevent_data.insert(&key, &serialized)?;
self.servernamepduids.insert(&key, b"")?; self.sender.unbounded_send((key, serialized)).unwrap();
self.sender.unbounded_send(key).unwrap();
Ok(()) Ok(())
} }
@ -415,8 +421,8 @@ impl Sending {
key.extend_from_slice(appservice_id.as_bytes()); key.extend_from_slice(appservice_id.as_bytes());
key.push(0xff); key.push(0xff);
key.extend_from_slice(pdu_id); key.extend_from_slice(pdu_id);
self.servernamepduids.insert(&key, b"")?; self.servernameevent_data.insert(&key, &[])?;
self.sender.unbounded_send(key).unwrap(); self.sender.unbounded_send((key, vec![])).unwrap();
Ok(()) Ok(())
} }
@ -451,7 +457,7 @@ impl Sending {
( (
kind.clone(), kind.clone(),
Error::bad_database( Error::bad_database(
"[Appservice] Event in servernamepduids not found in db.", "[Appservice] Event in servernameevent_data not found in db.",
), ),
) )
})? })?
@ -508,7 +514,7 @@ impl Sending {
( (
kind.clone(), kind.clone(),
Error::bad_database( Error::bad_database(
"[Push] Event in servernamepduids not found in db.", "[Push] Event in servernamevent_datas not found in db.",
), ),
) )
})?, })?,
@ -602,7 +608,7 @@ impl Sending {
( (
OutgoingKind::Normal(server.clone()), OutgoingKind::Normal(server.clone()),
Error::bad_database( Error::bad_database(
"[Normal] Event in servernamepduids not found in db.", "[Normal] Event in servernamevent_datas not found in db.",
), ),
) )
})?, })?,
@ -662,7 +668,10 @@ impl Sending {
} }
#[tracing::instrument(skip(key))] #[tracing::instrument(skip(key))]
fn parse_servercurrentevent(key: &[u8]) -> Result<(OutgoingKind, SendingEventType)> { fn parse_servercurrentevent(
key: &[u8],
value: Vec<u8>,
) -> Result<(OutgoingKind, SendingEventType)> {
// Appservices start with a plus // Appservices start with a plus
Ok::<_, Error>(if key.starts_with(b"+") { Ok::<_, Error>(if key.starts_with(b"+") {
let mut parts = key[1..].splitn(2, |&b| b == 0xff); let mut parts = key[1..].splitn(2, |&b| b == 0xff);
@ -680,7 +689,7 @@ impl Sending {
Error::bad_database("Invalid server string in server_currenttransaction") Error::bad_database("Invalid server string in server_currenttransaction")
})?), })?),
if event.starts_with(b"*") { if event.starts_with(b"*") {
SendingEventType::Edu(event[1..].to_vec()) SendingEventType::Edu(value.to_vec())
} else { } else {
SendingEventType::Pdu(event.to_vec()) SendingEventType::Pdu(event.to_vec())
}, },

3
src/error.rs
View File

@ -42,6 +42,9 @@ pub enum Error {
#[from] #[from]
source: rusqlite::Error, source: rusqlite::Error,
}, },
#[cfg(feature = "heed")]
#[error("There was a problem with the connection to the heed database: {error}")]
HeedError { error: String },
#[error("Could not generate an image.")] #[error("Could not generate an image.")]
ImageError { ImageError {
#[from] #[from]