Merge branch 'appservices' into 'master'

Appservices

Closes #29

See merge request famedly/conduit!11

.gitignore

@@ -2,3 +2,4 @@
 **/*.rs.bk
 
 Rocket.toml
+conduit.toml

Cargo.toml

@@ -14,22 +14,23 @@ edition = "2018"
 [dependencies]
 # Used to handle requests
 # TODO: This can become optional as soon as proper configs are supported
-#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", default-features = false, features = ["tls"] } # Used to handle requests
+rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "1f1f44f336e5a172361fc1860461bb03667b1ed2", features = ["tls"] } # Used to handle requests
-rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] }
+#rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] }
 
 # Used for matrix spec type definitions and helpers
-#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" }
+ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks", "unstable-exhaustive-types"], rev = "ee814aa84934530d76f5e4b275d739805b49bdef" }
-ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" }
+# ruma = { git = "https://github.com/DevinR528/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "unstable-join" }
-#ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] }
+# ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] }
 
 # Used when doing state resolution
-state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] }
+# state-res = { git = "https://github.com/timokoesters/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec"] }
-#state-res = { path = "../state-res", features = ["unstable-pre-spec"] }
+state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec", "gen-eventid"] }
+#state-res = { path = "../state-res", features = ["unstable-pre-spec", "gen-eventid"] }
 
-# Used for long polling
+# Used for long polling and federation sender, should be the same as rocket::tokio
-tokio = "0.2.22"
+tokio = { version = "0.2.23" }
 # Used for storing data permanently
-sled = { version = "0.34.4", default-features = false }
+sled = { version = "0.34.6", default-features = false }
 # Used for emitting log entries
 log = "0.4.11"
 # Used for rocket<->ruma conversions
@@ -39,25 +40,29 @@ directories = "3.0.1"
 # Used for number types for ruma
 js_int = "0.1.9"
 # Used for ruma wrapper
-serde_json = { version = "1.0.57", features = ["raw_value"] }
+serde_json = { version = "1.0.60", features = ["raw_value"] }
+# Used for appservice registration files
+serde_yaml = "0.8.14"
 # Used for pdu definition
-serde = "1.0.116"
+serde = "1.0.117"
 # Used for secure identifiers
 rand = "0.7.3"
 # Used to hash passwords
-rust-argon2 = "0.8.2"
+rust-argon2 = "0.8.3"
 # Used to send requests
-reqwest = "0.10.8"
+reqwest = "0.10.9"
 # Used for conduit::Error type
-thiserror = "1.0.20"
+thiserror = "1.0.22"
 # Used to generate thumbnails for images
-image = { version = "0.23.9", default-features = false, features = ["jpeg", "png", "gif"] }
+image = { version = "0.23.12", default-features = false, features = ["jpeg", "png", "gif"] }
 # Used to encode server public key
-base64 = "0.12.3"
+base64 = "0.13.0"
 # Used when hashing the state
-ring = "0.16.15"
+ring = "0.16.19"
 # Used when querying the SRV record of other servers
-trust-dns-resolver = "0.19.5"
+trust-dns-resolver = "0.19.6"
+# Used to find matching events for appservices
+regex = "1.4.2"
 
 [features]
 default = ["conduit_bin"]

DEPLOY.md

@@ -0,0 +1,164 @@
+# Deploying Conduit
+
+## Getting help
+
+If you run into any problems while setting up Conduit, write an email to `support@conduit.rs`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
+
+## Installing Conduit
+
+You have to download the binary that fits your machine. Run `uname -m` to see
+what you need. Now copy the right url:
+- x84_64: `https://conduit.rs/master/x86_64/conduit-bin`
+- armv7: `https://conduit.rs/master/armv7/conduit-bin`
+- armv8: `https://conduit.rs/master/armv8/conduit-bin`
+- arm: `https://conduit.rs/master/arm/conduit-bin`
+
+```bash
+$ sudo wget -O /usr/local/bin/matrix-conduit <url>
+$ sudo chmod +x /usr/local/bin/matrix-conduit
+```
+
+
+## Setting up a systemd service
+
+Now we'll set up a systemd service for Conduit, so it's easy to start/stop
+Conduit and set it to autostart when your server reboots. Simply paste the
+default systemd service you can find below into
+`/etc/systemd/system/conduit.service`.
+
+```systemd
+[Unit]
+Description=Conduit Matrix Server
+After=network.target
+
+[Service]
+Environment="CONDUIT_CONFIG=/etc/matrix-conduit/conduit.toml"
+User=root
+Group=root
+Restart=always
+ExecStart=/usr/local/bin/matrix-conduit
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Finally, run
+```bash
+$ sudo systemctl daemon-reload
+```
+
+
+## Creating the Conduit configuration file
+
+Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment to read it. You need to change at least the server name.**
+```toml
+[global]
+# The server_name is the name of this server. It is used as a suffix for user
+# and room ids. Examples: matrix.org, conduit.rs
+# The Conduit server needs to be reachable at https://your.server.name/ on port
+# 443 (client-server) and 8448 (federation) OR you can create /.well-known
+# files to redirect requests. See
+# https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client
+# and https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server
+# for more information
+
+# YOU NEED TO EDIT THIS
+#server_name = "your.server.name"
+
+# This is the only directory where Conduit will save its data
+database_path = "/var/lib/matrix-conduit/conduit_db"
+
+# The port Conduit will be running on. You need to set up a reverse proxy in
+# your web server (e.g. apache or nginx), so all requests to /_matrix on port
+# 443 and 8448 will be forwarded to the Conduit instance running on this port
+port = 6167
+
+# Max size for uploads
+max_request_size = 20_000_000 # in bytes
+
+# Disabling registration means no new users will be able to register on this server
+allow_registration = false
+
+# Disable encryption, so no new encrypted rooms can be created
+# Note: existing rooms will continue to work
+allow_encryption = true
+allow_federation = true
+
+#cache_capacity = 1073741824 # in bytes, 1024 * 1024 * 1024
+#max_concurrent_requests = 4 # How many requests Conduit sends to other servers at the same time
+#workers = 4 # default: cpu core count * 2
+
+address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy
+```
+
+
+## Setting up the Reverse Proxy
+
+This depends on whether you use Apache, Nginx or another web server.
+
+### Apache
+
+Create `/etc/apache2/sites-enabled/050-conduit.conf` and copy-and-paste this:
+```
+Listen 8448
+
+<VirtualHost *:443 *:8448>
+
+ServerName your.server.name # EDIT THIS
+
+AllowEncodedSlashes NoDecode
+ProxyPass /_matrix/ http://localhost:6167/
+ProxyPassReverse /_matrix/ http://localhost:6167/
+
+Include /etc/letsencrypt/options-ssl-apache.conf
+SSLCertificateFile /etc/letsencrypt/live/your.server.name/fullchain.pem # EDIT THIS
+SSLCertificateKeyFile /etc/letsencrypt/live/your.server.name/privkey.pem # EDIT THIS
+</VirtualHost>
+```
+
+**You need to make some edits again.** When you are done, run
+```bash
+$ sudo systemctl reload apache2
+```
+
+
+### Nginx
+
+If you use Nginx and not Apache, add the following server section inside the
+http section of `/etc/nginx/nginx.conf`
+```
+server {
+    listen 443;
+    listen 8448;
+    server_name your.server.name; # EDIT THIS
+
+    location /_matrix/ {
+        proxy_pass http://localhost:6167/_matrix/;
+    }
+}
+```
+**You need to make some edits again.** When you are done, run
+```bash
+$ sudo systemctl reload nginx
+```
+
+
+## SSL Certificate
+
+The easiest way to get an SSL certificate, if you don't have one already, is to install `certbot` and run this:
+```bash
+$ sudo certbot -d your.server.name
+```
+
+
+## You're done!
+
+Now you can start Conduit with:
+```bash
+$ sudo systemctl start conduit
+```
+
+Set it to start automatically when your system boots with:
+```bash
+$ sudo systemctl enable conduit
+```

DEPLOY_FROM_SOURCE.md

@@ -1,103 +0,0 @@
-# Deploy from source
-
-## Prerequisites
-
-Make sure you have `libssl-dev` and `pkg-config` installed and the [rust toolchain](https://rustup.rs) is available on at least on user.
-
-
-## Install Conduit
-
-```bash
-$ sudo useradd -m conduit
-$ sudo -u conduit cargo install --git "https://git.koesters.xyz/timo/conduit.git"
-```
-
-
-## Setup systemd service
-
-In this guide, we set up a systemd service for Conduit, so it's easy to start, stop Conduit and set it to autostart when your server reboots. Paste the default systemd service below and configure it to fit your setup (in /etc/systemd/system/conduit.service).
-
-```systemd
-[Unit]
-Description=Conduit
-After=network.target
-
-[Service]
-Environment="ROCKET_SERVER_NAME=YOURSERVERNAME.HERE" # EDIT THIS
-
-Environment="ROCKET_PORT=14004" # Reverse proxy port
-
-#Environment="ROCKET_MAX_REQUEST_SIZE=20000000" # in bytes
-#Environment="ROCKET_REGISTRATION_DISABLED=true"
-#Environment="ROCKET_ENCRYPTION_DISABLED=true"
-#Environment="ROCKET_FEDERATION_ENABLED=true"
-#Environment="ROCKET_LOG=normal" # Detailed logging
-
-Environment="ROCKET_ENV=production"
-User=conduit
-Group=conduit
-Type=simple
-Restart=always
-ExecStart=/home/conduit/.cargo/bin/conduit
-
-[Install]
-WantedBy=multi-user.target
-```
-
-Finally, run
-```bash
-$ sudo systemctl daemon-reload
-```
-
-
-## Setup Reverse Proxy
-
-This depends on whether you use Apache, Nginx or something else. For Apache it looks like this (in /etc/apache2/sites-enabled/050-conduit.conf):
-```
-<VirtualHost *:443>
-
-ServerName conduit.koesters.xyz # EDIT THIS
-
-AllowEncodedSlashes NoDecode
-
-ServerAlias conduit.koesters.xyz # EDIT THIS
-
-ProxyPreserveHost On
-ProxyRequests off
-AllowEncodedSlashes NoDecode
-ProxyPass / http://localhost:14004/ nocanon
-ProxyPassReverse / http://localhost:14004/ nocanon
-
-Include /etc/letsencrypt/options-ssl-apache.conf
-
-# EDIT THESE:
-SSLCertificateFile /etc/letsencrypt/live/conduit.koesters.xyz/fullchain.pem
-SSLCertificateKeyFile /etc/letsencrypt/live/conduit.koesters.xyz/privkey.pem
-</VirtualHost>
-```
-
-Then run
-```bash
-$ sudo systemctl reload apache2
-```
-
-
-## SSL Certificate
-
-The easiest way to get an SSL certificate for the domain is to install `certbot` and run this:
-```bash
-$ sudo certbot -d conduit.koesters.xyz
-```
-
-
-## You're done!
-
-Now you can start Conduit with
-```bash
-$ sudo systemctl start conduit
-```
-
-and set it to start automatically when your system boots with
-```bash
-$ sudo systemctl enable conduit
-```

README.md

@@ -17,13 +17,12 @@ example) and register on the `https://conduit.koesters.xyz` homeserver.
 
 #### How can I deploy my own?
 
-##### From source
+##### Deploy
 
-Clone the repo, build it with `cargo build --release` and call the binary
+Download or compile a conduit binary and call it from somewhere like a systemd script. [Read
-(target/release/conduit) from somewhere like a systemd script. [Read
+more](DEPLOY.md)
-more](DEPLOY_FROM_SOURCE.md)
 
-##### Using Docker
+##### Deploy using Docker
 
 Pull and run the docker image with
 

Rocket-example.toml

@@ -1,31 +0,0 @@
-[global]
-# The name of this server
-# Note: If server name != hostname, you need a .well-known file for federation
-# to work
-server_name = "your.server.name"
-
-port = 14004
-
-# Max size for uploads
-#max_request_size = 20_000_000 # in bytes, ~20 MB
-
-# Disable registration. No new users will be able to register on this server
-#registration_disabled = true
-
-# Disable encryption, so no new encrypted rooms can be created
-# Note: existing rooms will continue to work
-#encryption_disabled = true
-
-#federation_enabled = true
-
-# Default path is in this user's data
-#database_path = "/home/timo/MyConduitServer"
-
-# You should probably leave this at 0.0.0.0
-address = "0.0.0.0"
-
-# TLS support
-# Note: Not necessary when using a reverse proxy:
-#[global.tls]
-#certs = "/etc/letsencrypt/live/your.server.name/fullchain.pem"
-#key = "/etc/letsencrypt/live/your.server.name/privkey.pem"

conduit-example.toml

@@ -0,0 +1,37 @@
+[global]
+# The server_name is the name of this server. It is used as a suffix for user
+# and room ids. Examples: matrix.org, conduit.rs
+# The Conduit server needs to be reachable at https://your.server.name/ on port
+# 443 (client-server) and 8448 (federation) OR you can create /.well-known
+# files to redirect requests. See
+# https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client
+# and https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server
+# for more information
+
+# YOU NEED TO EDIT THIS
+#server_name = "your.server.name"
+
+# This is the only directly where Conduit will save its data
+database_path = "/var/lib/conduit/conduit.db"
+
+# The port Conduit will be running on. You need to set up a reverse proxy in
+# your web server (e.g. apache or nginx), so all requests to /_matrix on port
+# 443 and 8448 will be forwarded to the Conduit instance running on this port
+port = 6167
+
+# Max size for uploads
+max_request_size = 20_000_000 # in bytes
+
+# Disable registration. No new users will be able to register on this server
+#allow_registration = true
+
+# Disable encryption, so no new encrypted rooms can be created
+# Note: existing rooms will continue to work
+#allow_encryption = true
+#allow_federation = false
+
+#cache_capacity = 1073741824 # in bytes, 1024 * 1024 * 1024
+#max_concurrent_requests = 4 # How many requests Conduit sends to other servers at the same time
+#workers = 4 # default: cpu core count * 2
+
+address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy

src/appservice_server.rs

@@ -0,0 +1,104 @@
+use crate::{utils, Error, Result};
+use http::header::{HeaderValue, CONTENT_TYPE};
+use log::{info, warn};
+use ruma::api::OutgoingRequest;
+use std::{
+    convert::{TryFrom, TryInto},
+    fmt::Debug,
+    time::Duration,
+};
+
+pub async fn send_request<T: OutgoingRequest>(
+    globals: &crate::database::globals::Globals,
+    registration: serde_yaml::Value,
+    request: T,
+) -> Result<T::IncomingResponse>
+where
+    T: Debug,
+{
+    let destination = registration.get("url").unwrap().as_str().unwrap();
+    let hs_token = registration.get("hs_token").unwrap().as_str().unwrap();
+
+    let mut http_request = request
+        .try_into_http_request(&destination, Some(""))
+        .unwrap();
+
+    let mut parts = http_request.uri().clone().into_parts();
+    let old_path_and_query = parts.path_and_query.unwrap().as_str().to_owned();
+    let symbol = if old_path_and_query.contains("?") {
+        "&"
+    } else {
+        "?"
+    };
+
+    parts.path_and_query = Some(
+        (old_path_and_query + symbol + "access_token=" + hs_token)
+            .parse()
+            .unwrap(),
+    );
+    *http_request.uri_mut() = parts.try_into().expect("our manipulation is always valid");
+
+    http_request.headers_mut().insert(
+        CONTENT_TYPE,
+        HeaderValue::from_str("application/json").unwrap(),
+    );
+
+    let mut reqwest_request = reqwest::Request::try_from(http_request)
+        .expect("all http requests are valid reqwest requests");
+
+    *reqwest_request.timeout_mut() = Some(Duration::from_secs(30));
+
+    let url = reqwest_request.url().clone();
+    let reqwest_response = globals.reqwest_client().execute(reqwest_request).await;
+
+    // Because reqwest::Response -> http::Response is complicated:
+    match reqwest_response {
+        Ok(mut reqwest_response) => {
+            let status = reqwest_response.status();
+            let mut http_response = http::Response::builder().status(status);
+            let headers = http_response.headers_mut().unwrap();
+
+            for (k, v) in reqwest_response.headers_mut().drain() {
+                if let Some(key) = k {
+                    headers.insert(key, v);
+                }
+            }
+
+            let status = reqwest_response.status();
+
+            let body = reqwest_response
+                .bytes()
+                .await
+                .unwrap_or_else(|e| {
+                    warn!("server error: {}", e);
+                    Vec::new().into()
+                }) // TODO: handle timeout
+                .into_iter()
+                .collect::<Vec<_>>();
+
+            if status != 200 {
+                warn!(
+                    "Appservice returned bad response {} {}\n{}\n{:?}",
+                    destination,
+                    status,
+                    url,
+                    utils::string_from_bytes(&body)
+                );
+            }
+
+            let response = T::IncomingResponse::try_from(
+                http_response
+                    .body(body)
+                    .expect("reqwest body is valid http body"),
+            );
+            response.map_err(|_| {
+                warn!(
+                    "Appservice returned invalid response bytes {}\n{}",
+                    destination, url
+                );
+                Error::BadServerResponse("Server returned bad response.")
+            })
+        }
+        Err(e) => Err(e.into()),
+    }
+}

src/client_server/account.rs

@@ -15,13 +15,10 @@ use ruma::{
         },
     },
     events::{
-        room::canonical_alias,
+        room::{
-        room::guest_access,
+            canonical_alias, guest_access, history_visibility, join_rules, member, message, name,
-        room::history_visibility,
+            topic,
-        room::join_rules,
+        },
-        room::member,
-        room::name,
-        room::{message, topic},
         EventType,
     },
     RoomAliasId, RoomId, RoomVersionId, UserId,
@@ -89,7 +86,7 @@ pub async fn register_route(
     db: State<'_, Database>,
     body: Ruma<register::Request<'_>>,
 ) -> ConduitResult<register::Response> {
-    if db.globals.registration_disabled() {
+    if !db.globals.allow_registration() {
         return Err(Error::BadRequest(
             ErrorKind::Forbidden,
             "Registration has been disabled.",
@@ -142,18 +139,20 @@ pub async fn register_route(
         auth_error: None,
     };
 
-    if let Some(auth) = &body.auth {
+    if !body.from_appservice {
-        let (worked, uiaainfo) =
+        if let Some(auth) = &body.auth {
-            db.uiaa
+            let (worked, uiaainfo) =
-                .try_auth(&user_id, "".into(), auth, &uiaainfo, &db.users, &db.globals)?;
+                db.uiaa
-        if !worked {
+                    .try_auth(&user_id, "".into(), auth, &uiaainfo, &db.users, &db.globals)?;
+            if !worked {
+                return Err(Error::Uiaa(uiaainfo));
+            }
+        // Success!
+        } else {
+            uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
+            db.uiaa.create(&user_id, "".into(), &uiaainfo)?;
             return Err(Error::Uiaa(uiaainfo));
         }
-    // Success!
-    } else {
-        uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&user_id, "".into(), &uiaainfo)?;
-        return Err(Error::Uiaa(uiaainfo));
     }
 
     if missing_username {
@@ -244,6 +243,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         // 2. Make conduit bot join
@@ -268,6 +268,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         // 3. Power levels
@@ -305,6 +306,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         // 4.1 Join Rules
@@ -325,6 +327,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         // 4.2 History Visibility
@@ -347,6 +350,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         // 4.3 Guest Access
@@ -367,6 +371,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         // 6. Events implied by name and topic
@@ -389,6 +394,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         db.rooms.build_and_append_pdu(
@@ -408,6 +414,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         // Room alias
@@ -433,6 +440,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?;
@@ -459,6 +467,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
         db.rooms.build_and_append_pdu(
             PduBuilder {
@@ -481,6 +490,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         // Send welcome message
@@ -495,6 +505,7 @@ pub async fn register_route(
                             body: "Thanks for trying out Conduit! This software is still in development, so expect many bugs and missing features. If you have federation enabled, you can join the Conduit chat room by typing <code>/join #conduit:matrix.org</code>. <strong>Important: Please don't join any other Matrix rooms over federation without permission from the room's admins.</strong> Some actions might trigger bugs in other server implementations, breaking the chat for everyone else.".to_owned(),
                         }),
                         relates_to: None,
+                        new_content: None,
                     },
                 ))
                 .expect("event is valid, we just created it"),
@@ -508,6 +519,7 @@ pub async fn register_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+        &db.appservice,
         )?;
     }
 
@@ -683,6 +695,7 @@ pub async fn deactivate_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
     }
 

src/client_server/alias.rs

@@ -1,7 +1,8 @@
 use super::State;
-use crate::{server_server, ConduitResult, Database, Error, Ruma};
+use crate::{ConduitResult, Database, Error, Ruma};
 use ruma::{
     api::{
+        appservice,
         client::{
             error::ErrorKind,
             r0::alias::{create_alias, delete_alias, get_alias},
@@ -65,23 +66,51 @@ pub async fn get_alias_helper(
     room_alias: &RoomAliasId,
 ) -> ConduitResult<get_alias::Response> {
     if room_alias.server_name() != db.globals.server_name() {
-        let response = server_server::send_request(
+        let response = db
-            &db.globals,
+            .sending
-            room_alias.server_name().to_owned(),
+            .send_federation_request(
-            federation::query::get_room_information::v1::Request { room_alias },
+                &db.globals,
-        )
+                room_alias.server_name().to_owned(),
-        .await?;
+                federation::query::get_room_information::v1::Request { room_alias },
+            )
+            .await?;
 
         return Ok(get_alias::Response::new(response.room_id, response.servers).into());
     }
 
-    let room_id = db
+    let mut room_id = None;
-        .rooms
+    match db.rooms.id_from_alias(&room_alias)? {
-        .id_from_alias(&room_alias)?
+        Some(r) => room_id = Some(r),
-        .ok_or(Error::BadRequest(
+        None => {
-            ErrorKind::NotFound,
+            for (_id, registration) in db.appservice.iter_all().filter_map(|r| r.ok()) {
-            "Room with alias not found.",
+                if db
-        ))?;
+                    .sending
+                    .send_appservice_request(
+                        &db.globals,
+                        registration,
+                        appservice::query::query_room_alias::v1::Request { room_alias },
+                    )
+                    .await
+                    .is_ok()
+                {
+                    room_id = Some(db.rooms.id_from_alias(&room_alias)?.ok_or_else(|| {
+                        Error::bad_config("Appservice lied to us. Room does not exist.")
+                    })?);
+                    break;
+                }
+            }
+        }
+    };
+
+    let room_id = match room_id {
+        Some(room_id) => room_id,
+        None => {
+            return Err(Error::BadRequest(
+                ErrorKind::NotFound,
+                "Room with alias not found.",
+            ))
+        }
+    };
 
     Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_owned()]).into())
 }

src/client_server/backup.rs

@@ -107,7 +107,7 @@ pub async fn get_backup_route(
 )]
 pub async fn delete_backup_route(
     db: State<'_, Database>,
-    body: Ruma<delete_backup::Request>,
+    body: Ruma<delete_backup::Request<'_>>,
 ) -> ConduitResult<delete_backup::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
@@ -158,7 +158,7 @@ pub async fn add_backup_keys_route(
 )]
 pub async fn add_backup_key_sessions_route(
     db: State<'_, Database>,
-    body: Ruma<add_backup_key_sessions::Request>,
+    body: Ruma<add_backup_key_sessions::Request<'_>>,
 ) -> ConduitResult<add_backup_key_sessions::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
@@ -189,7 +189,7 @@ pub async fn add_backup_key_sessions_route(
 )]
 pub async fn add_backup_key_session_route(
     db: State<'_, Database>,
-    body: Ruma<add_backup_key_session::Request>,
+    body: Ruma<add_backup_key_session::Request<'_>>,
 ) -> ConduitResult<add_backup_key_session::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
@@ -232,7 +232,7 @@ pub async fn get_backup_keys_route(
 )]
 pub async fn get_backup_key_sessions_route(
     db: State<'_, Database>,
-    body: Ruma<get_backup_key_sessions::Request>,
+    body: Ruma<get_backup_key_sessions::Request<'_>>,
 ) -> ConduitResult<get_backup_key_sessions::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
@@ -249,13 +249,19 @@ pub async fn get_backup_key_sessions_route(
 )]
 pub async fn get_backup_key_session_route(
     db: State<'_, Database>,
-    body: Ruma<get_backup_key_session::Request>,
+    body: Ruma<get_backup_key_session::Request<'_>>,
 ) -> ConduitResult<get_backup_key_session::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    let key_data =
+    let key_data = db
-        db.key_backups
+        .key_backups
-            .get_session(&sender_user, &body.version, &body.room_id, &body.session_id)?;
+        .get_session(&sender_user, &body.version, &body.room_id, &body.session_id)?
+        .ok_or_else(|| {
+            Error::BadRequest(
+                ErrorKind::NotFound,
+                "Backup key not found for this user's session.",
+            )
+        })?;
 
     Ok(get_backup_key_session::Response { key_data }.into())
 }
@@ -266,7 +272,7 @@ pub async fn get_backup_key_session_route(
 )]
 pub async fn delete_backup_keys_route(
     db: State<'_, Database>,
-    body: Ruma<delete_backup_keys::Request>,
+    body: Ruma<delete_backup_keys::Request<'_>>,
 ) -> ConduitResult<delete_backup_keys::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
@@ -288,7 +294,7 @@ pub async fn delete_backup_keys_route(
 )]
 pub async fn delete_backup_key_sessions_route(
     db: State<'_, Database>,
-    body: Ruma<delete_backup_key_sessions::Request>,
+    body: Ruma<delete_backup_key_sessions::Request<'_>>,
 ) -> ConduitResult<delete_backup_key_sessions::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
@@ -310,7 +316,7 @@ pub async fn delete_backup_key_sessions_route(
 )]
 pub async fn delete_backup_key_session_route(
     db: State<'_, Database>,
-    body: Ruma<delete_backup_key_session::Request>,
+    body: Ruma<delete_backup_key_session::Request<'_>>,
 ) -> ConduitResult<delete_backup_key_session::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 

src/client_server/capabilities.rs

@@ -22,11 +22,11 @@ pub async fn get_capabilities_route() -> ConduitResult<get_capabilities::Respons
 
     Ok(get_capabilities::Response {
         capabilities: get_capabilities::Capabilities {
-            change_password: None, // None means it is possible
+            change_password: get_capabilities::ChangePasswordCapability::default(), // enabled by default
-            room_versions: Some(get_capabilities::RoomVersionsCapability {
+            room_versions: get_capabilities::RoomVersionsCapability {
-                default: "6".to_owned(),
+                default: RoomVersionId::Version6,
                 available,
-            }),
+            },
             custom_capabilities: BTreeMap::new(),
         },
     }

src/client_server/config.rs

@@ -6,7 +6,7 @@ use ruma::{
         r0::config::{get_global_account_data, set_global_account_data},
     },
     events::{custom::CustomEventContent, BasicEvent},
-    Raw,
+    serde::Raw,
 };
 
 #[cfg(feature = "conduit_bin")]

src/client_server/directory.rs

@@ -1,5 +1,5 @@
 use super::State;
-use crate::{server_server, ConduitResult, Database, Error, Result, Ruma};
+use crate::{ConduitResult, Database, Error, Result, Ruma};
 use log::info;
 use ruma::{
     api::{
@@ -15,14 +15,13 @@ use ruma::{
         },
         federation,
     },
-    directory::Filter,
+    directory::{Filter, IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk, RoomNetwork},
-    directory::RoomNetwork,
-    directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk},
     events::{
         room::{avatar, canonical_alias, guest_access, history_visibility, name, topic},
         EventType,
     },
-    Raw, ServerName,
+    serde::Raw,
+    ServerName,
 };
 
 #[cfg(feature = "conduit_bin")]
@@ -85,7 +84,13 @@ pub async fn set_room_visibility_route(
 ) -> ConduitResult<set_room_visibility::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    match body.visibility {
+    match &body.visibility {
+        room::Visibility::_Custom(_s) => {
+            return Err(Error::BadRequest(
+                ErrorKind::InvalidParam,
+                "Room visibility type is not supported.",
+            ));
+        }
         room::Visibility::Public => {
             db.rooms.set_public(&body.room_id, true)?;
             info!("{} made {} public", sender_user, body.room_id);
@@ -128,19 +133,21 @@ pub async fn get_public_rooms_filtered_helper(
         .clone()
         .filter(|server| *server != db.globals.server_name().as_str())
     {
-        let response = server_server::send_request(
+        let response = db
-            &db.globals,
+            .sending
-            other_server.to_owned(),
+            .send_federation_request(
-            federation::directory::get_public_rooms_filtered::v1::Request {
+                &db.globals,
-                limit,
+                other_server.to_owned(),
-                since: since.as_deref(),
+                federation::directory::get_public_rooms_filtered::v1::Request {
-                filter: Filter {
+                    limit,
-                    generic_search_term: filter.generic_search_term.as_deref(),
+                    since: since.as_deref(),
+                    filter: Filter {
+                        generic_search_term: filter.generic_search_term.as_deref(),
+                    },
+                    room_network: RoomNetwork::Matrix,
                 },
-                room_network: RoomNetwork::Matrix,
+            )
-            },
+            .await?;
-        )
-        .await?;
 
         return Ok(get_public_rooms_filtered::Response {
             chunk: response
@@ -296,7 +303,9 @@ pub async fn get_public_rooms_filtered_helper(
                                 .url,
                             )
                         })
-                        .transpose()?,
+                        .transpose()?
+                        // url is now an Option<String> so we must flatten
+                        .flatten(),
                 };
                 Ok(chunk)
             })

src/client_server/filter.rs

@@ -9,10 +9,10 @@ pub async fn get_filter_route() -> ConduitResult<get_filter::Response> {
     // TODO
     Ok(get_filter::Response::new(filter::IncomingFilterDefinition {
         event_fields: None,
-        event_format: None,
+        event_format: filter::EventFormat::default(),
-        account_data: None,
+        account_data: filter::IncomingFilter::default(),
-        room: None,
+        room: filter::IncomingRoomFilter::default(),
-        presence: None,
+        presence: filter::IncomingFilter::default(),
     })
     .into())
 }

src/client_server/keys.rs

@@ -11,7 +11,7 @@ use ruma::{
             uiaa::{AuthFlow, UiaaInfo},
         },
     },
-    encryption::IncomingUnsignedDeviceInfo,
+    encryption::UnsignedDeviceInfo,
 };
 use std::collections::{BTreeMap, HashSet};
 
@@ -24,7 +24,7 @@ use rocket::{get, post};
 )]
 pub async fn upload_keys_route(
     db: State<'_, Database>,
-    body: Ruma<upload_keys::Request<'_>>,
+    body: Ruma<upload_keys::Request>,
 ) -> ConduitResult<upload_keys::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
     let sender_device = body.sender_device.as_ref().expect("user is authenticated");
@@ -94,7 +94,7 @@ pub async fn get_keys_route(
                             Error::bad_database("all_device_keys contained nonexistent device.")
                         })?;
 
-                    keys.unsigned = IncomingUnsignedDeviceInfo {
+                    keys.unsigned = UnsignedDeviceInfo {
                         device_display_name: metadata.display_name,
                     };
 
@@ -113,7 +113,7 @@ pub async fn get_keys_route(
                         ),
                     )?;
 
-                    keys.unsigned = IncomingUnsignedDeviceInfo {
+                    keys.unsigned = UnsignedDeviceInfo {
                         device_display_name: metadata.display_name,
                     };
 

src/client_server/media.rs

@@ -1,7 +1,5 @@
 use super::State;
-use crate::{
+use crate::{database::media::FileMeta, utils, ConduitResult, Database, Error, Ruma};
-    database::media::FileMeta, server_server, utils, ConduitResult, Database, Error, Ruma,
-};
 use ruma::api::client::{
     error::ErrorKind,
     r0::media::{create_content, get_content, get_content_thumbnail, get_media_config},
@@ -39,13 +37,17 @@ pub async fn create_content_route(
     db.media.create(
         mxc.clone(),
         &body.filename.as_deref(),
-        &body.content_type,
+        &body.content_type.as_deref(),
         &body.file,
     )?;
 
     db.flush().await?;
 
-    Ok(create_content::Response { content_uri: mxc }.into())
+    Ok(create_content::Response {
+        content_uri: mxc,
+        blurhash: None,
+    }
+    .into())
 }
 
 #[cfg_attr(
@@ -67,25 +69,27 @@ pub async fn get_content_route(
         Ok(get_content::Response {
             file,
             content_type,
-            content_disposition: filename.unwrap_or_default(), // TODO: Spec says this should be optional
+            content_disposition: filename,
         }
         .into())
     } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
-        let get_content_response = server_server::send_request(
+        let get_content_response = db
-            &db.globals,
+            .sending
-            body.server_name.clone(),
+            .send_federation_request(
-            get_content::Request {
+                &db.globals,
-                allow_remote: false,
+                body.server_name.clone(),
-                server_name: &body.server_name,
+                get_content::Request {
-                media_id: &body.media_id,
+                    allow_remote: false,
-            },
+                    server_name: &body.server_name,
-        )
+                    media_id: &body.media_id,
-        .await?;
+                },
+            )
+            .await?;
 
         db.media.create(
             mxc,
-            &Some(&get_content_response.content_disposition),
+            &get_content_response.content_disposition.as_deref(),
-            &get_content_response.content_type,
+            &get_content_response.content_type.as_deref(),
             &get_content_response.file,
         )?;
 
@@ -118,19 +122,21 @@ pub async fn get_content_thumbnail_route(
     )? {
         Ok(get_content_thumbnail::Response { file, content_type }.into())
     } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
-        let get_thumbnail_response = server_server::send_request(
+        let get_thumbnail_response = db
-            &db.globals,
+            .sending
-            body.server_name.clone(),
+            .send_federation_request(
-            get_content_thumbnail::Request {
+                &db.globals,
-                allow_remote: false,
+                body.server_name.clone(),
-                height: body.height,
+                get_content_thumbnail::Request {
-                width: body.width,
+                    allow_remote: false,
-                method: body.method,
+                    height: body.height,
-                server_name: &body.server_name,
+                    width: body.width,
-                media_id: &body.media_id,
+                    method: body.method,
-            },
+                    server_name: &body.server_name,
-        )
+                    media_id: &body.media_id,
-        .await?;
+                },
+            )
+            .await?;
 
         db.media.upload_thumbnail(
             mxc,

src/client_server/membership.rs

@@ -2,7 +2,7 @@ use super::State;
 use crate::{
     client_server,
     pdu::{PduBuilder, PduEvent},
-    server_server, utils, ConduitResult, Database, Error, Result, Ruma,
+    utils, ConduitResult, Database, Error, Result, Ruma,
 };
 use log::warn;
 use ruma::{
@@ -17,13 +17,15 @@ use ruma::{
         },
         federation,
     },
-    events::pdu::Pdu,
+    events::{pdu::Pdu, room::member, EventType},
-    events::{room::member, EventType},
+    serde::{to_canonical_value, CanonicalJsonObject, Raw},
-    EventId, Raw, RoomId, RoomVersionId, ServerName, UserId,
+    EventId, RoomId, RoomVersionId, ServerName, UserId,
 };
 use state_res::StateEvent;
 use std::{
-    collections::BTreeMap, collections::HashMap, collections::HashSet, convert::TryFrom, iter,
+    collections::{BTreeMap, HashMap, HashSet},
+    convert::TryFrom,
+    iter,
     sync::Arc,
 };
 
@@ -126,6 +128,7 @@ pub async fn leave_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     db.flush().await?;
@@ -165,6 +168,7 @@ pub async fn invite_user_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         db.flush().await?;
@@ -220,6 +224,7 @@ pub async fn kick_user_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     db.flush().await?;
@@ -279,6 +284,7 @@ pub async fn ban_user_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     db.flush().await?;
@@ -330,6 +336,7 @@ pub async fn unban_user_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     db.flush().await?;
@@ -394,9 +401,10 @@ pub async fn get_member_events_route(
     Ok(get_member_events::Response {
         chunk: db
             .rooms
-            .room_state_type(&body.room_id, &EventType::RoomMember)?
+            .room_state_full(&body.room_id)?
-            .values()
+            .iter()
-            .map(|pdu| pdu.to_member_event())
+            .filter(|(key, _)| key.0 == EventType::RoomMember)
+            .map(|(_, pdu)| pdu.to_member_event())
             .collect(),
     }
     .into())
@@ -456,16 +464,18 @@ async fn join_room_by_id_helper(
         ));
 
         for remote_server in servers {
-            let make_join_response = server_server::send_request(
+            let make_join_response = db
-                &db.globals,
+                .sending
-                remote_server.clone(),
+                .send_federation_request(
-                federation::membership::create_join_event_template::v1::Request {
+                    &db.globals,
-                    room_id,
+                    remote_server.clone(),
-                    user_id: sender_user,
+                    federation::membership::create_join_event_template::v1::Request {
-                    ver: &[RoomVersionId::Version5, RoomVersionId::Version6],
+                        room_id,
-                },
+                        user_id: sender_user,
-            )
+                        ver: &[RoomVersionId::Version5, RoomVersionId::Version6],
-            .await;
+                    },
+                )
+                .await;
 
             make_join_response_and_server = make_join_response.map(|r| (r, remote_server));
 
@@ -476,30 +486,25 @@ async fn join_room_by_id_helper(
 
         let (make_join_response, remote_server) = make_join_response_and_server?;
 
-        let mut join_event_stub_value =
+        let mut join_event_stub =
-            serde_json::from_str::<serde_json::Value>(make_join_response.event.json().get())
+            serde_json::from_str::<CanonicalJsonObject>(make_join_response.event.json().get())
                 .map_err(|_| {
                     Error::BadServerResponse("Invalid make_join event json received from server.")
                 })?;
 
-        let join_event_stub =
-            join_event_stub_value
-                .as_object_mut()
-                .ok_or(Error::BadServerResponse(
-                    "Invalid make join event object received from server.",
-                ))?;
-
         join_event_stub.insert(
             "origin".to_owned(),
-            db.globals.server_name().to_owned().to_string().into(),
+            to_canonical_value(db.globals.server_name())
+                .map_err(|_| Error::bad_database("Invalid server name found"))?,
         );
         join_event_stub.insert(
             "origin_server_ts".to_owned(),
-            utils::millis_since_unix_epoch().into(),
+            to_canonical_value(utils::millis_since_unix_epoch())
+                .expect("Timestamp is valid js_int value"),
         );
         join_event_stub.insert(
             "content".to_owned(),
-            serde_json::to_value(member::MemberEventContent {
+            to_canonical_value(member::MemberEventContent {
                 membership: member::MembershipState::Join,
                 displayname: db.users.displayname(&sender_user)?,
                 avatar_url: db.users.avatar_url(&sender_user)?,
@@ -509,57 +514,63 @@ async fn join_room_by_id_helper(
             .expect("event is valid, we just created it"),
         );
 
+        // We don't leave the event id in the pdu because that's only allowed in v1 or v2 rooms
+        join_event_stub.remove("event_id");
+
+        // In order to create a compatible ref hash (EventID) the `hashes` field needs to be present
+        ruma::signatures::hash_and_sign_event(
+            db.globals.server_name().as_str(),
+            db.globals.keypair(),
+            &mut join_event_stub,
+            &RoomVersionId::Version6,
+        )
+        .expect("event is valid, we just created it");
+
         // Generate event id
         let event_id = EventId::try_from(&*format!(
             "${}",
-            ruma::signatures::reference_hash(&join_event_stub_value)
+            ruma::signatures::reference_hash(&join_event_stub, &RoomVersionId::Version6)
                 .expect("ruma can calculate reference hashes")
         ))
         .expect("ruma's reference hashes are valid event ids");
 
-        // We don't leave the event id into the pdu because that's only allowed in v1 or v2 rooms
-        let join_event_stub = join_event_stub_value.as_object_mut().unwrap();
-        join_event_stub.remove("event_id");
-
-        ruma::signatures::hash_and_sign_event(
-            db.globals.server_name().as_str(),
-            db.globals.keypair(),
-            &mut join_event_stub_value,
-        )
-        .expect("event is valid, we just created it");
-
         // Add event_id back
-        let join_event_stub = join_event_stub_value.as_object_mut().unwrap();
+        join_event_stub.insert(
-        join_event_stub.insert("event_id".to_owned(), event_id.to_string().into());
+            "event_id".to_owned(),
+            to_canonical_value(&event_id).expect("EventId is a valid CanonicalJsonValue"),
+        );
 
         // It has enough fields to be called a proper event now
-        let join_event = join_event_stub_value;
+        let join_event = join_event_stub;
 
-        let send_join_response = server_server::send_request(
+        let send_join_response = db
-            &db.globals,
+            .sending
-            remote_server.clone(),
+            .send_federation_request(
-            federation::membership::create_join_event::v2::Request {
+                &db.globals,
-                room_id,
+                remote_server.clone(),
-                event_id: &event_id,
+                federation::membership::create_join_event::v2::Request {
-                pdu_stub: PduEvent::convert_to_outgoing_federation_event(join_event.clone()),
+                    room_id,
-            },
+                    event_id: &event_id,
-        )
+                    pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()),
-        .await?;
+                },
+            )
+            .await?;
 
-        let add_event_id = |pdu: &Raw<Pdu>| {
+        let add_event_id = |pdu: &Raw<Pdu>| -> Result<(EventId, CanonicalJsonObject)> {
             let mut value = serde_json::from_str(pdu.json().get())
                 .expect("converting raw jsons to values always works");
             let event_id = EventId::try_from(&*format!(
                 "${}",
-                ruma::signatures::reference_hash(&value)
+                ruma::signatures::reference_hash(&value, &RoomVersionId::Version6)
                     .expect("ruma can calculate reference hashes")
             ))
             .expect("ruma's reference hashes are valid event ids");
 
-            value
+            value.insert(
-                .as_object_mut()
+                "event_id".to_owned(),
-                .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))?
+                to_canonical_value(&event_id)
-                .insert("event_id".to_owned(), event_id.to_string().into());
+                    .expect("a valid EventId can be converted to CanonicalJsonValue"),
+            );
 
             Ok((event_id, value))
         };
@@ -568,7 +579,7 @@ async fn join_room_by_id_helper(
 
         let state_events = room_state
             .clone()
-            .map(|pdu: Result<(EventId, serde_json::Value)>| Ok(pdu?.0))
+            .map(|pdu: Result<(EventId, CanonicalJsonObject)>| Ok(pdu?.0))
             .chain(iter::once(Ok(event_id.clone()))) // Add join event we just created
             .collect::<Result<HashSet<EventId>>>()?;
 
@@ -583,11 +594,11 @@ async fn join_room_by_id_helper(
             .chain(iter::once(Ok((event_id, join_event)))) // Add join event we just created
             .map(|r| {
                 let (event_id, value) = r?;
-                serde_json::from_value::<StateEvent>(value.clone())
+                state_res::StateEvent::from_id_canon_obj(event_id.clone(), value.clone())
                     .map(|ev| (event_id, Arc::new(ev)))
                     .map_err(|e| {
-                        warn!("{}: {}", value, e);
+                        warn!("{:?}: {}", value, e);
-                        Error::BadServerResponse("Invalid PDU bytes in send_join response.")
+                        Error::BadServerResponse("Invalid PDU in send_join response.")
                     })
             })
             .collect::<Result<BTreeMap<EventId, Arc<StateEvent>>>>()?;
@@ -595,7 +606,7 @@ async fn join_room_by_id_helper(
         let control_events = event_map
             .values()
             .filter(|pdu| pdu.is_power_event())
-            .map(|pdu| pdu.event_id().clone())
+            .map(|pdu| pdu.event_id())
             .collect::<Vec<_>>();
 
         // These events are not guaranteed to be sorted but they are resolved according to spec
@@ -623,7 +634,9 @@ async fn join_room_by_id_helper(
         .expect("iterative auth check failed on resolved events");
 
         // This removes the control events that failed auth, leaving the resolved
-        // to be mainline sorted
+        // to be mainline sorted. In the actual `state_res::StateResolution::resolve`
+        // function both are removed since these are all events we don't know of
+        // we must keep track of everything to add to our DB.
         let events_to_sort = event_map
             .keys()
             .filter(|id| {
@@ -673,7 +686,7 @@ async fn join_room_by_id_helper(
             pdu_id.extend_from_slice(&count.to_be_bytes());
             db.rooms.append_pdu(
                 &PduEvent::from(&**pdu),
-                &serde_json::to_value(&**pdu).expect("PDU is valid value"),
+                utils::to_canonical_object(&**pdu).expect("Pdu is valid canonical object"),
                 count,
                 pdu_id.clone().into(),
                 &db.globals,
@@ -686,7 +699,7 @@ async fn join_room_by_id_helper(
             }
         }
 
-        db.rooms.force_state(room_id, state)?;
+        db.rooms.force_state(room_id, state, &db.globals)?;
     } else {
         let event = member::MemberEventContent {
             membership: member::MembershipState::Join,
@@ -710,6 +723,7 @@ async fn join_room_by_id_helper(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
     }
 

src/client_server/message.rs

@@ -22,7 +22,7 @@ pub async fn send_message_event_route(
     body: Ruma<send_message_event::Request<'_>>,
 ) -> ConduitResult<send_message_event::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-    let sender_device = body.sender_device.as_ref().expect("user is authenticated");
+    let sender_device = body.sender_device.as_deref();
 
     // Check if this is a new transaction id
     if let Some(response) =
@@ -69,6 +69,7 @@ pub async fn send_message_event_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     db.transaction_ids.add_txnid(

src/client_server/presence.rs

@@ -32,7 +32,7 @@ pub async fn set_presence_route(
                             .try_into()
                             .expect("time is valid"),
                     ),
-                    presence: body.presence,
+                    presence: body.presence.clone(),
                     status_msg: body.status_msg.clone(),
                 },
                 sender: sender_user.clone(),

src/client_server/profile.rs

@@ -8,7 +8,7 @@ use ruma::{
         },
     },
     events::EventType,
-    Raw,
+    serde::Raw,
 };
 
 #[cfg(feature = "conduit_bin")]
@@ -67,6 +67,7 @@ pub async fn set_displayname_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         // Presence update
@@ -163,6 +164,7 @@ pub async fn set_avatar_url_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
 
         // Presence update

src/client_server/push.rs

@@ -1,16 +1,22 @@
 use super::State;
 use crate::{ConduitResult, Database, Error, Ruma};
-use log::warn;
 use ruma::{
     api::client::{
         error::ErrorKind,
-        r0::push::{get_pushers, get_pushrules_all, set_pushrule, set_pushrule_enabled},
+        r0::push::{
+            delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions, get_pushrule_enabled,
+            get_pushrules_all, set_pushrule, set_pushrule_actions, set_pushrule_enabled, RuleKind,
+        },
     },
     events::EventType,
+    push::{
+        ConditionalPushRuleInit, ContentPushRule, OverridePushRule, PatternedPushRuleInit,
+        RoomPushRule, SenderPushRule, SimplePushRuleInit, UnderridePushRule,
+    },
 };
 
 #[cfg(feature = "conduit_bin")]
-use rocket::{get, post, put};
+use rocket::{delete, get, post, put};
 
 #[cfg_attr(
     feature = "conduit_bin",
@@ -36,16 +42,201 @@ pub async fn get_pushrules_all_route(
     .into())
 }
 
-#[cfg_attr(feature = "conduit_bin", put(
+#[cfg_attr(
-    "/_matrix/client/r0/pushrules/<_>/<_>/<_>",
+    feature = "conduit_bin",
-    //data = "<body>"
+    get("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
-))]
+)]
+pub async fn get_pushrule_route(
+    db: State<'_, Database>,
+    body: Ruma<get_pushrule::Request<'_>>,
+) -> ConduitResult<get_pushrule::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    let event = db
+        .account_data
+        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = event.content.global;
+    let rule = match body.kind {
+        RuleKind::Override => global
+            .override_
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.0.clone().into()),
+        RuleKind::Underride => global
+            .underride
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.0.clone().into()),
+        RuleKind::Sender => global
+            .sender
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.0.clone().into()),
+        RuleKind::Room => global
+            .room
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.0.clone().into()),
+        RuleKind::Content => global
+            .content
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.0.clone().into()),
+        RuleKind::_Custom(_) => None,
+    };
+
+    if let Some(rule) = rule {
+        Ok(get_pushrule::Response { rule }.into())
+    } else {
+        Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.").into())
+    }
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
+)]
 pub async fn set_pushrule_route(
     db: State<'_, Database>,
-    //body: Ruma<set_pushrule::Request>,
+    body: Ruma<set_pushrule::Request<'_>>,
 ) -> ConduitResult<set_pushrule::Response> {
-    // TODO
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-    warn!("TODO: set_pushrule_route");
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    match body.kind {
+        RuleKind::Override => {
+            if let Some(rule) = global
+                .override_
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.override_.remove(&rule);
+            }
+
+            global.override_.insert(OverridePushRule(
+                ConditionalPushRuleInit {
+                    actions: body.actions.clone(),
+                    default: false,
+                    enabled: true,
+                    rule_id: body.rule_id.clone(),
+                    conditions: body.conditions.clone(),
+                }
+                .into(),
+            ));
+        }
+        RuleKind::Underride => {
+            if let Some(rule) = global
+                .underride
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.underride.remove(&rule);
+            }
+
+            global.underride.insert(UnderridePushRule(
+                ConditionalPushRuleInit {
+                    actions: body.actions.clone(),
+                    default: false,
+                    enabled: true,
+                    rule_id: body.rule_id.clone(),
+                    conditions: body.conditions.clone(),
+                }
+                .into(),
+            ));
+        }
+        RuleKind::Sender => {
+            if let Some(rule) = global
+                .sender
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.sender.remove(&rule);
+            }
+
+            global.sender.insert(SenderPushRule(
+                SimplePushRuleInit {
+                    actions: body.actions.clone(),
+                    default: false,
+                    enabled: true,
+                    rule_id: body.rule_id.clone(),
+                }
+                .into(),
+            ));
+        }
+        RuleKind::Room => {
+            if let Some(rule) = global
+                .room
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.room.remove(&rule);
+            }
+
+            global.room.insert(RoomPushRule(
+                SimplePushRuleInit {
+                    actions: body.actions.clone(),
+                    default: false,
+                    enabled: true,
+                    rule_id: body.rule_id.clone(),
+                }
+                .into(),
+            ));
+        }
+        RuleKind::Content => {
+            if let Some(rule) = global
+                .content
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.content.remove(&rule);
+            }
+
+            global.content.insert(ContentPushRule(
+                PatternedPushRuleInit {
+                    actions: body.actions.clone(),
+                    default: false,
+                    enabled: true,
+                    rule_id: body.rule_id.clone(),
+                    pattern: body.pattern.clone().unwrap_or_default(),
+                }
+                .into(),
+            ));
+        }
+        RuleKind::_Custom(_) => {}
+    }
+
+    db.account_data.update(
+        None,
+        &sender_user,
+        EventType::PushRules,
+        &event,
+        &db.globals,
+    )?;
 
     db.flush().await?;
 
@@ -54,19 +245,426 @@ pub async fn set_pushrule_route(
 
 #[cfg_attr(
     feature = "conduit_bin",
-    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled")
+    get("/_matrix/client/r0/pushrules/<_>/<_>/<_>/actions", data = "<body>")
+)]
+pub async fn get_pushrule_actions_route(
+    db: State<'_, Database>,
+    body: Ruma<get_pushrule_actions::Request<'_>>,
+) -> ConduitResult<get_pushrule_actions::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    let actions = match body.kind {
+        RuleKind::Override => global
+            .override_
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.0.actions.clone()),
+        RuleKind::Underride => global
+            .underride
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.0.actions.clone()),
+        RuleKind::Sender => global
+            .sender
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.0.actions.clone()),
+        RuleKind::Room => global
+            .room
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.0.actions.clone()),
+        RuleKind::Content => global
+            .content
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.0.actions.clone()),
+        RuleKind::_Custom(_) => None,
+    };
+
+    db.flush().await?;
+
+    Ok(get_pushrule_actions::Response {
+        actions: actions.unwrap_or_default(),
+    }
+    .into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/actions", data = "<body>")
+)]
+pub async fn set_pushrule_actions_route(
+    db: State<'_, Database>,
+    body: Ruma<set_pushrule_actions::Request<'_>>,
+) -> ConduitResult<set_pushrule_actions::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    match body.kind {
+        RuleKind::Override => {
+            if let Some(mut rule) = global
+                .override_
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.override_.remove(&rule);
+                rule.0.actions = body.actions.clone();
+                global.override_.insert(rule);
+            }
+        }
+        RuleKind::Underride => {
+            if let Some(mut rule) = global
+                .underride
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.underride.remove(&rule);
+                rule.0.actions = body.actions.clone();
+                global.underride.insert(rule);
+            }
+        }
+        RuleKind::Sender => {
+            if let Some(mut rule) = global
+                .sender
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.sender.remove(&rule);
+                rule.0.actions = body.actions.clone();
+                global.sender.insert(rule);
+            }
+        }
+        RuleKind::Room => {
+            if let Some(mut rule) = global
+                .room
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.room.remove(&rule);
+                rule.0.actions = body.actions.clone();
+                global.room.insert(rule);
+            }
+        }
+        RuleKind::Content => {
+            if let Some(mut rule) = global
+                .content
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.content.remove(&rule);
+                rule.0.actions = body.actions.clone();
+                global.content.insert(rule);
+            }
+        }
+        RuleKind::_Custom(_) => {}
+    };
+
+    db.account_data.update(
+        None,
+        &sender_user,
+        EventType::PushRules,
+        &event,
+        &db.globals,
+    )?;
+
+    db.flush().await?;
+
+    Ok(set_pushrule_actions::Response.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled", data = "<body>")
+)]
+pub async fn get_pushrule_enabled_route(
+    db: State<'_, Database>,
+    body: Ruma<get_pushrule_enabled::Request<'_>>,
+) -> ConduitResult<get_pushrule_enabled::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    let enabled = match body.kind {
+        RuleKind::Override => global
+            .override_
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.0.enabled),
+        RuleKind::Underride => global
+            .underride
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.0.enabled),
+        RuleKind::Sender => global
+            .sender
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.0.enabled),
+        RuleKind::Room => global
+            .room
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.0.enabled),
+        RuleKind::Content => global
+            .content
+            .iter()
+            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.0.enabled),
+        RuleKind::_Custom(_) => false,
+    };
+
+    db.flush().await?;
+
+    Ok(get_pushrule_enabled::Response { enabled }.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled", data = "<body>")
 )]
 pub async fn set_pushrule_enabled_route(
     db: State<'_, Database>,
+    body: Ruma<set_pushrule_enabled::Request<'_>>,
 ) -> ConduitResult<set_pushrule_enabled::Response> {
-    // TODO
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-    warn!("TODO: set_pushrule_enabled_route");
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    match body.kind {
+        RuleKind::Override => {
+            if let Some(mut rule) = global
+                .override_
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.override_.remove(&rule);
+                rule.0.enabled = body.enabled;
+                global.override_.insert(rule);
+            }
+        }
+        RuleKind::Underride => {
+            if let Some(mut rule) = global
+                .underride
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.underride.remove(&rule);
+                rule.0.enabled = body.enabled;
+                global.underride.insert(rule);
+            }
+        }
+        RuleKind::Sender => {
+            if let Some(mut rule) = global
+                .sender
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.sender.remove(&rule);
+                rule.0.enabled = body.enabled;
+                global.sender.insert(rule);
+            }
+        }
+        RuleKind::Room => {
+            if let Some(mut rule) = global
+                .room
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.room.remove(&rule);
+                rule.0.enabled = body.enabled;
+                global.room.insert(rule);
+            }
+        }
+        RuleKind::Content => {
+            if let Some(mut rule) = global
+                .content
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.content.remove(&rule);
+                rule.0.enabled = body.enabled;
+                global.content.insert(rule);
+            }
+        }
+        RuleKind::_Custom(_) => {}
+    }
+
+    db.account_data.update(
+        None,
+        &sender_user,
+        EventType::PushRules,
+        &event,
+        &db.globals,
+    )?;
 
     db.flush().await?;
 
     Ok(set_pushrule_enabled::Response.into())
 }
 
+#[cfg_attr(
+    feature = "conduit_bin",
+    delete("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
+)]
+pub async fn delete_pushrule_route(
+    db: State<'_, Database>,
+    body: Ruma<delete_pushrule::Request<'_>>,
+) -> ConduitResult<delete_pushrule::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    match body.kind {
+        RuleKind::Override => {
+            if let Some(rule) = global
+                .override_
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.override_.remove(&rule);
+            }
+        }
+        RuleKind::Underride => {
+            if let Some(rule) = global
+                .underride
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.underride.remove(&rule);
+            }
+        }
+        RuleKind::Sender => {
+            if let Some(rule) = global
+                .sender
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.sender.remove(&rule);
+            }
+        }
+        RuleKind::Room => {
+            if let Some(rule) = global
+                .room
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.room.remove(&rule);
+            }
+        }
+        RuleKind::Content => {
+            if let Some(rule) = global
+                .content
+                .iter()
+                .find(|rule| rule.0.rule_id == body.rule_id)
+                .cloned()
+            {
+                global.content.remove(&rule);
+            }
+        }
+        RuleKind::_Custom(_) => {}
+    }
+
+    db.account_data.update(
+        None,
+        &sender_user,
+        EventType::PushRules,
+        &event,
+        &db.globals,
+    )?;
+
+    db.flush().await?;
+
+    Ok(delete_pushrule::Response.into())
+}
+
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/pushers"))]
 pub async fn get_pushers_route() -> ConduitResult<get_pushers::Response> {
     Ok(get_pushers::Response {

src/client_server/read_marker.rs

@@ -1,7 +1,9 @@
 use super::State;
 use crate::{ConduitResult, Database, Error, Ruma};
 use ruma::{
-    api::client::{error::ErrorKind, r0::read_marker::set_read_marker},
+    api::client::{
+        error::ErrorKind, r0::capabilities::get_capabilities, r0::read_marker::set_read_marker,
+    },
     events::{AnyEphemeralRoomEvent, AnyEvent, EventType},
 };
 
@@ -76,3 +78,18 @@ pub async fn set_read_marker_route(
 
     Ok(set_read_marker::Response.into())
 }
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    post("/_matrix/client/r0/rooms/<_>/receipt/<_>/<_>", data = "<body>")
+)]
+pub async fn set_receipt_route(
+    db: State<'_, Database>,
+    body: Ruma<get_capabilities::Request>,
+) -> ConduitResult<set_read_marker::Response> {
+    let _sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    db.flush().await?;
+
+    Ok(set_read_marker::Response.into())
+}

src/client_server/redact.rs

@@ -35,6 +35,7 @@ pub async fn redact_event_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     db.flush().await?;

src/client_server/room.rs

@@ -10,7 +10,8 @@ use ruma::{
         room::{guest_access, history_visibility, join_rules, member, name, topic},
         EventType,
     },
-    Raw, RoomAliasId, RoomId, RoomVersionId,
+    serde::Raw,
+    RoomAliasId, RoomId, RoomVersionId,
 };
 use std::{cmp::max, collections::BTreeMap, convert::TryFrom};
 
@@ -68,6 +69,7 @@ pub async fn create_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     // 2. Let the room creator join
@@ -92,6 +94,7 @@ pub async fn create_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     // 3. Power levels
@@ -136,15 +139,20 @@ pub async fn create_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     // 4. Events set by preset
 
     // Figure out preset. We need it for preset specific events
-    let preset = body.preset.unwrap_or_else(|| match body.visibility {
+    let preset = body
-        room::Visibility::Private => create_room::RoomPreset::PrivateChat,
+        .preset
-        room::Visibility::Public => create_room::RoomPreset::PublicChat,
+        .clone()
-    });
+        .unwrap_or_else(|| match &body.visibility {
+            room::Visibility::Private => create_room::RoomPreset::PrivateChat,
+            room::Visibility::Public => create_room::RoomPreset::PublicChat,
+            room::Visibility::_Custom(s) => create_room::RoomPreset::_Custom(s.into()),
+        });
 
     // 4.1 Join Rules
     db.rooms.build_and_append_pdu(
@@ -171,6 +179,7 @@ pub async fn create_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     // 4.2 History Visibility
@@ -191,6 +200,7 @@ pub async fn create_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     // 4.3 Guest Access
@@ -219,6 +229,7 @@ pub async fn create_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     // 5. Events listed in initial_state
@@ -229,7 +240,7 @@ pub async fn create_room_route(
         .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?;
 
         // Silently skip encryption events if they are not allowed
-        if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() {
+        if pdu_builder.event_type == EventType::RoomEncryption && !db.globals.allow_encryption() {
             continue;
         }
 
@@ -241,6 +252,7 @@ pub async fn create_room_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
     }
 
@@ -265,6 +277,7 @@ pub async fn create_room_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
     }
 
@@ -286,6 +299,7 @@ pub async fn create_room_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
     }
 
@@ -312,6 +326,7 @@ pub async fn create_room_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
     }
 
@@ -402,6 +417,7 @@ pub async fn upgrade_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     // Get the old room federations status
@@ -445,6 +461,7 @@ pub async fn upgrade_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     // Join the new room
@@ -469,6 +486,7 @@ pub async fn upgrade_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     // Recommended transferable state events list from the specs
@@ -505,6 +523,7 @@ pub async fn upgrade_room_route(
             &db.sending,
             &db.admin,
             &db.account_data,
+            &db.appservice,
         )?;
     }
 
@@ -551,6 +570,7 @@ pub async fn upgrade_room_route(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     db.flush().await?;

src/client_server/search.rs

@@ -77,8 +77,8 @@ pub async fn search_events_route(
 
     Ok(search_events::Response::new(ResultCategories {
         room_events: ResultRoomEvents {
-            count: None,             // TODO? maybe not
+            count: Some((results.len() as u32).into()), // TODO: set this to none. Element shouldn't depend on it
-            groups: BTreeMap::new(), // TODO
+            groups: BTreeMap::new(),                    // TODO
             next_batch,
             results,
             state: BTreeMap::new(), // TODO

src/client_server/state.rs

@@ -9,9 +9,8 @@ use ruma::{
         },
     },
     events::{
-        room::history_visibility::HistoryVisibility,
+        room::history_visibility::{HistoryVisibility, HistoryVisibilityEventContent},
-        room::history_visibility::HistoryVisibilityEventContent, AnyStateEventContent,
+        AnyStateEventContent, EventContent, EventType,
-        EventContent, EventType,
     },
     EventId, RoomId, UserId,
 };
@@ -64,8 +63,8 @@ pub async fn send_state_event_for_empty_key_route(
     let Ruma {
         body,
         sender_user,
-        sender_device: _,
         json_body,
+        ..
     } = body;
 
     let json = serde_json::from_str::<serde_json::Value>(
@@ -99,14 +98,15 @@ pub async fn send_state_event_for_empty_key_route(
 )]
 pub async fn get_state_events_route(
     db: State<'_, Database>,
-    body: Ruma<get_state_events::Request>,
+    body: Ruma<get_state_events::Request<'_>>,
 ) -> ConduitResult<get_state_events::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
+    #[allow(clippy::blocks_in_if_conditions)]
     // Users not in the room should not be able to access the state unless history_visibility is
     // WorldReadable
-    if !db.rooms.is_joined(sender_user, &body.room_id)? {
+    if !db.rooms.is_joined(sender_user, &body.room_id)?
-        if !matches!(
+        && !matches!(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
                 .map(|(_, event)| {
@@ -119,12 +119,12 @@ pub async fn get_state_events_route(
                         .map(|e| e.history_visibility)
                 }),
             Some(Ok(HistoryVisibility::WorldReadable))
-        ) {
+        )
-            return Err(Error::BadRequest(
+    {
-                ErrorKind::Forbidden,
+        return Err(Error::BadRequest(
-                "You don't have permission to view the room state.",
+            ErrorKind::Forbidden,
-            ));
+            "You don't have permission to view the room state.",
-        }
+        ));
     }
 
     Ok(get_state_events::Response {
@@ -144,14 +144,15 @@ pub async fn get_state_events_route(
 )]
 pub async fn get_state_events_for_key_route(
     db: State<'_, Database>,
-    body: Ruma<get_state_events_for_key::Request>,
+    body: Ruma<get_state_events_for_key::Request<'_>>,
 ) -> ConduitResult<get_state_events_for_key::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
+    #[allow(clippy::blocks_in_if_conditions)]
     // Users not in the room should not be able to access the state unless history_visibility is
     // WorldReadable
-    if !db.rooms.is_joined(sender_user, &body.room_id)? {
+    if !db.rooms.is_joined(sender_user, &body.room_id)?
-        if !matches!(
+        && !matches!(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
                 .map(|(_, event)| {
@@ -164,12 +165,12 @@ pub async fn get_state_events_for_key_route(
                         .map(|e| e.history_visibility)
                 }),
             Some(Ok(HistoryVisibility::WorldReadable))
-        ) {
+        )
-            return Err(Error::BadRequest(
+    {
-                ErrorKind::Forbidden,
+        return Err(Error::BadRequest(
-                "You don't have permission to view the room state.",
+            ErrorKind::Forbidden,
-            ));
+            "You don't have permission to view the room state.",
-        }
+        ));
     }
 
     let event = db
@@ -194,14 +195,15 @@ pub async fn get_state_events_for_key_route(
 )]
 pub async fn get_state_events_for_empty_key_route(
     db: State<'_, Database>,
-    body: Ruma<get_state_events_for_empty_key::Request>,
+    body: Ruma<get_state_events_for_empty_key::Request<'_>>,
 ) -> ConduitResult<get_state_events_for_empty_key::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
+    #[allow(clippy::blocks_in_if_conditions)]
     // Users not in the room should not be able to access the state unless history_visibility is
     // WorldReadable
-    if !db.rooms.is_joined(sender_user, &body.room_id)? {
+    if !db.rooms.is_joined(sender_user, &body.room_id)?
-        if !matches!(
+        && !matches!(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
                 .map(|(_, event)| {
@@ -214,12 +216,12 @@ pub async fn get_state_events_for_empty_key_route(
                         .map(|e| e.history_visibility)
                 }),
             Some(Ok(HistoryVisibility::WorldReadable))
-        ) {
+        )
-            return Err(Error::BadRequest(
+    {
-                ErrorKind::Forbidden,
+        return Err(Error::BadRequest(
-                "You don't have permission to view the room state.",
+            ErrorKind::Forbidden,
-            ));
+            "You don't have permission to view the room state.",
-        }
+        ));
     }
 
     let event = db
@@ -232,7 +234,7 @@ pub async fn get_state_events_for_empty_key_route(
         .1;
 
     Ok(get_state_events_for_empty_key::Response {
-        content: serde_json::value::to_raw_value(&event)
+        content: serde_json::value::to_raw_value(&event.content)
             .map_err(|_| Error::bad_database("Invalid event content in database"))?,
     }
     .into())
@@ -286,6 +288,7 @@ pub async fn send_state_event_for_key_helper(
         &db.sending,
         &db.admin,
         &db.account_data,
+        &db.appservice,
     )?;
 
     Ok(event_id)

src/client_server/sync.rs

@@ -3,7 +3,8 @@ use crate::{ConduitResult, Database, Error, Ruma};
 use ruma::{
     api::client::r0::sync::sync_events,
     events::{room::member::MembershipState, AnySyncEphemeralRoomEvent, EventType},
-    Raw, RoomId, UserId,
+    serde::Raw,
+    RoomId, UserId,
 };
 
 #[cfg(feature = "conduit_bin")]
@@ -90,21 +91,11 @@ pub async fn sync_events_route(
 
         // They /sync response doesn't always return all messages, so we say the output is
         // limited unless there are events in non_timeline_pdus
-        let mut limited = false;
+        let limited = non_timeline_pdus.next().is_some();
-
-        let mut state_pdus = Vec::new();
-        for (_, pdu) in non_timeline_pdus {
-            if pdu.state_key.is_some() {
-                state_pdus.push(pdu);
-            }
-            limited = true;
-        }
 
         // Database queries:
-        let encrypted_room = db
+
-            .rooms
+        let current_state_hash = db.rooms.current_state_hash(&room_id)?;
-            .room_state_get(&room_id, &EventType::RoomEncryption, "")?
-            .is_some();
 
         // These type is Option<Option<_>>. The outer Option is None when there is no event between
         // since and the current room state, meaning there should be no updates.
@@ -116,121 +107,247 @@ pub async fn sync_events_route(
             .as_ref()
             .map(|pdu| db.rooms.pdu_state_hash(&pdu.as_ref().ok()?.0).ok()?);
 
-        let since_members = since_state_hash.as_ref().map(|state_hash| {
+        let (
-            state_hash.as_ref().and_then(|state_hash| {
+            heroes,
-                db.rooms
+            joined_member_count,
-                    .state_type(&state_hash, &EventType::RoomMember)
+            invited_member_count,
-                    .ok()
+            joined_since_last_sync,
-            })
+            state_events,
-        });
+        ) = if since_state_hash != None && Some(&current_state_hash) != since_state_hash.as_ref() {
+            let current_state = db.rooms.room_state_full(&room_id)?;
+            let current_members = current_state
+                .iter()
+                .filter(|(key, _)| key.0 == EventType::RoomMember)
+                .map(|(key, value)| (&key.1, value)) // Only keep state key
+                .collect::<Vec<_>>();
+            let encrypted_room = current_state
+                .get(&(EventType::RoomEncryption, "".to_owned()))
+                .is_some();
+            let since_state = since_state_hash.as_ref().map(|state_hash| {
+                state_hash
+                    .as_ref()
+                    .and_then(|state_hash| db.rooms.state_full(&room_id, &state_hash).ok())
+            });
 
-        let since_encryption = since_state_hash.as_ref().map(|state_hash| {
+            let since_encryption = since_state.as_ref().map(|state| {
-            state_hash.as_ref().and_then(|state_hash| {
+                state
-                db.rooms
+                    .as_ref()
-                    .state_get(&state_hash, &EventType::RoomEncryption, "")
+                    .map(|state| state.get(&(EventType::RoomEncryption, "".to_owned())))
-                    .ok()
+            });
-            })
-        });
 
-        let current_members = db.rooms.room_state_type(&room_id, &EventType::RoomMember)?;
+            // Calculations:
+            let new_encrypted_room =
+                encrypted_room && since_encryption.map_or(false, |encryption| encryption.is_none());
 
-        // Calculations:
+            let send_member_count = since_state.as_ref().map_or(false, |since_state| {
-        let new_encrypted_room =
+                since_state.as_ref().map_or(true, |since_state| {
-            encrypted_room && since_encryption.map_or(false, |encryption| encryption.is_none());
+                    current_members.len()
+                        != since_state
+                            .iter()
+                            .filter(|(key, _)| key.0 == EventType::RoomMember)
+                            .count()
+                })
+            });
 
-        let send_member_count = since_members.as_ref().map_or(false, |since_members| {
+            let since_sender_member = since_state.as_ref().map(|since_state| {
-            since_members.as_ref().map_or(true, |since_members| {
+                since_state.as_ref().and_then(|state| {
-                current_members.len() != since_members.len()
+                    state
-            })
+                        .get(&(EventType::RoomMember, sender_user.as_str().to_owned()))
-        });
+                        .and_then(|pdu| {
+                            serde_json::from_value::<
+                                Raw<ruma::events::room::member::MemberEventContent>,
+                            >(pdu.content.clone())
+                            .expect("Raw::from_value always works")
+                            .deserialize()
+                            .map_err(|_| Error::bad_database("Invalid PDU in database."))
+                            .ok()
+                        })
+                })
+            });
 
-        let since_sender_member = since_members.as_ref().map(|since_members| {
+            if encrypted_room {
-            since_members.as_ref().and_then(|members| {
+                for (user_id, current_member) in current_members {
-                members.get(sender_user.as_str()).and_then(|pdu| {
+                    let current_membership = serde_json::from_value::<
-                    serde_json::from_value::<Raw<ruma::events::room::member::MemberEventContent>>(
+                        Raw<ruma::events::room::member::MemberEventContent>,
-                        pdu.content.clone(),
+                    >(current_member.content.clone())
-                    )
                     .expect("Raw::from_value always works")
                     .deserialize()
-                    .map_err(|_| Error::bad_database("Invalid PDU in database."))
+                    .map_err(|_| Error::bad_database("Invalid PDU in database."))?
-                    .ok()
+                    .membership;
-                })
-            })
-        });
 
-        if encrypted_room {
+                    let since_membership =
-            for (user_id, current_member) in current_members {
+                        since_state
-                let current_membership = serde_json::from_value::<
+                            .as_ref()
-                    Raw<ruma::events::room::member::MemberEventContent>,
+                            .map_or(MembershipState::Join, |since_state| {
-                >(current_member.content.clone())
+                                since_state
-                .expect("Raw::from_value always works")
+                                    .as_ref()
-                .deserialize()
+                                    .and_then(|since_state| {
-                .map_err(|_| Error::bad_database("Invalid PDU in database."))?
+                                        since_state
-                .membership;
+                                            .get(&(EventType::RoomMember, user_id.clone()))
-
+                                            .and_then(|since_member| {
-                let since_membership =
+                                                serde_json::from_value::<
-                    since_members
+                                                Raw<ruma::events::room::member::MemberEventContent>,
-                        .as_ref()
+                                            >(
-                        .map_or(MembershipState::Join, |members| {
+                                                since_member.content.clone()
-                            members
+                                            )
-                                .as_ref()
+                                            .expect("Raw::from_value always works")
-                                .and_then(|members| {
+                                            .deserialize()
-                                    members.get(&user_id).and_then(|since_member| {
+                                            .map_err(|_| {
-                                        serde_json::from_value::<
+                                                Error::bad_database("Invalid PDU in database.")
-                                            Raw<ruma::events::room::member::MemberEventContent>,
+                                            })
-                                        >(
+                                            .ok()
-                                            since_member.content.clone()
+                                            })
-                                        )
-                                        .expect("Raw::from_value always works")
-                                        .deserialize()
-                                        .map_err(|_| {
-                                            Error::bad_database("Invalid PDU in database.")
-                                        })
-                                        .ok()
                                     })
-                                })
+                                    .map_or(MembershipState::Leave, |member| member.membership)
-                                .map_or(MembershipState::Leave, |member| member.membership)
+                            });
-                        });
 
-                let user_id = UserId::try_from(user_id)
+                    let user_id = UserId::try_from(user_id.clone())
-                    .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?;
+                        .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?;
 
-                match (since_membership, current_membership) {
+                    match (since_membership, current_membership) {
-                    (MembershipState::Leave, MembershipState::Join) => {
+                        (MembershipState::Leave, MembershipState::Join) => {
-                        // A new user joined an encrypted room
+                            // A new user joined an encrypted room
-                        if !share_encrypted_room(&db, &sender_user, &user_id, &room_id) {
+                            if !share_encrypted_room(&db, &sender_user, &user_id, &room_id) {
-                            device_list_updates.insert(user_id);
+                                device_list_updates.insert(user_id);
+                            }
                         }
+                        (MembershipState::Join, MembershipState::Leave) => {
+                            // Write down users that have left encrypted rooms we are in
+                            left_encrypted_users.insert(user_id);
+                        }
+                        _ => {}
                     }
-                    (MembershipState::Join, MembershipState::Leave) => {
-                        // Write down users that have left encrypted rooms we are in
-                        left_encrypted_users.insert(user_id);
-                    }
-                    _ => {}
                 }
             }
-        }
 
-        let joined_since_last_sync = since_sender_member.map_or(false, |member| {
+            let joined_since_last_sync = since_sender_member.map_or(false, |member| {
-            member.map_or(true, |member| member.membership != MembershipState::Join)
+                member.map_or(true, |member| member.membership != MembershipState::Join)
-        });
+            });
 
-        if joined_since_last_sync && encrypted_room || new_encrypted_room {
+            if joined_since_last_sync && encrypted_room || new_encrypted_room {
-            // If the user is in a new encrypted room, give them all joined users
+                // If the user is in a new encrypted room, give them all joined users
-            device_list_updates.extend(
+                device_list_updates.extend(
+                    db.rooms
+                        .room_members(&room_id)
+                        .filter_map(|user_id| Some(user_id.ok()?))
+                        .filter(|user_id| {
+                            // Don't send key updates from the sender to the sender
+                            sender_user != user_id
+                        })
+                        .filter(|user_id| {
+                            // Only send keys if the sender doesn't share an encrypted room with the target already
+                            !share_encrypted_room(&db, sender_user, user_id, &room_id)
+                        }),
+                );
+            }
+
+            let (joined_member_count, invited_member_count, heroes) = if send_member_count {
+                let joined_member_count = db.rooms.room_members(&room_id).count();
+                let invited_member_count = db.rooms.room_members_invited(&room_id).count();
+
+                // Recalculate heroes (first 5 members)
+                let mut heroes = Vec::new();
+
+                if joined_member_count + invited_member_count <= 5 {
+                    // Go through all PDUs and for each member event, check if the user is still joined or
+                    // invited until we have 5 or we reach the end
+
+                    for hero in db
+                        .rooms
+                        .all_pdus(&sender_user, &room_id)?
+                        .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus
+                        .filter(|(_, pdu)| pdu.kind == EventType::RoomMember)
+                        .map(|(_, pdu)| {
+                            let content = serde_json::from_value::<
+                                Raw<ruma::events::room::member::MemberEventContent>,
+                            >(pdu.content.clone())
+                            .expect("Raw::from_value always works")
+                            .deserialize()
+                            .map_err(|_| {
+                                Error::bad_database("Invalid member event in database.")
+                            })?;
+
+                            if let Some(state_key) = &pdu.state_key {
+                                let user_id =
+                                    UserId::try_from(state_key.clone()).map_err(|_| {
+                                        Error::bad_database("Invalid UserId in member PDU.")
+                                    })?;
+
+                                // The membership was and still is invite or join
+                                if matches!(
+                                    content.membership,
+                                    MembershipState::Join | MembershipState::Invite
+                                ) && (db.rooms.is_joined(&user_id, &room_id)?
+                                    || db.rooms.is_invited(&user_id, &room_id)?)
+                                {
+                                    Ok::<_, Error>(Some(state_key.clone()))
+                                } else {
+                                    Ok(None)
+                                }
+                            } else {
+                                Ok(None)
+                            }
+                        })
+                        .filter_map(|u| u.ok()) // Filter out buggy users
+                        // Filter for possible heroes
+                        .filter_map(|u| u)
+                    {
+                        if heroes.contains(&hero) || hero == sender_user.as_str() {
+                            continue;
+                        }
+
+                        heroes.push(hero);
+                    }
+                }
+
+                (
+                    Some(joined_member_count),
+                    Some(invited_member_count),
+                    heroes,
+                )
+            } else {
+                (None, None, Vec::new())
+            };
+
+            let state_events = if joined_since_last_sync {
                 db.rooms
-                    .room_members(&room_id)
+                    .room_state_full(&room_id)?
-                    .filter_map(|user_id| Some(user_id.ok()?))
+                    .into_iter()
-                    .filter(|user_id| {
+                    .map(|(_, pdu)| pdu.to_sync_state_event())
-                        // Don't send key updates from the sender to the sender
+                    .collect()
-                        sender_user != user_id
+            } else {
-                    })
+                match since_state {
-                    .filter(|user_id| {
+                    None => Vec::new(),
-                        // Only send keys if the sender doesn't share an encrypted room with the target already
+                    Some(Some(since_state)) => current_state
-                        !share_encrypted_room(&db, sender_user, user_id, &room_id)
+                        .iter()
-                    }),
+                        .filter(|(key, value)| {
-            );
+                            since_state.get(key).map(|e| &e.event_id) != Some(&value.event_id)
-        }
+                        })
+                        .filter(|(_, value)| {
+                            !timeline_pdus.iter().any(|(_, timeline_pdu)| {
+                                timeline_pdu.kind == value.kind
+                                    && timeline_pdu.state_key == value.state_key
+                            })
+                        })
+                        .map(|(_, pdu)| pdu.to_sync_state_event())
+                        .collect(),
+                    Some(None) => current_state
+                        .iter()
+                        .map(|(_, pdu)| pdu.to_sync_state_event())
+                        .collect(),
+                }
+            };
+
+            (
+                heroes,
+                joined_member_count,
+                invited_member_count,
+                joined_since_last_sync,
+                state_events,
+            )
+        } else {
+            (Vec::new(), None, None, false, Vec::new())
+        };
 
         // Look for device list updates in this room
         device_list_updates.extend(
@@ -239,71 +356,6 @@ pub async fn sync_events_route(
                 .filter_map(|r| r.ok()),
         );
 
-        let (joined_member_count, invited_member_count, heroes) = if send_member_count {
-            let joined_member_count = db.rooms.room_members(&room_id).count();
-            let invited_member_count = db.rooms.room_members_invited(&room_id).count();
-
-            // Recalculate heroes (first 5 members)
-            let mut heroes = Vec::new();
-
-            if joined_member_count + invited_member_count <= 5 {
-                // Go through all PDUs and for each member event, check if the user is still joined or
-                // invited until we have 5 or we reach the end
-
-                for hero in db
-                    .rooms
-                    .all_pdus(&sender_user, &room_id)?
-                    .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus
-                    .filter(|(_, pdu)| pdu.kind == EventType::RoomMember)
-                    .map(|(_, pdu)| {
-                        let content = serde_json::from_value::<
-                            Raw<ruma::events::room::member::MemberEventContent>,
-                        >(pdu.content.clone())
-                        .expect("Raw::from_value always works")
-                        .deserialize()
-                        .map_err(|_| Error::bad_database("Invalid member event in database."))?;
-
-                        if let Some(state_key) = &pdu.state_key {
-                            let user_id = UserId::try_from(state_key.clone()).map_err(|_| {
-                                Error::bad_database("Invalid UserId in member PDU.")
-                            })?;
-
-                            // The membership was and still is invite or join
-                            if matches!(
-                                content.membership,
-                                MembershipState::Join | MembershipState::Invite
-                            ) && (db.rooms.is_joined(&user_id, &room_id)?
-                                || db.rooms.is_invited(&user_id, &room_id)?)
-                            {
-                                Ok::<_, Error>(Some(state_key.clone()))
-                            } else {
-                                Ok(None)
-                            }
-                        } else {
-                            Ok(None)
-                        }
-                    })
-                    .filter_map(|u| u.ok()) // Filter out buggy users
-                    // Filter for possible heroes
-                    .filter_map(|u| u)
-                {
-                    if heroes.contains(&hero) || hero == sender_user.as_str() {
-                        continue;
-                    }
-
-                    heroes.push(hero);
-                }
-            }
-
-            (
-                Some(joined_member_count),
-                Some(invited_member_count),
-                heroes,
-            )
-        } else {
-            (None, None, Vec::new())
-        };
-
         let notification_count = if send_notification_counts {
             if let Some(last_read) = db.rooms.edus.private_read_get(&room_id, &sender_user)? {
                 Some(
@@ -333,7 +385,7 @@ pub async fn sync_events_route(
             })?;
 
         let room_events = timeline_pdus
-            .into_iter()
+            .iter()
             .map(|(_, pdu)| pdu.to_sync_room_event())
             .collect::<Vec<_>>();
 
@@ -383,17 +435,8 @@ pub async fn sync_events_route(
                 prev_batch,
                 events: room_events,
             },
-            // TODO: state before timeline
             state: sync_events::State {
-                events: if joined_since_last_sync {
+                events: state_events,
-                    db.rooms
-                        .room_state_full(&room_id)?
-                        .into_iter()
-                        .map(|(_, pdu)| pdu.to_sync_state_event())
-                        .collect()
-                } else {
-                    Vec::new()
-                },
             },
             ephemeral: sync_events::Ephemeral { events: edus },
         };
@@ -455,7 +498,12 @@ pub async fn sync_events_route(
             })
             .and_then(|state_hash| {
                 db.rooms
-                    .state_get(&state_hash, &EventType::RoomMember, sender_user.as_str())
+                    .state_get(
+                        &room_id,
+                        &state_hash,
+                        &EventType::RoomMember,
+                        sender_user.as_str(),
+                    )
                     .ok()?
                     .ok_or_else(|| Error::bad_database("State hash in db doesn't have a state."))
                     .ok()

src/client_server/to_device.rs

@@ -17,7 +17,7 @@ pub async fn send_event_to_device_route(
     body: Ruma<send_event_to_device::Request<'_>>,
 ) -> ConduitResult<send_event_to_device::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-    let sender_device = body.sender_device.as_ref().expect("user is authenticated");
+    let sender_device = body.sender_device.as_deref();
 
     // Check if this is a new transaction id
     if db

src/client_server/voip.rs

@@ -1,13 +1,17 @@
-use crate::{ConduitResult, Error};
+use crate::ConduitResult;
-use ruma::api::client::{error::ErrorKind, r0::message::send_message_event};
+use ruma::api::client::r0::voip::get_turn_server_info;
+use std::time::Duration;
 
 #[cfg(feature = "conduit_bin")]
 use rocket::get;
 
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/voip/turnServer"))]
-pub async fn turn_server_route() -> ConduitResult<send_message_event::Response> {
+pub async fn turn_server_route() -> ConduitResult<get_turn_server_info::Response> {
-    Err(Error::BadRequest(
+    Ok(get_turn_server_info::Response {
-        ErrorKind::NotFound,
+        username: "".to_owned(),
-        "There is no turn server yet.",
+        password: "".to_owned(),
-    ))
+        uris: Vec::new(),
+        ttl: Duration::from_secs(60 * 60 * 24),
+    }
+    .into())
 }

src/database.rs

@@ -1,5 +1,6 @@
 pub mod account_data;
 pub mod admin;
+pub mod appservice;
 pub mod globals;
 pub mod key_backups;
 pub mod media;
@@ -13,12 +14,51 @@ use crate::{Error, Result};
 use directories::ProjectDirs;
 use futures::StreamExt;
 use log::info;
-use rocket::{
+use rocket::futures::{self, channel::mpsc};
-    futures::{self, channel::mpsc},
+use ruma::{DeviceId, ServerName, UserId};
-    Config,
+use serde::Deserialize;
-};
+use std::collections::HashMap;
-use ruma::{DeviceId, UserId};
+use std::fs::remove_dir_all;
-use std::{convert::TryFrom, fs::remove_dir_all};
+use std::sync::{Arc, RwLock};
+use tokio::sync::Semaphore;
+
+#[derive(Clone, Deserialize)]
+pub struct Config {
+    server_name: Box<ServerName>,
+    database_path: String,
+    #[serde(default = "default_cache_capacity")]
+    cache_capacity: u32,
+    #[serde(default = "default_max_request_size")]
+    max_request_size: u32,
+    #[serde(default = "default_max_concurrent_requests")]
+    max_concurrent_requests: u16,
+    #[serde(default = "true_fn")]
+    allow_registration: bool,
+    #[serde(default = "true_fn")]
+    allow_encryption: bool,
+    #[serde(default = "false_fn")]
+    allow_federation: bool,
+}
+
+fn false_fn() -> bool {
+    false
+}
+
+fn true_fn() -> bool {
+    true
+}
+
+fn default_cache_capacity() -> u32 {
+    1024 * 1024 * 1024
+}
+
+fn default_max_request_size() -> u32 {
+    20 * 1024 * 1024 // Default to 20 MB
+}
+
+fn default_max_concurrent_requests() -> u16 {
+    4
+}
 
 #[derive(Clone)]
 pub struct Database {
@@ -32,6 +72,7 @@ pub struct Database {
     pub transaction_ids: transaction_ids::TransactionIds,
     pub sending: sending::Sending,
     pub admin: admin::Admin,
+    pub appservice: appservice::Appservice,
     pub _db: sled::Db,
 }
 
@@ -49,45 +90,18 @@ impl Database {
     }
 
     /// Load an existing database or create a new one.
-    pub fn load_or_create(config: &Config) -> Result<Self> {
+    pub async fn load_or_create(config: Config) -> Result<Self> {
-        let server_name = config.get_str("server_name").unwrap_or("localhost");
-
-        let path = config
-            .get_str("database_path")
-            .map(|x| Ok::<_, Error>(x.to_owned()))
-            .unwrap_or_else(|_| {
-                let path = ProjectDirs::from("xyz", "koesters", "conduit")
-                    .ok_or_else(|| {
-                        Error::bad_config("The OS didn't return a valid home directory path.")
-                    })?
-                    .data_dir()
-                    .join(server_name);
-
-                Ok(path
-                    .to_str()
-                    .ok_or_else(|| Error::bad_config("Database path contains invalid unicode."))?
-                    .to_owned())
-            })?;
-
         let db = sled::Config::default()
-            .path(&path)
+            .path(&config.database_path)
-            .cache_capacity(
+            .cache_capacity(config.cache_capacity as u64)
-                u64::try_from(
-                    config
-                        .get_int("cache_capacity")
-                        .unwrap_or(1024 * 1024 * 1024),
-                )
-                .map_err(|_| Error::bad_config("Cache capacity needs to be a u64."))?,
-            )
-            .print_profile_on_drop(false)
             .open()?;
 
-        info!("Opened sled database at {}", path);
+        info!("Opened sled database at {}", config.database_path);
 
         let (admin_sender, admin_receiver) = mpsc::unbounded();
 
         let db = Self {
-            globals: globals::Globals::load(db.open_tree("global")?, config)?,
+            globals: globals::Globals::load(db.open_tree("global")?, config).await?,
             users: users::Users {
                 userid_password: db.open_tree("userid_password")?,
                 userid_displayname: db.open_tree("userid_displayname")?,
@@ -136,6 +150,7 @@ impl Database {
                 roomuserid_invited: db.open_tree("roomuserid_invited")?,
                 userroomid_left: db.open_tree("userroomid_left")?,
 
+                statekey_short: db.open_tree("statekey_short")?,
                 stateid_pduid: db.open_tree("stateid_pduid")?,
                 pduid_statehash: db.open_tree("pduid_statehash")?,
                 roomid_statehash: db.open_tree("roomid_statehash")?,
@@ -157,10 +172,15 @@ impl Database {
             sending: sending::Sending {
                 servernamepduids: db.open_tree("servernamepduids")?,
                 servercurrentpdus: db.open_tree("servercurrentpdus")?,
+                maximum_requests: Arc::new(Semaphore::new(10)),
             },
             admin: admin::Admin {
                 sender: admin_sender,
             },
+            appservice: appservice::Appservice {
+                cached_registrations: Arc::new(RwLock::new(HashMap::new())),
+                id_appserviceregistrations: db.open_tree("id_appserviceregistrations")?,
+            },
             _db: db,
         };
 

src/database/account_data.rs

@@ -2,7 +2,8 @@ use crate::{utils, Error, Result};
 use ruma::{
     api::client::error::ErrorKind,
     events::{AnyEvent as EduEvent, EventType},
-    Raw, RoomId, UserId,
+    serde::Raw,
+    RoomId, UserId,
 };
 use serde::{de::DeserializeOwned, Serialize};
 use sled::IVec;

src/database/admin.rs

@@ -3,11 +3,16 @@ use std::convert::{TryFrom, TryInto};
 use crate::pdu::PduBuilder;
 use log::warn;
 use rocket::futures::{channel::mpsc, stream::StreamExt};
-use ruma::{events::room::message, events::EventType, UserId};
+use ruma::{
+    events::{room::message, EventType},
+    UserId,
+};
 use tokio::select;
 
 pub enum AdminCommand {
-    SendTextMessage(message::TextMessageEventContent),
+    RegisterAppservice(serde_yaml::Value),
+    ListAppservices,
+    SendMessage(message::MessageEventContent),
 }
 
 #[derive(Clone)]
@@ -38,33 +43,52 @@ impl Admin {
                 .unwrap();
 
             if conduit_room.is_none() {
-                warn!("Conduit instance does not have an #admins room. Logging to that room will not work.");
+                warn!("Conduit instance does not have an #admins room. Logging to that room will not work. Restart Conduit after creating a user to fix this.");
             }
 
+            let send_message = |message: message::MessageEventContent| {
+                if let Some(conduit_room) = &conduit_room {
+                    db.rooms
+                        .build_and_append_pdu(
+                            PduBuilder {
+                                event_type: EventType::RoomMessage,
+                                content: serde_json::to_value(message)
+                                    .expect("event is valid, we just created it"),
+                                unsigned: None,
+                                state_key: None,
+                                redacts: None,
+                            },
+                            &conduit_user,
+                            &conduit_room,
+                            &db.globals,
+                            &db.sending,
+                            &db.admin,
+                            &db.account_data,
+                            &db.appservice,
+                        )
+                        .unwrap();
+                }
+            };
+
             loop {
                 select! {
                     Some(event) = receiver.next() => {
                         match event {
-                            AdminCommand::SendTextMessage(message) => {
+                            AdminCommand::RegisterAppservice(yaml) => {
-                                println!("{:?}", message);
+                                db.appservice.register_appservice(yaml).unwrap(); // TODO handle error
-
+                            }
-                                if let Some(conduit_room) = &conduit_room {
+                            AdminCommand::ListAppservices => {
-                                    db.rooms.build_and_append_pdu(
+                                let appservices = db.appservice.iter_ids().collect::<Vec<_>>();
-                                        PduBuilder {
+                                let count = appservices.len();
-                                            event_type: EventType::RoomMessage,
+                                let output = format!(
-                                            content: serde_json::to_value(message).expect("event is valid, we just created it"),
+                                    "Appservices ({}): {}",
-                                            unsigned: None,
+                                    count,
-                                            state_key: None,
+                                    appservices.into_iter().filter_map(|r| r.ok()).collect::<Vec<_>>().join(", ")
-                                            redacts: None,
+                                );
-                                        },
+                                send_message(message::MessageEventContent::text_plain(output));
-                                        &conduit_user,
+                            }
-                                        &conduit_room,
+                            AdminCommand::SendMessage(message) => {
-                                        &db.globals,
+                                send_message(message);
-                                        &db.sending,
-                                        &db.admin,
-                                        &db.account_data,
-                                    ).unwrap();
-                                }
                             }
                         }
                     }

src/database/appservice.rs

@@ -0,0 +1,67 @@
+use crate::{utils, Error, Result};
+use std::collections::HashMap;
+use std::sync::{Arc, RwLock};
+
+#[derive(Clone)]
+pub struct Appservice {
+    pub(super) cached_registrations: Arc<RwLock<HashMap<String, serde_yaml::Value>>>,
+    pub(super) id_appserviceregistrations: sled::Tree,
+}
+
+impl Appservice {
+    pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> {
+        // TODO: Rumaify
+        let id = yaml.get("id").unwrap().as_str().unwrap();
+        self.id_appserviceregistrations
+            .insert(id, serde_yaml::to_string(&yaml).unwrap().as_bytes())?;
+        self.cached_registrations
+            .write()
+            .unwrap()
+            .insert(id.to_owned(), yaml);
+
+        Ok(())
+    }
+
+    pub fn get_registration(&self, id: &str) -> Result<Option<serde_yaml::Value>> {
+        self.cached_registrations
+            .read()
+            .unwrap()
+            .get(id)
+            .map_or_else(
+                || {
+                    Ok(self
+                        .id_appserviceregistrations
+                        .get(id)?
+                        .map(|bytes| {
+                            Ok::<_, Error>(serde_yaml::from_slice(&bytes).map_err(|_| {
+                                Error::bad_database(
+                                    "Invalid registration bytes in id_appserviceregistrations.",
+                                )
+                            })?)
+                        })
+                        .transpose()?)
+                },
+                |r| Ok(Some(r.clone())),
+            )
+    }
+
+    pub fn iter_ids(&self) -> impl Iterator<Item = Result<String>> {
+        self.id_appserviceregistrations.iter().keys().map(|id| {
+            Ok(utils::string_from_bytes(&id?).map_err(|_| {
+                Error::bad_database("Invalid id bytes in id_appserviceregistrations.")
+            })?)
+        })
+    }
+
+    pub fn iter_all<'a>(
+        &'a self,
+    ) -> impl Iterator<Item = Result<(String, serde_yaml::Value)>> + 'a {
+        self.iter_ids().filter_map(|id| id.ok()).map(move |id| {
+            Ok((
+                id.clone(),
+                self.get_registration(&id)?
+                    .expect("iter_ids only returns appservices that exist"),
+            ))
+        })
+    }
+}

src/database/globals.rs

@@ -1,24 +1,26 @@
-use crate::{utils, Error, Result};
+use crate::{database::Config, utils, Error, Result};
 use log::error;
 use ruma::ServerName;
-use std::{convert::TryInto, sync::Arc};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::RwLock;
+use std::time::Duration;
+use trust_dns_resolver::TokioAsyncResolver;
 
 pub const COUNTER: &str = "c";
 
 #[derive(Clone)]
 pub struct Globals {
     pub(super) globals: sled::Tree,
+    config: Config,
     keypair: Arc<ruma::signatures::Ed25519KeyPair>,
     reqwest_client: reqwest::Client,
-    server_name: Box<ServerName>,
+    pub actual_destination_cache: Arc<RwLock<HashMap<Box<ServerName>, (String, Option<String>)>>>, // actual_destination, host
-    max_request_size: u32,
+    dns_resolver: TokioAsyncResolver,
-    registration_disabled: bool,
-    encryption_disabled: bool,
-    federation_enabled: bool,
 }
 
 impl Globals {
-    pub fn load(globals: sled::Tree, config: &rocket::Config) -> Result<Self> {
+    pub async fn load(globals: sled::Tree, config: Config) -> Result<Self> {
         let bytes = &*globals
             .update_and_fetch("keypair", utils::generate_keypair)?
             .expect("utils::generate_keypair always returns Some");
@@ -53,24 +55,24 @@ impl Globals {
             }
         };
 
+        let reqwest_client = reqwest::Client::builder()
+            .connect_timeout(Duration::from_secs(30))
+            .timeout(Duration::from_secs(60 * 3))
+            .pool_max_idle_per_host(1)
+            .build()
+            .unwrap();
+
         Ok(Self {
             globals,
+            config,
             keypair: Arc::new(keypair),
-            reqwest_client: reqwest::Client::new(),
+            reqwest_client,
-            server_name: config
+            dns_resolver: TokioAsyncResolver::tokio_from_system_conf()
-                .get_str("server_name")
+                .await
-                .unwrap_or("localhost")
+                .map_err(|_| {
-                .to_string()
+                    Error::bad_config("Failed to set up trust dns resolver with system config.")
-                .try_into()
+                })?,
-                .map_err(|_| Error::bad_config("Invalid server_name."))?,
+            actual_destination_cache: Arc::new(RwLock::new(HashMap::new())),
-            max_request_size: config
-                .get_int("max_request_size")
-                .unwrap_or(20 * 1024 * 1024) // Default to 20 MB
-                .try_into()
-                .map_err(|_| Error::bad_config("Invalid max_request_size."))?,
-            registration_disabled: config.get_bool("registration_disabled").unwrap_or(false),
-            encryption_disabled: config.get_bool("encryption_disabled").unwrap_or(false),
-            federation_enabled: config.get_bool("federation_enabled").unwrap_or(false),
         })
     }
 
@@ -102,22 +104,26 @@ impl Globals {
     }
 
     pub fn server_name(&self) -> &ServerName {
-        self.server_name.as_ref()
+        self.config.server_name.as_ref()
     }
 
     pub fn max_request_size(&self) -> u32 {
-        self.max_request_size
+        self.config.max_request_size
     }
 
-    pub fn registration_disabled(&self) -> bool {
+    pub fn allow_registration(&self) -> bool {
-        self.registration_disabled
+        self.config.allow_registration
     }
 
-    pub fn encryption_disabled(&self) -> bool {
+    pub fn allow_encryption(&self) -> bool {
-        self.encryption_disabled
+        self.config.allow_encryption
     }
 
-    pub fn federation_enabled(&self) -> bool {
+    pub fn allow_federation(&self) -> bool {
-        self.federation_enabled
+        self.config.allow_federation
+    }
+
+    pub fn dns_resolver(&self) -> &TokioAsyncResolver {
+        &self.dns_resolver
     }
 }

src/database/media.rs

@@ -5,7 +5,7 @@ use std::mem;
 
 pub struct FileMeta {
     pub filename: Option<String>,
-    pub content_type: String,
+    pub content_type: Option<String>,
     pub file: Vec<u8>,
 }
 
@@ -20,7 +20,7 @@ impl Media {
         &self,
         mxc: String,
         filename: &Option<&str>,
-        content_type: &str,
+        content_type: &Option<&str>,
         file: &[u8],
     ) -> Result<()> {
         let mut key = mxc.as_bytes().to_vec();
@@ -30,7 +30,12 @@ impl Media {
         key.push(0xff);
         key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default());
         key.push(0xff);
-        key.extend_from_slice(content_type.as_bytes());
+        key.extend_from_slice(
+            content_type
+                .as_ref()
+                .map(|c| c.as_bytes())
+                .unwrap_or_default(),
+        );
 
         self.mediaid_file.insert(key, file)?;
 
@@ -42,7 +47,7 @@ impl Media {
         &self,
         mxc: String,
         filename: &Option<String>,
-        content_type: &str,
+        content_type: &Option<String>,
         width: u32,
         height: u32,
         file: &[u8],
@@ -54,7 +59,12 @@ impl Media {
         key.push(0xff);
         key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default());
         key.push(0xff);
-        key.extend_from_slice(content_type.as_bytes());
+        key.extend_from_slice(
+            content_type
+                .as_ref()
+                .map(|c| c.as_bytes())
+                .unwrap_or_default(),
+        );
 
         self.mediaid_file.insert(key, file)?;
 
@@ -73,12 +83,14 @@ impl Media {
             let (key, file) = r?;
             let mut parts = key.rsplit(|&b| b == 0xff);
 
-            let content_type = utils::string_from_bytes(
+            let content_type = parts
-                parts
+                .next()
-                    .next()
+                .map(|bytes| {
-                    .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?,
+                    Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| {
-            )
+                        Error::bad_database("Content type in mediaid_file is invalid unicode.")
-            .map_err(|_| Error::bad_database("Content type in mediaid_file is invalid unicode."))?;
+                    })?)
+                })
+                .transpose()?;
 
             let filename_bytes = parts
                 .next()
@@ -148,12 +160,14 @@ impl Media {
             let (key, file) = r?;
             let mut parts = key.rsplit(|&b| b == 0xff);
 
-            let content_type = utils::string_from_bytes(
+            let content_type = parts
-                parts
+                .next()
-                    .next()
+                .map(|bytes| {
-                    .ok_or_else(|| Error::bad_database("Invalid Media ID in db"))?,
+                    Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| {
-            )
+                        Error::bad_database("Content type in mediaid_file is invalid unicode.")
-            .map_err(|_| Error::bad_database("Content type in mediaid_file is invalid unicode."))?;
+                    })?)
+                })
+                .transpose()?;
 
             let filename_bytes = parts
                 .next()
@@ -179,12 +193,14 @@ impl Media {
             let (key, file) = r?;
             let mut parts = key.rsplit(|&b| b == 0xff);
 
-            let content_type = utils::string_from_bytes(
+            let content_type = parts
-                parts
+                .next()
-                    .next()
+                .map(|bytes| {
-                    .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?,
+                    Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| {
-            )
+                        Error::bad_database("Content type in mediaid_file is invalid unicode.")
-            .map_err(|_| Error::bad_database("Content type in mediaid_file is invalid unicode."))?;
+                    })?)
+                })
+                .transpose()?;
 
             let filename_bytes = parts
                 .next()
@@ -274,7 +290,12 @@ impl Media {
                     file: thumbnail_bytes.to_vec(),
                 }))
             } else {
-                Ok(None)
+                // Couldn't parse file to generate thumbnail, send original
+                Ok(Some(FileMeta {
+                    filename,
+                    content_type,
+                    file: file.to_vec(),
+                }))
             }
         } else {
             Ok(None)

src/database/rooms.rs

@@ -4,6 +4,7 @@ pub use edus::RoomEdus;
 
 use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result};
 use log::error;
+use regex::Regex;
 use ring::digest;
 use ruma::{
     api::client::error::ErrorKind,
@@ -15,7 +16,8 @@ use ruma::{
         },
         EventType,
     },
-    EventId, Raw, RoomAliasId, RoomId, ServerName, UserId,
+    serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue, Raw},
+    EventId, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId,
 };
 use sled::IVec;
 use state_res::{event_auth, Error as StateError, Requester, StateEvent, StateMap, StateStore};
@@ -61,7 +63,8 @@ pub struct Rooms {
     /// Remember the state hash at events in the past.
     pub(super) pduid_statehash: sled::Tree,
     /// The state for a given state hash.
-    pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + EventType + StateKey
+    pub(super) statekey_short: sled::Tree, // StateKey = EventType + StateKey, Short = Count
+    pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + Short, PduId = Count (without roomid)
 }
 
 impl StateStore for Rooms {
@@ -74,7 +77,10 @@ impl StateStore for Rooms {
             .get_pdu_id(event_id)
             .map_err(StateError::custom)?
             .ok_or_else(|| {
-                StateError::NotFound("PDU via room_id and event_id not found in the db.".into())
+                StateError::NotFound(format!(
+                    "PDU via room_id and event_id not found in the db: {}",
+                    event_id.as_str()
+                ))
             })?;
 
         serde_json::from_slice(
@@ -88,7 +94,7 @@ impl StateStore for Rooms {
         .and_then(|pdu: StateEvent| {
             // conduit's PDU's always contain a room_id but some
             // of ruma's do not so this must be an Option
-            if pdu.room_id() == Some(room_id) {
+            if pdu.room_id() == room_id {
                 Ok(Arc::new(pdu))
             } else {
                 Err(StateError::NotFound(
@@ -102,21 +108,28 @@ impl StateStore for Rooms {
 impl Rooms {
     /// Builds a StateMap by iterating over all keys that start
     /// with state_hash, this gives the full state for the given state_hash.
-    pub fn state_full(&self, state_hash: &StateHashId) -> Result<StateMap<PduEvent>> {
+    pub fn state_full(
+        &self,
+        room_id: &RoomId,
+        state_hash: &StateHashId,
+    ) -> Result<StateMap<PduEvent>> {
         self.stateid_pduid
             .scan_prefix(&state_hash)
             .values()
-            .map(|pduid| {
+            .map(|pduid_short| {
-                self.pduid_pdu.get(&pduid?)?.map_or_else(
+                let mut pduid = room_id.as_bytes().to_vec();
-                    || Err(Error::bad_database("Failed to find StateMap.")),
+                pduid.push(0xff);
+                pduid.extend_from_slice(&pduid_short?);
+                self.pduid_pdu.get(&pduid)?.map_or_else(
+                    || Err(Error::bad_database("Failed to find PDU in state snapshot.")),
                     |b| {
                         serde_json::from_slice::<PduEvent>(&b)
                             .map_err(|_| Error::bad_database("Invalid PDU in db."))
                     },
                 )
             })
+            .filter_map(|r| r.ok())
             .map(|pdu| {
-                let pdu = pdu?;
                 Ok((
                     (
                         pdu.kind.clone(),
@@ -131,64 +144,45 @@ impl Rooms {
             .collect::<Result<StateMap<_>>>()
     }
 
-    /// Returns all state entries for this type.
-    pub fn state_type(
-        &self,
-        state_hash: &StateHashId,
-        event_type: &EventType,
-    ) -> Result<HashMap<String, PduEvent>> {
-        let mut prefix = state_hash.to_vec();
-        prefix.push(0xff);
-        prefix.extend_from_slice(&event_type.to_string().as_bytes());
-        prefix.push(0xff);
-
-        let mut hashmap = HashMap::new();
-        for pdu in self
-            .stateid_pduid
-            .scan_prefix(&prefix)
-            .values()
-            .map(|pdu_id| {
-                Ok::<_, Error>(
-                    serde_json::from_slice::<PduEvent>(&self.pduid_pdu.get(pdu_id?)?.ok_or_else(
-                        || Error::bad_database("PDU in state not found in database."),
-                    )?)
-                    .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?,
-                )
-            })
-        {
-            let pdu = pdu?;
-            let state_key = pdu.state_key.clone().ok_or_else(|| {
-                Error::bad_database("Room state contains event without state_key.")
-            })?;
-            hashmap.insert(state_key, pdu);
-        }
-        Ok(hashmap)
-    }
-
     /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`).
     pub fn state_get(
         &self,
+        room_id: &RoomId,
         state_hash: &StateHashId,
         event_type: &EventType,
         state_key: &str,
     ) -> Result<Option<(IVec, PduEvent)>> {
-        let mut key = state_hash.to_vec();
+        let mut key = event_type.to_string().as_bytes().to_vec();
-        key.push(0xff);
-        key.extend_from_slice(&event_type.to_string().as_bytes());
         key.push(0xff);
         key.extend_from_slice(&state_key.as_bytes());
 
-        self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| {
+        let short = self.statekey_short.get(&key)?;
-            Ok::<_, Error>(Some((
+
-                pdu_id.clone(),
+        if let Some(short) = short {
-                serde_json::from_slice::<PduEvent>(
+            let mut stateid = state_hash.to_vec();
-                    &self.pduid_pdu.get(&pdu_id)?.ok_or_else(|| {
+            stateid.push(0xff);
-                        Error::bad_database("PDU in state not found in database.")
+            stateid.extend_from_slice(&short);
-                    })?,
+
-                )
+            self.stateid_pduid
-                .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?,
+                .get(&stateid)?
-            )))
+                .map_or(Ok(None), |pdu_id_short| {
-        })
+                    let mut pdu_id = room_id.as_bytes().to_vec();
+                    pdu_id.push(0xff);
+                    pdu_id.extend_from_slice(&pdu_id_short);
+
+                    Ok::<_, Error>(Some((
+                        pdu_id.clone().into(),
+                        serde_json::from_slice::<PduEvent>(
+                            &self.pduid_pdu.get(&pdu_id)?.ok_or_else(|| {
+                                Error::bad_database("PDU in state not found in database.")
+                            })?,
+                        )
+                        .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?,
+                    )))
+                })
+        } else {
+            return Ok(None);
+        }
     }
 
     /// Returns the last state hash key added to the db.
@@ -196,7 +190,7 @@ impl Rooms {
         Ok(self.pduid_statehash.get(pdu_id)?)
     }
 
-    /// Returns the last state hash key added to the db.
+    /// Returns the last state hash key added to the db for the given room.
     pub fn current_state_hash(&self, room_id: &RoomId) -> Result<Option<StateHashId>> {
         Ok(self.roomid_statehash.get(room_id.as_bytes())?)
     }
@@ -249,11 +243,14 @@ impl Rooms {
             .is_some())
     }
 
-    /// Returns the full room state.
+    /// Force the creation of a new StateHash and insert it into the db.
+    ///
+    /// Whatever `state` is supplied to `force_state` __is__ the current room state snapshot.
     pub fn force_state(
         &self,
         room_id: &RoomId,
         state: HashMap<(EventType, String), Vec<u8>>,
+        globals: &super::globals::Globals,
     ) -> Result<()> {
         let state_hash =
             self.calculate_hash(&state.values().map(|pdu_id| &**pdu_id).collect::<Vec<_>>())?;
@@ -261,11 +258,29 @@ impl Rooms {
         prefix.push(0xff);
 
         for ((event_type, state_key), pdu_id) in state {
+            let mut statekey = event_type.as_ref().as_bytes().to_vec();
+            statekey.push(0xff);
+            statekey.extend_from_slice(&state_key.as_bytes());
+
+            let short = match self.statekey_short.get(&statekey)? {
+                Some(short) => utils::u64_from_bytes(&short)
+                    .map_err(|_| Error::bad_database("Invalid short bytes in statekey_short."))?,
+                None => {
+                    let short = globals.next_count()?;
+                    self.statekey_short
+                        .insert(&statekey, &short.to_be_bytes())?;
+                    short
+                }
+            };
+
+            let pdu_id_short = pdu_id
+                .splitn(2, |&b| b == 0xff)
+                .nth(1)
+                .ok_or_else(|| Error::bad_database("Invalid pduid in state."))?;
+
             let mut state_id = prefix.clone();
-            state_id.extend_from_slice(&event_type.as_str().as_bytes());
+            state_id.extend_from_slice(&short.to_be_bytes());
-            state_id.push(0xff);
+            self.stateid_pduid.insert(state_id, pdu_id_short)?;
-            state_id.extend_from_slice(&state_key.as_bytes());
-            self.stateid_pduid.insert(state_id, pdu_id)?;
         }
 
         self.roomid_statehash
@@ -277,25 +292,12 @@ impl Rooms {
     /// Returns the full room state.
     pub fn room_state_full(&self, room_id: &RoomId) -> Result<StateMap<PduEvent>> {
         if let Some(current_state_hash) = self.current_state_hash(room_id)? {
-            self.state_full(&current_state_hash)
+            self.state_full(&room_id, &current_state_hash)
         } else {
             Ok(BTreeMap::new())
         }
     }
 
-    /// Returns all state entries for this type.
-    pub fn room_state_type(
-        &self,
-        room_id: &RoomId,
-        event_type: &EventType,
-    ) -> Result<HashMap<String, PduEvent>> {
-        if let Some(current_state_hash) = self.current_state_hash(room_id)? {
-            self.state_type(&current_state_hash, event_type)
-        } else {
-            Ok(HashMap::new())
-        }
-    }
-
     /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`).
     pub fn room_state_get(
         &self,
@@ -304,7 +306,7 @@ impl Rooms {
         state_key: &str,
     ) -> Result<Option<(IVec, PduEvent)>> {
         if let Some(current_state_hash) = self.current_state_hash(room_id)? {
-            self.state_get(&current_state_hash, event_type, state_key)
+            self.state_get(&room_id, &current_state_hash, event_type, state_key)
         } else {
             Ok(None)
         }
@@ -369,8 +371,8 @@ impl Rooms {
         })
     }
 
-    /// Returns the pdu.
+    /// Returns the pdu as a `BTreeMap<String, CanonicalJsonValue>`.
-    pub fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result<Option<serde_json::Value>> {
+    pub fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result<Option<CanonicalJsonObject>> {
         self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| {
             Ok(Some(
                 serde_json::from_slice(&pdu)
@@ -437,16 +439,46 @@ impl Rooms {
     }
 
     /// Creates a new persisted data unit and adds it to a room.
+    ///
+    /// By this point the incoming event should be fully authenticated, no auth happens
+    /// in `append_pdu`.
     pub fn append_pdu(
         &self,
         pdu: &PduEvent,
-        pdu_json: &serde_json::Value,
+        mut pdu_json: CanonicalJsonObject,
         count: u64,
         pdu_id: IVec,
         globals: &super::globals::Globals,
         account_data: &super::account_data::AccountData,
         admin: &super::admin::Admin,
     ) -> Result<()> {
+        // Make unsigned fields correct. This is not properly documented in the spec, but state
+        // events need to have previous content in the unsigned field, so clients can easily
+        // interpret things like membership changes
+        if let Some(state_key) = &pdu.state_key {
+            if let CanonicalJsonValue::Object(unsigned) = pdu_json
+                .entry("unsigned".to_owned())
+                .or_insert_with(|| CanonicalJsonValue::Object(Default::default()))
+            {
+                if let Some(prev_state_hash) = self.pdu_state_hash(&pdu_id).unwrap() {
+                    if let Some(prev_state) = self
+                        .state_get(&pdu.room_id, &prev_state_hash, &pdu.kind, &state_key)
+                        .unwrap()
+                    {
+                        unsigned.insert(
+                            "prev_content".to_owned(),
+                            CanonicalJsonValue::Object(
+                                utils::to_canonical_object(prev_state.1.content)
+                                    .expect("event is valid, we just created it"),
+                            ),
+                        );
+                    }
+                }
+            } else {
+                error!("Invalid unsigned type in pdu.");
+            }
+        }
+
         self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?;
 
         // Mark as read first so the sending client doesn't get a notification even if appending
@@ -454,7 +486,11 @@ impl Rooms {
         self.edus
             .private_read_set(&pdu.room_id, &pdu.sender, count, &globals)?;
 
-        self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?;
+        self.pduid_pdu.insert(
+            &pdu_id,
+            &*serde_json::to_string(&pdu_json)
+                .expect("CanonicalJsonObject is always a valid String"),
+        )?;
 
         self.eventid_pduid
             .insert(pdu.event_id.as_bytes(), &*pdu_id)?;
@@ -512,17 +548,59 @@ impl Rooms {
                             .as_ref()
                             == Some(&pdu.room_id)
                     {
-                        let mut parts = body.split_whitespace().skip(1);
+                        let mut lines = body.lines();
+                        let command_line = lines.next().expect("each string has at least one line");
+                        let body = lines.collect::<Vec<_>>();
+
+                        let mut parts = command_line.split_whitespace().skip(1);
                         if let Some(command) = parts.next() {
                             let args = parts.collect::<Vec<_>>();
 
-                            admin.send(AdminCommand::SendTextMessage(
+                            match command {
-                                message::TextMessageEventContent {
+                                "register_appservice" => {
-                                    body: format!("Command: {}, Args: {:?}", command, args),
+                                    if body.len() > 2
-                                    formatted: None,
+                                        && body[0].trim() == "```"
-                                    relates_to: None,
+                                        && body.last().unwrap().trim() == "```"
-                                },
+                                    {
-                            ));
+                                        let appservice_config = body[1..body.len() - 1].join("\n");
+                                        let parsed_config = serde_yaml::from_str::<serde_yaml::Value>(
+                                            &appservice_config,
+                                        );
+                                        match parsed_config {
+                                            Ok(yaml) => {
+                                                admin.send(AdminCommand::RegisterAppservice(yaml));
+                                            }
+                                            Err(e) => {
+                                                admin.send(AdminCommand::SendMessage(
+                                                    message::MessageEventContent::text_plain(
+                                                        format!(
+                                                            "Could not parse appservice config: {}",
+                                                            e
+                                                        ),
+                                                    ),
+                                                ));
+                                            }
+                                        }
+                                    } else {
+                                        admin.send(AdminCommand::SendMessage(
+                                            message::MessageEventContent::text_plain(
+                                                "Expected code block in command body.",
+                                            ),
+                                        ));
+                                    }
+                                }
+                                "list_appservices" => {
+                                    admin.send(AdminCommand::ListAppservices);
+                                }
+                                _ => {
+                                    admin.send(AdminCommand::SendMessage(
+                                        message::MessageEventContent::text_plain(format!(
+                                            "Command: {}, Args: {:?}",
+                                            command, args
+                                        )),
+                                    ));
+                                }
+                            }
                         }
                     }
                 }
@@ -538,7 +616,12 @@ impl Rooms {
     /// This adds all current state events (not including the incoming event)
     /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`.
     /// The incoming event is the `pdu_id` passed to this method.
-    pub fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result<StateHashId> {
+    pub fn append_to_state(
+        &self,
+        new_pdu_id: &[u8],
+        new_pdu: &PduEvent,
+        globals: &super::globals::Globals,
+    ) -> Result<StateHashId> {
         let old_state =
             if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? {
                 // Store state for event. The state does not include the event itself.
@@ -553,6 +636,7 @@ impl Rooms {
                 self.stateid_pduid
                     .scan_prefix(&prefix)
                     .filter_map(|pdu| pdu.map_err(|e| error!("{}", e)).ok())
+                    // Chop the old state_hash out leaving behind the short key (u64)
                     .map(|(k, v)| (k.subslice(prefix.len(), k.len() - prefix.len()), v))
                     .collect::<HashMap<IVec, IVec>>()
             } else {
@@ -561,10 +645,26 @@ impl Rooms {
 
         if let Some(state_key) = &new_pdu.state_key {
             let mut new_state = old_state;
-            let mut pdu_key = new_pdu.kind.as_str().as_bytes().to_vec();
+            let mut pdu_key = new_pdu.kind.as_ref().as_bytes().to_vec();
             pdu_key.push(0xff);
             pdu_key.extend_from_slice(state_key.as_bytes());
-            new_state.insert(pdu_key.into(), new_pdu_id.into());
+
+            let short = match self.statekey_short.get(&pdu_key)? {
+                Some(short) => utils::u64_from_bytes(&short)
+                    .map_err(|_| Error::bad_database("Invalid short bytes in statekey_short."))?,
+                None => {
+                    let short = globals.next_count()?;
+                    self.statekey_short.insert(&pdu_key, &short.to_be_bytes())?;
+                    short
+                }
+            };
+
+            let new_pdu_id_short = new_pdu_id
+                .splitn(2, |&b| b == 0xff)
+                .nth(1)
+                .ok_or_else(|| Error::bad_database("Invalid pduid in state."))?;
+
+            new_state.insert((&short.to_be_bytes()).into(), new_pdu_id_short.into());
 
             let new_state_hash =
                 self.calculate_hash(&new_state.values().map(|b| &**b).collect::<Vec<_>>())?;
@@ -572,17 +672,12 @@ impl Rooms {
             let mut key = new_state_hash.to_vec();
             key.push(0xff);
 
-            // TODO: we could avoid writing to the DB on every state event by keeping
+            for (short, short_pdu_id) in new_state {
-            // track of the delta and write that every so often
-            for (key_without_prefix, pdu_id) in new_state {
                 let mut state_id = key.clone();
-                state_id.extend_from_slice(&key_without_prefix);
+                state_id.extend_from_slice(&short);
-                self.stateid_pduid.insert(&state_id, &pdu_id)?;
+                self.stateid_pduid.insert(&state_id, &short_pdu_id)?;
             }
 
-            self.roomid_statehash
-                .insert(new_pdu.room_id.as_bytes(), &*new_state_hash)?;
-
             Ok(new_state_hash)
         } else {
             Err(Error::bad_database(
@@ -591,7 +686,15 @@ impl Rooms {
         }
     }
 
+    pub fn set_room_state(&self, room_id: &RoomId, state_hash: &StateHashId) -> Result<()> {
+        self.roomid_statehash
+            .insert(room_id.as_bytes(), state_hash)?;
+
+        Ok(())
+    }
+
     /// Creates a new persisted data unit and adds it to a room.
+    #[allow(clippy::too_many_arguments)]
     pub fn build_and_append_pdu(
         &self,
         pdu_builder: PduBuilder,
@@ -601,6 +704,7 @@ impl Rooms {
         sending: &super::sending::Sending,
         admin: &super::admin::Admin,
         account_data: &super::account_data::AccountData,
+        appservice: &super::appservice::Appservice,
     ) -> Result<EventId> {
         let PduBuilder {
             event_type,
@@ -682,12 +786,12 @@ impl Rooms {
             #[allow(clippy::blocks_in_if_conditions)]
             if !match event_type {
                 EventType::RoomEncryption => {
-                    // Don't allow encryption events when it's disabled
+                    // Only allow encryption events if it's allowed in the config
-                    !globals.encryption_disabled()
+                    globals.allow_encryption()
                 }
                 EventType::RoomMember => {
                     let prev_event = self
-                        .get_pdu(prev_events.iter().next().ok_or(Error::BadRequest(
+                        .get_pdu(prev_events.get(0).ok_or(Error::BadRequest(
                             ErrorKind::Unknown,
                             "Membership can't be the first event",
                         ))?)?
@@ -703,7 +807,7 @@ impl Rooms {
                             sender: &sender,
                         },
                         prev_event,
-                        None,
+                        None, // TODO: third party invite
                         &auth_events
                             .iter()
                             .map(|((ty, key), pdu)| {
@@ -761,7 +865,7 @@ impl Rooms {
         }
 
         let mut pdu = PduEvent {
-            event_id: EventId::try_from("$thiswillbefilledinlater").expect("we know this is valid"),
+            event_id: ruma::event_id!("$thiswillbefilledinlater"),
             room_id: room_id.clone(),
             sender: sender.clone(),
             origin_server_ts: utils::millis_since_unix_epoch()
@@ -787,37 +891,38 @@ impl Rooms {
         };
 
         // Hash and sign
-        let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it");
+        let mut pdu_json =
-        pdu_json
+            utils::to_canonical_object(&pdu).expect("event is valid, we just created it");
-            .as_object_mut()
+
-            .expect("json is object")
+        pdu_json.remove("event_id");
-            .remove("event_id");
 
         // Add origin because synapse likes that (and it's required in the spec)
-        pdu_json
+        pdu_json.insert(
-            .as_object_mut()
+            "origin".to_owned(),
-            .expect("json is object")
+            to_canonical_value(globals.server_name())
-            .insert("origin".to_owned(), globals.server_name().as_str().into());
+                .expect("server name is a valid CanonicalJsonValue"),
+        );
 
         ruma::signatures::hash_and_sign_event(
             globals.server_name().as_str(),
             globals.keypair(),
             &mut pdu_json,
+            &RoomVersionId::Version6,
         )
         .expect("event is valid, we just created it");
 
         // Generate event id
         pdu.event_id = EventId::try_from(&*format!(
             "${}",
-            ruma::signatures::reference_hash(&pdu_json)
+            ruma::signatures::reference_hash(&pdu_json, &RoomVersionId::Version6)
                 .expect("ruma can calculate reference hashes")
         ))
         .expect("ruma's reference hashes are valid event ids");
 
-        pdu_json
+        pdu_json.insert(
-            .as_object_mut()
+            "event_id".to_owned(),
-            .expect("json is object")
+            to_canonical_value(&pdu.event_id).expect("EventId is a valid CanonicalJsonValue"),
-            .insert("event_id".to_owned(), pdu.event_id.to_string().into());
+        );
 
         // Increment the last index and use that
         // This is also the next_batch/since value
@@ -828,11 +933,11 @@ impl Rooms {
 
         // We append to state before appending the pdu, so we don't have a moment in time with the
         // pdu without it's state. This is okay because append_pdu can't fail.
-        self.append_to_state(&pdu_id, &pdu)?;
+        let statehashid = self.append_to_state(&pdu_id, &pdu, &globals)?;
 
         self.append_pdu(
             &pdu,
-            &pdu_json,
+            pdu_json,
             count,
             pdu_id.clone().into(),
             globals,
@@ -840,12 +945,79 @@ impl Rooms {
             admin,
         )?;
 
+        // We set the room state after inserting the pdu, so that we never have a moment in time
+        // where events in the current room state do not exist
+        self.set_room_state(&room_id, &statehashid)?;
+
         for server in self
             .room_servers(room_id)
             .filter_map(|r| r.ok())
             .filter(|server| &**server != globals.server_name())
         {
-            sending.send_pdu(server, &pdu_id)?;
+            sending.send_pdu(&server, &pdu_id)?;
+        }
+
+        for appservice in appservice.iter_all().filter_map(|r| r.ok()) {
+            if let Some(namespaces) = appservice.1.get("namespaces") {
+                let users = namespaces
+                    .get("users")
+                    .and_then(|users| users.as_sequence())
+                    .map_or_else(
+                        || Vec::new(),
+                        |users| {
+                            users
+                                .iter()
+                                .map(|users| {
+                                    users
+                                        .get("regex")
+                                        .and_then(|regex| regex.as_str())
+                                        .and_then(|regex| Regex::new(regex).ok())
+                                })
+                                .filter_map(|o| o)
+                                .collect::<Vec<_>>()
+                        },
+                    );
+                let aliases = namespaces
+                    .get("aliases")
+                    .and_then(|users| users.get("regex"))
+                    .and_then(|regex| regex.as_str())
+                    .and_then(|regex| Regex::new(regex).ok());
+                let rooms = namespaces
+                    .get("rooms")
+                    .and_then(|rooms| rooms.as_sequence());
+
+                let room_aliases = self.room_aliases(&room_id);
+
+                let bridge_user_id = appservice
+                    .1
+                    .get("sender_localpart")
+                    .and_then(|string| string.as_str())
+                    .and_then(|string| {
+                        UserId::parse_with_server_name(string, globals.server_name()).ok()
+                    });
+
+                if bridge_user_id.map_or(false, |bridge_user_id| {
+                    self.is_joined(&bridge_user_id, room_id).unwrap_or(false)
+                }) || users.iter().any(|users| {
+                    users.is_match(pdu.sender.as_str())
+                        || pdu.kind == EventType::RoomMember
+                            && pdu
+                                .state_key
+                                .as_ref()
+                                .map_or(false, |state_key| users.is_match(&state_key))
+                }) || aliases.map_or(false, |aliases| {
+                    room_aliases
+                        .filter_map(|r| r.ok())
+                        .any(|room_alias| aliases.is_match(room_alias.as_str()))
+                }) || rooms.map_or(false, |rooms| rooms.contains(&room_id.as_str().into()))
+                    || self
+                        .room_members(&room_id)
+                        .filter_map(|r| r.ok())
+                        .any(|member| users.iter().any(|regex| regex.is_match(member.as_str())))
+                {
+                    sending.send_pdu_appservice(&appservice.0, &pdu_id)?;
+                }
+            }
         }
 
         Ok(pdu.event_id)

src/database/rooms/edus.rs

@@ -6,7 +6,8 @@ use ruma::{
         AnyEvent as EduEvent, SyncEphemeralRoomEvent,
     },
     presence::PresenceState,
-    Raw, RoomId, UserId,
+    serde::Raw,
+    RoomId, UserId,
 };
 use std::{
     collections::HashMap,
@@ -385,8 +386,6 @@ impl RoomEdus {
             .take_while(|(_, timestamp)| current_timestamp - timestamp > 5 * 60_000)
         // 5 Minutes
         {
-            self.userid_lastpresenceupdate.remove(&user_id_bytes)?;
-
             // Send new presence events to set the user offline
             let count = globals.next_count()?.to_be_bytes();
             let user_id = utils::string_from_bytes(&user_id_bytes)
@@ -420,6 +419,11 @@ impl RoomEdus {
                     .expect("PresenceEvent can be serialized"),
                 )?;
             }
+
+            self.userid_lastpresenceupdate.insert(
+                &user_id.to_string().as_bytes(),
+                &utils::millis_since_unix_epoch().to_be_bytes(),
+            )?;
         }
 
         Ok(())

src/database/sending.rs

@@ -1,26 +1,43 @@
-use std::{collections::HashMap, convert::TryFrom, time::SystemTime};
+use std::{
+    collections::HashMap,
+    convert::TryFrom,
+    fmt::Debug,
+    sync::Arc,
+    time::{Duration, Instant, SystemTime},
+};
 
-use crate::{server_server, utils, Error, PduEvent, Result};
+use crate::{appservice_server, server_server, utils, Error, PduEvent, Result};
 use federation::transactions::send_transaction_message;
-use log::debug;
+use log::{error, info};
 use rocket::futures::stream::{FuturesUnordered, StreamExt};
-use ruma::{api::federation, ServerName};
+use ruma::{
+    api::{appservice, federation, OutgoingRequest},
+    ServerName,
+};
 use sled::IVec;
 use tokio::select;
+use tokio::sync::Semaphore;
 
 #[derive(Clone)]
 pub struct Sending {
     /// The state for a given state hash.
-    pub(super) servernamepduids: sled::Tree, // ServernamePduId = ServerName + PduId
+    pub(super) servernamepduids: sled::Tree, // ServernamePduId = (+)ServerName + PduId
-    pub(super) servercurrentpdus: sled::Tree, // ServerCurrentPdus = ServerName + PduId (pduid can be empty for reservation)
+    pub(super) servercurrentpdus: sled::Tree, // ServerCurrentPdus = (+)ServerName + PduId (pduid can be empty for reservation)
+    pub(super) maximum_requests: Arc<Semaphore>,
 }
 
 impl Sending {
-    pub fn start_handler(&self, globals: &super::globals::Globals, rooms: &super::rooms::Rooms) {
+    pub fn start_handler(
+        &self,
+        globals: &super::globals::Globals,
+        rooms: &super::rooms::Rooms,
+        appservice: &super::appservice::Appservice,
+    ) {
         let servernamepduids = self.servernamepduids.clone();
         let servercurrentpdus = self.servercurrentpdus.clone();
         let rooms = rooms.clone();
         let globals = globals.clone();
+        let appservice = appservice.clone();
 
         tokio::spawn(async move {
             let mut futures = FuturesUnordered::new();
@@ -28,55 +45,45 @@ impl Sending {
             // Retry requests we could not finish yet
             let mut current_transactions = HashMap::new();
 
-            for (server, pdu) in servercurrentpdus
+            for (server, pdu, is_appservice) in servercurrentpdus
                 .iter()
                 .filter_map(|r| r.ok())
-                .map(|(key, _)| {
+                .filter_map(|(key, _)| Self::parse_servercurrentpdus(key).ok())
-                    let mut parts = key.splitn(2, |&b| b == 0xff);
+                .filter(|(_, pdu, _)| !pdu.is_empty()) // Skip reservation key
-                    let server = parts.next().expect("splitn always returns one element");
-                    let pdu = parts.next().ok_or_else(|| {
-                        Error::bad_database("Invalid bytes in servercurrentpdus.")
-                    })?;
-
-                    Ok::<_, Error>((
-                        Box::<ServerName>::try_from(utils::string_from_bytes(&server).map_err(
-                            |_| {
-                                Error::bad_database(
-                                    "Invalid server bytes in server_currenttransaction",
-                                )
-                            },
-                        )?)
-                        .map_err(|_| {
-                            Error::bad_database(
-                                "Invalid server string in server_currenttransaction",
-                            )
-                        })?,
-                        IVec::from(pdu),
-                    ))
-                })
-                .filter_map(|r| r.ok())
-                .filter(|(_, pdu)| !pdu.is_empty()) // Skip reservation key
                 .take(50)
             // This should not contain more than 50 anyway
             {
                 current_transactions
-                    .entry(server)
+                    .entry((server, is_appservice))
                     .or_insert_with(Vec::new)
                     .push(pdu);
             }
 
-            for (server, pdus) in current_transactions {
+            for ((server, is_appservice), pdus) in current_transactions {
-                futures.push(Self::handle_event(server, pdus, &globals, &rooms));
+                futures.push(Self::handle_event(
+                    server,
+                    is_appservice,
+                    pdus,
+                    &globals,
+                    &rooms,
+                    &appservice,
+                ));
             }
 
+            let mut last_failed_try: HashMap<Box<ServerName>, (u32, Instant)> = HashMap::new();
+
             let mut subscriber = servernamepduids.watch_prefix(b"");
             loop {
                 select! {
-                    Some(server) = futures.next() => {
+                    Some(response) = futures.next() => {
-                        debug!("response: {:?}", &server);
+                        match response {
-                        match server {
+                            Ok((server, is_appservice)) => {
-                            Ok((server, _response)) => {
+                                let mut prefix = if is_appservice {
-                                let mut prefix = server.as_bytes().to_vec();
+                                    "+".as_bytes().to_vec()
+                                } else {
+                                    Vec::new()
+                                };
+                                prefix.extend_from_slice(server.as_bytes());
                                 prefix.push(0xff);
 
                                 for key in servercurrentpdus
@@ -109,14 +116,31 @@ impl Sending {
                                         servernamepduids.remove(&current_key).unwrap();
                                     }
 
-                                    futures.push(Self::handle_event(server, new_pdus, &globals, &rooms));
+                                    futures.push(Self::handle_event(server, is_appservice, new_pdus, &globals, &rooms, &appservice));
                                 } else {
                                     servercurrentpdus.remove(&prefix).unwrap();
                                     // servercurrentpdus with the prefix should be empty now
                                 }
                             }
-                            Err((_server, _e)) => {
+                            Err((server, is_appservice, e)) => {
-                                // TODO: exponential backoff
+                                info!("Couldn't send transaction to {}\n{}", server, e);
+                                let mut prefix = if is_appservice {
+                                    "+".as_bytes().to_vec()
+                                } else {
+                                    Vec::new()
+                                };
+                                prefix.extend_from_slice(server.as_bytes());
+                                prefix.push(0xff);
+
+                                last_failed_try.insert(server.clone(), match last_failed_try.get(&server) {
+                                    Some(last_failed) => {
+                                        (last_failed.0+1, Instant::now())
+                                    },
+                                    None => {
+                                        (1, Instant::now())
+                                    }
+                                });
+                                servercurrentpdus.remove(&prefix).unwrap();
                             }
                         };
                     },
@@ -125,24 +149,48 @@ impl Sending {
                             let servernamepduid = key.clone();
                             let mut parts = servernamepduid.splitn(2, |&b| b == 0xff);
 
-                            if let Some((server, pdu_id)) = utils::string_from_bytes(
+                            if let Some((server, is_appservice, pdu_id)) = utils::string_from_bytes(
                                     parts
                                         .next()
                                         .expect("splitn will always return 1 or more elements"),
                                 )
                                 .map_err(|_| Error::bad_database("ServerName in servernamepduid bytes are invalid."))
-                                .and_then(|server_str|Box::<ServerName>::try_from(server_str)
+                                .map(|server_str| {
-                                    .map_err(|_| Error::bad_database("ServerName in servernamepduid is invalid.")))
+                                    // Appservices start with a plus
+                                    if server_str.starts_with("+") {
+                                        (server_str[1..].to_owned(), true)
+                                    } else {
+                                        (server_str, false)
+                                    }
+                                })
+                                .and_then(|(server_str, is_appservice)| Box::<ServerName>::try_from(server_str)
+                                    .map_err(|_| Error::bad_database("ServerName in servernamepduid is invalid.")).map(|s| (s, is_appservice)))
                                 .ok()
-                                .and_then(|server| parts
+                                .and_then(|(server, is_appservice)| parts
                                     .next()
                                     .ok_or_else(|| Error::bad_database("Invalid servernamepduid in db."))
                                     .ok()
-                                    .map(|pdu_id| (server, pdu_id))
+                                    .map(|pdu_id| (server, is_appservice, pdu_id))
                                 )
-                                // TODO: exponential backoff
+                                .filter(|(server, is_appservice, _)| {
-                                .filter(|(server, _)| {
+                                    if last_failed_try.get(server).map_or(false, |(tries, instant)| {
-                                    let mut prefix = server.to_string().as_bytes().to_vec();
+                                        // Fail if a request has failed recently (exponential backoff)
+                                        let mut min_elapsed_duration = Duration::from_secs(60) * *tries * *tries;
+                                        if min_elapsed_duration > Duration::from_secs(60*60*24) {
+                                            min_elapsed_duration = Duration::from_secs(60*60*24);
+                                        }
+
+                                        instant.elapsed() < min_elapsed_duration
+                                    }) {
+                                        return false;
+                                    }
+
+                                    let mut prefix = if *is_appservice {
+                                        "+".as_bytes().to_vec()
+                                    } else {
+                                        Vec::new()
+                                    };
+                                    prefix.extend_from_slice(server.as_bytes());
                                     prefix.push(0xff);
 
                                     servercurrentpdus
@@ -153,7 +201,7 @@ impl Sending {
                                 servercurrentpdus.insert(&key, &[]).unwrap();
                                 servernamepduids.remove(&key).unwrap();
 
-                                futures.push(Self::handle_event(server, vec![pdu_id.into()], &globals, &rooms));
+                                futures.push(Self::handle_event(server, is_appservice, vec![pdu_id.into()], &globals, &rooms, &appservice));
                             }
                         }
                     }
@@ -162,7 +210,7 @@ impl Sending {
         });
     }
 
-    pub fn send_pdu(&self, server: Box<ServerName>, pdu_id: &[u8]) -> Result<()> {
+    pub fn send_pdu(&self, server: &ServerName, pdu_id: &[u8]) -> Result<()> {
         let mut key = server.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(pdu_id);
@@ -171,46 +219,161 @@ impl Sending {
         Ok(())
     }
 
+    pub fn send_pdu_appservice(&self, appservice_id: &str, pdu_id: &[u8]) -> Result<()> {
+        let mut key = "+".as_bytes().to_vec();
+        key.extend_from_slice(appservice_id.as_bytes());
+        key.push(0xff);
+        key.extend_from_slice(pdu_id);
+        self.servernamepduids.insert(key, b"")?;
+
+        Ok(())
+    }
+
     async fn handle_event(
         server: Box<ServerName>,
+        is_appservice: bool,
         pdu_ids: Vec<IVec>,
         globals: &super::globals::Globals,
         rooms: &super::rooms::Rooms,
-    ) -> std::result::Result<
+        appservice: &super::appservice::Appservice,
-        (Box<ServerName>, send_transaction_message::v1::Response),
+    ) -> std::result::Result<(Box<ServerName>, bool), (Box<ServerName>, bool, Error)> {
-        (Box<ServerName>, Error),
+        if is_appservice {
-    > {
+            let pdu_jsons = pdu_ids
-        let pdu_jsons = pdu_ids
+                .iter()
-            .iter()
+                .map(|pdu_id| {
-            .map(|pdu_id| {
+                    Ok::<_, (Box<ServerName>, Error)>(
-                Ok::<_, (Box<ServerName>, Error)>(PduEvent::convert_to_outgoing_federation_event(
+                        rooms
-                    rooms
+                            .get_pdu_from_id(pdu_id)
-                        .get_pdu_json_from_id(pdu_id)
+                            .map_err(|e| (server.clone(), e))?
-                        .map_err(|e| (server.clone(), e))?
+                            .ok_or_else(|| {
-                        .ok_or_else(|| {
+                                (
-                            (
+                                    server.clone(),
-                                server.clone(),
+                                    Error::bad_database(
-                                Error::bad_database("Event in servernamepduids not found in db."),
+                                        "Event in servernamepduids not found in db.",
+                                    ),
+                                )
+                            })?
+                            .to_any_event(),
+                    )
+                })
+                .filter_map(|r| r.ok())
+                .collect::<Vec<_>>();
+            appservice_server::send_request(
+                &globals,
+                appservice
+                    .get_registration(server.as_str())
+                    .unwrap()
+                    .unwrap(), // TODO: handle error
+                appservice::event::push_events::v1::Request {
+                    events: &pdu_jsons,
+                    txn_id: &utils::random_string(16),
+                },
+            )
+            .await
+            .map(|_response| (server.clone(), is_appservice))
+            .map_err(|e| (server, is_appservice, e))
+        } else {
+            let pdu_jsons = pdu_ids
+                .iter()
+                .map(|pdu_id| {
+                    Ok::<_, (Box<ServerName>, Error)>(
+                        // TODO: check room version and remove event_id if needed
+                        serde_json::from_str(
+                            PduEvent::convert_to_outgoing_federation_event(
+                                rooms
+                                    .get_pdu_json_from_id(pdu_id)
+                                    .map_err(|e| (server.clone(), e))?
+                                    .ok_or_else(|| {
+                                        (
+                                            server.clone(),
+                                            Error::bad_database(
+                                                "Event in servernamepduids not found in db.",
+                                            ),
+                                        )
+                                    })?,
                             )
-                        })?,
+                            .json()
-                ))
+                            .get(),
-            })
+                        )
-            .filter_map(|r| r.ok())
+                        .expect("Raw<..> is always valid"),
-            .collect::<Vec<_>>();
+                    )
+                })
+                .filter_map(|r| r.ok())
+                .collect::<Vec<_>>();
 
-        server_server::send_request(
+            server_server::send_request(
-            &globals,
+                &globals,
-            server.clone(),
+                server.clone(),
-            send_transaction_message::v1::Request {
+                send_transaction_message::v1::Request {
-                origin: globals.server_name(),
+                    origin: globals.server_name(),
-                pdus: &pdu_jsons,
+                    pdus: &pdu_jsons,
-                edus: &[],
+                    edus: &[],
-                origin_server_ts: SystemTime::now(),
+                    origin_server_ts: SystemTime::now(),
-                transaction_id: &utils::random_string(16),
+                    transaction_id: &utils::random_string(16),
-            },
+                },
-        )
+            )
-        .await
+            .await
-        .map(|response| (server.clone(), response))
+            .map(|_response| (server.clone(), is_appservice))
-        .map_err(|e| (server, e))
+            .map_err(|e| (server, is_appservice, e))
+        }
+    }
+
+    fn parse_servercurrentpdus(key: IVec) -> Result<(Box<ServerName>, IVec, bool)> {
+        let mut parts = key.splitn(2, |&b| b == 0xff);
+        let server = parts.next().expect("splitn always returns one element");
+        let pdu = parts
+            .next()
+            .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+
+        let server = utils::string_from_bytes(&server).map_err(|_| {
+            Error::bad_database("Invalid server bytes in server_currenttransaction")
+        })?;
+
+        // Appservices start with a plus
+        let (server, is_appservice) = if server.starts_with("+") {
+            (&server[1..], true)
+        } else {
+            (&*server, false)
+        };
+
+        Ok::<_, Error>((
+            Box::<ServerName>::try_from(server).map_err(|_| {
+                Error::bad_database("Invalid server string in server_currenttransaction")
+            })?,
+            IVec::from(pdu),
+            is_appservice,
+        ))
+    }
+
+    pub async fn send_federation_request<T: OutgoingRequest>(
+        &self,
+        globals: &crate::database::globals::Globals,
+        destination: Box<ServerName>,
+        request: T,
+    ) -> Result<T::IncomingResponse>
+    where
+        T: Debug,
+    {
+        let permit = self.maximum_requests.acquire().await;
+        let response = server_server::send_request(globals, destination, request).await;
+        drop(permit);
+
+        response
+    }
+
+    pub async fn send_appservice_request<T: OutgoingRequest>(
+        &self,
+        globals: &crate::database::globals::Globals,
+        registration: serde_yaml::Value,
+        request: T,
+    ) -> Result<T::IncomingResponse>
+    where
+        T: Debug,
+    {
+        let permit = self.maximum_requests.acquire().await;
+        let response = appservice_server::send_request(globals, registration, request).await;
+        drop(permit);
+
+        response
     }
 }

src/database/transaction_ids.rs

@@ -11,13 +11,13 @@ impl TransactionIds {
     pub fn add_txnid(
         &self,
         user_id: &UserId,
-        device_id: &DeviceId,
+        device_id: Option<&DeviceId>,
         txn_id: &str,
         data: &[u8],
     ) -> Result<()> {
         let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(device_id.as_bytes());
+        key.extend_from_slice(device_id.map(|d| d.as_bytes()).unwrap_or_default());
         key.push(0xff);
         key.extend_from_slice(txn_id.as_bytes());
 
@@ -29,12 +29,12 @@ impl TransactionIds {
     pub fn existing_txnid(
         &self,
         user_id: &UserId,
-        device_id: &DeviceId,
+        device_id: Option<&DeviceId>,
         txn_id: &str,
     ) -> Result<Option<IVec>> {
         let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(device_id.as_bytes());
+        key.extend_from_slice(device_id.map(|d| d.as_bytes()).unwrap_or_default());
         key.push(0xff);
         key.extend_from_slice(txn_id.as_bytes());
 

src/database/users.rs

@@ -8,9 +8,10 @@ use ruma::{
             keys::{CrossSigningKey, OneTimeKey},
         },
     },
-    encryption::IncomingDeviceKeys,
+    encryption::DeviceKeys,
     events::{AnyToDeviceEvent, EventType},
-    DeviceId, DeviceKeyAlgorithm, DeviceKeyId, Raw, UserId,
+    serde::Raw,
+    DeviceId, DeviceKeyAlgorithm, DeviceKeyId, UserId,
 };
 use std::{collections::BTreeMap, convert::TryFrom, mem, time::SystemTime};
 
@@ -401,7 +402,7 @@ impl Users {
         &self,
         user_id: &UserId,
         device_id: &DeviceId,
-        device_keys: &IncomingDeviceKeys,
+        device_keys: &DeviceKeys,
         rooms: &super::rooms::Rooms,
         globals: &super::globals::Globals,
     ) -> Result<()> {
@@ -631,7 +632,7 @@ impl Users {
         &self,
         user_id: &UserId,
         device_id: &DeviceId,
-    ) -> Result<Option<IncomingDeviceKeys>> {
+    ) -> Result<Option<DeviceKeys>> {
         let mut key = user_id.to_string().as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(device_id.as_bytes());

src/error.rs

@@ -34,7 +34,7 @@ pub enum Error {
         #[from]
         source: image::error::ImageError,
     },
-    #[error("Could not connect to server.")]
+    #[error("Could not connect to server: {source}")]
     ReqwestError {
         #[from]
         source: reqwest::Error,
@@ -121,33 +121,45 @@ impl log::Log for ConduitLogger {
     fn log(&self, record: &log::Record<'_>) {
         let output = format!("{} - {}", record.level(), record.args());
 
-        println!("{}", output);
-
         if self.enabled(record.metadata())
-            && record
+            && (record
                 .module_path()
                 .map_or(false, |path| path.starts_with("conduit::"))
+                || record
+                    .module_path()
+                    .map_or(true, |path| !path.starts_with("rocket::")) // Rockets logs are annoying
+                    && record.metadata().level() <= log::Level::Warn)
         {
+            let first_line = output
+                .lines()
+                .next()
+                .expect("lines always returns one item");
+
+            eprintln!("{}", output);
+
+            let mute_duration = match record.metadata().level() {
+                log::Level::Error => Duration::from_secs(60 * 5), // 5 minutes
+                log::Level::Warn => Duration::from_secs(60 * 60 * 24), // A day
+                _ => Duration::from_secs(60 * 60 * 24 * 7),       // A week
+            };
+
             if self
                 .last_logs
                 .read()
                 .unwrap()
-                .get(&output)
+                .get(first_line)
-                .map_or(false, |i| i.elapsed() < Duration::from_secs(60 * 30))
+                .map_or(false, |i| i.elapsed() < mute_duration)
+            // Don't post this log again for some time
             {
                 return;
             }
 
             if let Ok(mut_last_logs) = &mut self.last_logs.try_write() {
-                mut_last_logs.insert(output.clone(), Instant::now());
+                mut_last_logs.insert(first_line.to_owned(), Instant::now());
             }
 
-            self.db.admin.send(AdminCommand::SendTextMessage(
+            self.db.admin.send(AdminCommand::SendMessage(
-                message::TextMessageEventContent {
+                message::MessageEventContent::notice_plain(output),
-                    body: output,
-                    formatted: None,
-                    relates_to: None,
-                },
             ));
         }
     }

src/lib.rs

@@ -1,3 +1,4 @@
+pub mod appservice_server;
 pub mod client_server;
 mod database;
 mod error;

src/main.rs

@@ -1,5 +1,6 @@
 #![warn(rust_2018_idioms)]
 
+pub mod appservice_server;
 pub mod client_server;
 pub mod server_server;
 
@@ -12,18 +13,33 @@ mod utils;
 
 pub use database::Database;
 pub use error::{ConduitLogger, Error, Result};
-use log::LevelFilter;
 pub use pdu::PduEvent;
 pub use rocket::State;
+use ruma::api::client::error::ErrorKind;
 pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse};
 
-use rocket::{fairing::AdHoc, routes};
+use log::LevelFilter;
+use rocket::figment::{
+    providers::{Env, Format, Toml},
+    Figment,
+};
+use rocket::{catch, catchers, fairing::AdHoc, routes, Request};
 
 fn setup_rocket() -> rocket::Rocket {
     // Force log level off, so we can use our own logger
-    std::env::set_var("ROCKET_LOG", "off");
+    std::env::set_var("CONDUIT_LOG_LEVEL", "off");
 
-    rocket::ignite()
+    let config =
+        Figment::from(rocket::Config::release_default())
+            .merge(
+                Toml::file(Env::var("CONDUIT_CONFIG").expect(
+                    "The CONDUIT_CONFIG env var needs to be set. Example: /etc/conduit.toml",
+                ))
+                .nested(),
+            )
+            .merge(Env::prefixed("CONDUIT_").global());
+
+    rocket::custom(config)
         .mount(
             "/",
             routes![
@@ -40,7 +56,12 @@ fn setup_rocket() -> rocket::Rocket {
                 client_server::get_capabilities_route,
                 client_server::get_pushrules_all_route,
                 client_server::set_pushrule_route,
+                client_server::get_pushrule_route,
                 client_server::set_pushrule_enabled_route,
+                client_server::get_pushrule_enabled_route,
+                client_server::get_pushrule_actions_route,
+                client_server::set_pushrule_actions_route,
+                client_server::delete_pushrule_route,
                 client_server::get_room_event_route,
                 client_server::get_filter_route,
                 client_server::create_filter_route,
@@ -69,6 +90,7 @@ fn setup_rocket() -> rocket::Rocket {
                 client_server::get_backup_key_sessions_route,
                 client_server::get_backup_keys_route,
                 client_server::set_read_marker_route,
+                client_server::set_receipt_route,
                 client_server::create_typing_event_route,
                 client_server::create_room_route,
                 client_server::redact_event_route,
@@ -123,9 +145,9 @@ fn setup_rocket() -> rocket::Rocket {
                 client_server::get_pushers_route,
                 client_server::set_pushers_route,
                 client_server::upgrade_room_route,
-                server_server::get_server_version,
+                server_server::get_server_version_route,
-                server_server::get_server_keys,
+                server_server::get_server_keys_route,
-                server_server::get_server_keys_deprecated,
+                server_server::get_server_keys_deprecated_route,
                 server_server::get_public_rooms_route,
                 server_server::get_public_rooms_filtered_route,
                 server_server::send_transaction_message_route,
@@ -133,10 +155,24 @@ fn setup_rocket() -> rocket::Rocket {
                 server_server::get_profile_information_route,
             ],
         )
-        .attach(AdHoc::on_attach("Config", |mut rocket| async {
+        .register(catchers![
-            let data = Database::load_or_create(rocket.config().await).expect("valid config");
+            not_found_catcher,
+            forbidden_catcher,
+            unknown_token_catcher,
+            missing_token_catcher,
+            bad_json_catcher
+        ])
+        .attach(AdHoc::on_attach("Config", |rocket| async {
+            let config = rocket
+                .figment()
+                .extract()
+                .expect("It looks like your config is invalid. Please take a look at the error");
+            let data = Database::load_or_create(config)
+                .await
+                .expect("config is valid");
 
-            data.sending.start_handler(&data.globals, &data.rooms);
+            data.sending
+                .start_handler(&data.globals, &data.rooms, &data.appservice);
             log::set_boxed_logger(Box::new(ConduitLogger {
                 db: data.clone(),
                 last_logs: Default::default(),
@@ -152,3 +188,31 @@ fn setup_rocket() -> rocket::Rocket {
 async fn main() {
     setup_rocket().launch().await.unwrap();
 }
+
+#[catch(404)]
+fn not_found_catcher(_req: &'_ Request<'_>) -> String {
+    "404 Not Found".to_owned()
+}
+
+#[catch(580)]
+fn forbidden_catcher() -> Result<()> {
+    Err(Error::BadRequest(ErrorKind::Forbidden, "Forbidden."))
+}
+
+#[catch(581)]
+fn unknown_token_catcher() -> Result<()> {
+    Err(Error::BadRequest(
+        ErrorKind::UnknownToken { soft_logout: false },
+        "Unknown token.",
+    ))
+}
+
+#[catch(582)]
+fn missing_token_catcher() -> Result<()> {
+    Err(Error::BadRequest(ErrorKind::MissingToken, "Missing token."))
+}
+
+#[catch(583)]
+fn bad_json_catcher() -> Result<()> {
+    Err(Error::BadRequest(ErrorKind::BadJson, "Bad json."))
+}

src/pdu.rs

@@ -5,11 +5,17 @@ use ruma::{
         pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent,
         AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent,
     },
-    EventId, Raw, RoomId, ServerKeyId, ServerName, UserId,
+    serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue, Raw},
+    EventId, RoomId, RoomVersionId, ServerName, ServerSigningKeyId, UserId,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH};
+use std::{
+    collections::BTreeMap,
+    convert::{TryFrom, TryInto},
+    sync::Arc,
+    time::UNIX_EPOCH,
+};
 
 #[derive(Deserialize, Serialize, Debug)]
 pub struct PduEvent {
@@ -30,7 +36,7 @@ pub struct PduEvent {
     #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
     pub unsigned: serde_json::Map<String, serde_json::Value>,
     pub hashes: EventHash,
-    pub signatures: BTreeMap<Box<ServerName>, BTreeMap<ServerKeyId, String>>,
+    pub signatures: BTreeMap<Box<ServerName>, BTreeMap<ServerSigningKeyId, String>>,
 }
 
 impl PduEvent {
@@ -199,34 +205,35 @@ impl PduEvent {
         serde_json::from_value(json).expect("Raw::from_value always works")
     }
 
+    /// This does not return a full `Pdu` it is only to satisfy ruma's types.
     pub fn convert_to_outgoing_federation_event(
-        mut pdu_json: serde_json::Value,
+        mut pdu_json: CanonicalJsonObject,
-    ) -> Raw<ruma::events::pdu::PduStub> {
+    ) -> Raw<ruma::events::pdu::Pdu> {
-        if let Some(unsigned) = pdu_json
+        if let Some(CanonicalJsonValue::Object(unsigned)) = pdu_json.get_mut("unsigned") {
-            .as_object_mut()
+            unsigned.remove("transaction_id");
-            .expect("json is object")
-            .get_mut("unsigned")
-        {
-            unsigned
-                .as_object_mut()
-                .expect("unsigned is object")
-                .remove("transaction_id");
         }
 
-        pdu_json
+        pdu_json.remove("event_id");
-            .as_object_mut()
-            .expect("json is object")
-            .remove("event_id");
 
-        serde_json::from_value::<Raw<_>>(pdu_json).expect("Raw::from_value always works")
+        // TODO: another option would be to convert it to a canonical string to validate size
+        // and return a Result<Raw<...>>
+        // serde_json::from_str::<Raw<_>>(
+        //     ruma::serde::to_canonical_json_string(pdu_json).expect("CanonicalJson is valid serde_json::Value"),
+        // )
+        // .expect("Raw::from_value always works")
+
+        serde_json::from_value::<Raw<_>>(
+            serde_json::to_value(pdu_json).expect("CanonicalJson is valid serde_json::Value"),
+        )
+        .expect("Raw::from_value always works")
     }
 }
 
 impl From<&state_res::StateEvent> for PduEvent {
     fn from(pdu: &state_res::StateEvent) -> Self {
         Self {
-            event_id: pdu.event_id().clone(),
+            event_id: pdu.event_id(),
-            room_id: pdu.room_id().unwrap().clone(),
+            room_id: pdu.room_id().clone(),
             sender: pdu.sender().clone(),
             origin_server_ts: (pdu
                 .origin_server_ts()
@@ -252,35 +259,56 @@ impl From<&state_res::StateEvent> for PduEvent {
 impl PduEvent {
     pub fn convert_for_state_res(&self) -> Arc<state_res::StateEvent> {
         Arc::new(
-            serde_json::from_value(json!({
+            // For consistency of eventId (just in case) we use the one
-                "event_id": self.event_id,
+            // generated by conduit for everything.
-                "room_id": self.room_id,
+            state_res::StateEvent::from_id_value(
-                "sender": self.sender,
+                self.event_id.clone(),
-                "origin_server_ts": self.origin_server_ts,
+                json!({
-                "type": self.kind,
+                    "event_id": self.event_id,
-                "content": self.content,
+                    "room_id": self.room_id,
-                "state_key": self.state_key,
+                    "sender": self.sender,
-                "prev_events": self.prev_events
+                    "origin_server_ts": self.origin_server_ts,
-                    .iter()
+                    "type": self.kind,
-                    // TODO How do we create one of these
+                    "content": self.content,
-                    .map(|id| (id, EventHash { sha256: "hello".into() }))
+                    "state_key": self.state_key,
-                    .collect::<Vec<_>>(),
+                    "prev_events": self.prev_events,
-                "depth": self.depth,
+                    "depth": self.depth,
-                "auth_events": self.auth_events
+                    "auth_events": self.auth_events,
-                    .iter()
+                    "redacts": self.redacts,
-                    // TODO How do we create one of these
+                    "unsigned": self.unsigned,
-                    .map(|id| (id, EventHash { sha256: "hello".into() }))
+                    "hashes": self.hashes,
-                    .collect::<Vec<_>>(),
+                    "signatures": self.signatures,
-                "redacts": self.redacts,
+                }),
-                "unsigned": self.unsigned,
+            )
-                "hashes": self.hashes,
-                "signatures": self.signatures,
-            }))
             .expect("all conduit PDUs are state events"),
         )
     }
 }
 
+/// Generates a correct eventId for the incoming pdu.
+///
+/// Returns a tuple of the new `EventId` and the PDU with the eventId inserted as a `serde_json::Value`.
+pub(crate) fn process_incoming_pdu(
+    pdu: &Raw<ruma::events::pdu::Pdu>,
+) -> (EventId, CanonicalJsonObject) {
+    let mut value =
+        serde_json::from_str(pdu.json().get()).expect("A Raw<...> is always valid JSON");
+
+    let event_id = EventId::try_from(&*format!(
+        "${}",
+        ruma::signatures::reference_hash(&value, &RoomVersionId::Version6)
+            .expect("ruma can calculate reference hashes")
+    ))
+    .expect("ruma's reference hashes are valid event ids");
+
+    value.insert(
+        "event_id".to_owned(),
+        to_canonical_value(&event_id).expect("EventId is a valid CanonicalJsonValue"),
+    );
+
+    (event_id, value)
+}
+
 /// Build the start of a PDU in order to add it to the `Database`.
 #[derive(Debug, Deserialize)]
 pub struct PduBuilder {

src/push_rules.rs

@@ -1,15 +1,18 @@
 use ruma::{
     push::{
-        Action, ConditionalPushRule, ConditionalPushRuleInit, PatternedPushRule,
+        Action, ConditionalPushRule, ConditionalPushRuleInit, ContentPushRule, OverridePushRule,
-        PatternedPushRuleInit, PushCondition, RoomMemberCountIs, Ruleset, Tweak,
+        PatternedPushRule, PatternedPushRuleInit, PushCondition, RoomMemberCountIs, Ruleset, Tweak,
+        UnderridePushRule,
     },
     UserId,
 };
 
 pub fn default_pushrules(user_id: &UserId) -> Ruleset {
     let mut rules = Ruleset::default();
-    rules.content = vec![contains_user_name_rule(&user_id)];
+
-    rules.override_ = vec![
+    rules.add(ContentPushRule(contains_user_name_rule(&user_id)));
+
+    for rule in vec![
         master_rule(),
         suppress_notices_rule(),
         invite_for_me_rule(),
@@ -17,14 +20,20 @@ pub fn default_pushrules(user_id: &UserId) -> Ruleset {
         contains_display_name_rule(),
         tombstone_rule(),
         roomnotif_rule(),
-    ];
+    ] {
-    rules.underride = vec![
+        rules.add(OverridePushRule(rule));
+    }
+
+    for rule in vec![
         call_rule(),
         encrypted_room_one_to_one_rule(),
         room_one_to_one_rule(),
         message_rule(),
         encrypted_rule(),
-    ];
+    ] {
+        rules.add(UnderridePushRule(rule));
+    }
+
     rules
 }
 

src/ruma_wrapper.rs

@@ -1,17 +1,22 @@
 use crate::Error;
 use ruma::{
-    api::{Outgoing, OutgoingRequest},
+    api::{AuthScheme, OutgoingRequest},
     identifiers::{DeviceId, UserId},
+    Outgoing,
+};
+use std::{
+    convert::{TryFrom, TryInto},
+    ops::Deref,
 };
-use std::{convert::TryFrom, convert::TryInto, ops::Deref};
 
 #[cfg(feature = "conduit_bin")]
 use {
     crate::utils,
-    log::warn,
+    log::{debug, warn},
     rocket::{
         data::{
-            Data, FromDataFuture, FromTransformedData, Transform, TransformFuture, Transformed,
+            ByteUnit, Data, FromDataFuture, FromTransformedData, Transform, TransformFuture,
+            Transformed,
         },
         http::Status,
         outcome::Outcome::*,
@@ -29,6 +34,7 @@ pub struct Ruma<T: Outgoing> {
     pub sender_user: Option<UserId>,
     pub sender_device: Option<Box<DeviceId>>,
     pub json_body: Option<Box<serde_json::value::RawValue>>, // This is None when body is not a valid string
+    pub from_appservice: bool,
 }
 
 #[cfg(feature = "conduit_bin")]
@@ -39,7 +45,7 @@ where
         http::request::Request<std::vec::Vec<u8>>,
     >>::Error: std::fmt::Debug,
 {
-    type Error = (); // TODO: Better error handling
+    type Error = ();
     type Owned = Data;
     type Borrowed = Self::Owned;
 
@@ -61,27 +67,75 @@ where
                 .await
                 .expect("database was loaded");
 
-            let (sender_user, sender_device) = if T::METADATA.requires_authentication {
+            // Get token from header or query value
-                // Get token from header or query value
+            let token = request
-                let token = match request
+                .headers()
-                    .headers()
+                .get_one("Authorization")
-                    .get_one("Authorization")
+                .map(|s| s[7..].to_owned()) // Split off "Bearer "
-                    .map(|s| s[7..].to_owned()) // Split off "Bearer "
+                .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok()));
-                    .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok()))
-                {
-                    // TODO: M_MISSING_TOKEN
-                    None => return Failure((Status::Unauthorized, ())),
-                    Some(token) => token,
-                };
 
-                // Check if token is valid
+            let (sender_user, sender_device, from_appservice) = if let Some((_id, registration)) =
-                match db.users.find_from_token(&token).unwrap() {
+                db.appservice
-                    // TODO: M_UNKNOWN_TOKEN
+                    .iter_all()
-                    None => return Failure((Status::Unauthorized, ())),
+                    .filter_map(|r| r.ok())
-                    Some((user_id, device_id)) => (Some(user_id), Some(device_id.into())),
+                    .find(|(_id, registration)| {
+                        registration
+                            .get("as_token")
+                            .and_then(|as_token| as_token.as_str())
+                            .map_or(false, |as_token| {
+                                dbg!(token.as_deref()) == dbg!(Some(as_token))
+                            })
+                    }) {
+                match T::METADATA.authentication {
+                    AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
+                        let user_id = request.get_query_value::<String>("user_id").map_or_else(
+                            || {
+                                UserId::parse_with_server_name(
+                                    registration
+                                        .get("sender_localpart")
+                                        .unwrap()
+                                        .as_str()
+                                        .unwrap(),
+                                    db.globals.server_name(),
+                                )
+                                .unwrap()
+                            },
+                            |string| {
+                                UserId::try_from(string.expect("parsing to string always works"))
+                                    .unwrap()
+                            },
+                        );
+
+                        if !db.users.exists(&user_id).unwrap() {
+                            // Forbidden
+                            return Failure((Status::raw(580), ()));
+                        }
+
+                        // TODO: Check if appservice is allowed to be that user
+                        (Some(user_id), None, true)
+                    }
+                    AuthScheme::ServerSignatures => (None, None, true),
+                    AuthScheme::None => (None, None, true),
                 }
             } else {
-                (None, None)
+                match T::METADATA.authentication {
+                    AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
+                        if let Some(token) = token {
+                            match db.users.find_from_token(&token).unwrap() {
+                                // Unknown Token
+                                None => return Failure((Status::raw(581), ())),
+                                Some((user_id, device_id)) => {
+                                    (Some(user_id), Some(device_id.into()), false)
+                                }
+                            }
+                        } else {
+                            // Missing Token
+                            return Failure((Status::raw(582), ()));
+                        }
+                    }
+                    AuthScheme::ServerSignatures => (None, None, false),
+                    AuthScheme::None => (None, None, false),
+                }
             };
 
             let mut http_request = http::Request::builder()
@@ -92,12 +146,12 @@ where
             }
 
             let limit = db.globals.max_request_size();
-            let mut handle = data.open().take(limit.into());
+            let mut handle = data.open(ByteUnit::Byte(limit.into()));
             let mut body = Vec::new();
             handle.read_to_end(&mut body).await.unwrap();
 
             let http_request = http_request.body(body.clone()).unwrap();
-            log::debug!("{:?}", http_request);
+            debug!("{:?}", http_request);
 
             match <T as Outgoing>::Incoming::try_from(http_request) {
                 Ok(t) => Success(Ruma {
@@ -108,10 +162,11 @@ where
                     json_body: utils::string_from_bytes(&body)
                         .ok()
                         .and_then(|s| serde_json::value::RawValue::from_string(s).ok()),
+                    from_appservice,
                 }),
                 Err(e) => {
                     warn!("{:?}", e);
-                    Failure((Status::BadRequest, ()))
+                    Failure((Status::raw(583), ()))
                 }
             }
         })

src/server_server.rs

@@ -1,14 +1,15 @@
-use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma};
+use crate::{client_server, utils, ConduitResult, Database, Error, PduEvent, Result, Ruma};
 use get_profile_information::v1::ProfileField;
 use http::header::{HeaderValue, AUTHORIZATION, HOST};
-use log::warn;
+use log::{info, warn};
 use rocket::{get, post, put, response::content::Json, State};
 use ruma::{
     api::{
         federation::{
             directory::{get_public_rooms, get_public_rooms_filtered},
             discovery::{
-                get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey,
+                get_server_keys, get_server_version::v1 as get_server_version, ServerSigningKeys,
+                VerifyKey,
             },
             event::get_missing_events,
             query::get_profile_information,
@@ -17,37 +18,15 @@ use ruma::{
         OutgoingRequest,
     },
     directory::{IncomingFilter, IncomingRoomNetwork},
-    EventId, ServerName,
+    EventId, RoomId, ServerName, ServerSigningKeyId, UserId,
 };
 use std::{
     collections::BTreeMap,
     convert::TryFrom,
     fmt::Debug,
+    net::{IpAddr, SocketAddr},
     time::{Duration, SystemTime},
 };
-use trust_dns_resolver::AsyncResolver;
-
-pub async fn request_well_known(
-    globals: &crate::database::globals::Globals,
-    destination: &str,
-) -> Option<String> {
-    let body: serde_json::Value = serde_json::from_str(
-        &globals
-            .reqwest_client()
-            .get(&format!(
-                "https://{}/.well-known/matrix/server",
-                destination
-            ))
-            .send()
-            .await
-            .ok()?
-            .text()
-            .await
-            .ok()?,
-    )
-    .ok()?;
-    Some(body.get("m.server")?.as_str()?.to_owned())
-}
 
 pub async fn send_request<T: OutgoingRequest>(
     globals: &crate::database::globals::Globals,
@@ -57,45 +36,33 @@ pub async fn send_request<T: OutgoingRequest>(
 where
     T: Debug,
 {
-    if !globals.federation_enabled() {
+    if !globals.allow_federation() {
         return Err(Error::bad_config("Federation is disabled."));
     }
 
-    let resolver = AsyncResolver::tokio_from_system_conf().await.map_err(|_| {
+    let maybe_result = globals
-        Error::bad_config("Failed to set up trust dns resolver with system config.")
+        .actual_destination_cache
-    })?;
+        .read()
+        .unwrap()
+        .get(&destination)
+        .cloned();
 
-    let mut host = None;
+    let (actual_destination, host) = if let Some(result) = maybe_result {
-
+        result
-    let actual_destination = "https://".to_owned()
+    } else {
-        + &if let Some(mut delegated_hostname) =
+        let result = find_actual_destination(globals, &destination).await;
-            request_well_known(globals, &destination.as_str()).await
+        globals
-        {
+            .actual_destination_cache
-            if let Ok(Some(srv)) = resolver
+            .write()
-                .srv_lookup(format!("_matrix._tcp.{}", delegated_hostname))
+            .unwrap()
-                .await
+            .insert(destination.clone(), result.clone());
-                .map(|srv| srv.iter().next().map(|result| result.target().to_string()))
+        result
-            {
+    };
-                host = Some(delegated_hostname);
-                srv.trim_end_matches('.').to_owned()
-            } else {
-                if delegated_hostname.find(':').is_none() {
-                    delegated_hostname += ":8448";
-                }
-                delegated_hostname
-            }
-        } else {
-            let mut destination = destination.as_str().to_owned();
-            if destination.find(':').is_none() {
-                destination += ":8448";
-            }
-            destination
-        };
 
     let mut http_request = request
         .try_into_http_request(&actual_destination, Some(""))
         .map_err(|e| {
-            warn!("{}: {}", actual_destination, e);
+            warn!("Failed to find destination {}: {}", actual_destination, e);
             Error::BadServerResponse("Invalid destination")
         })?;
 
@@ -122,7 +89,9 @@ where
     request_map.insert("origin".to_owned(), globals.server_name().as_str().into());
     request_map.insert("destination".to_owned(), destination.as_str().into());
 
-    let mut request_json = request_map.into();
+    let mut request_json =
+        serde_json::from_value(request_map.into()).expect("valid JSON is valid BTreeMap");
+
     ruma::signatures::sign_json(
         globals.server_name().as_str(),
         globals.keypair(),
@@ -130,6 +99,9 @@ where
     )
     .expect("our request json is what ruma expects");
 
+    let request_json: serde_json::Map<String, serde_json::Value> =
+        serde_json::from_slice(&serde_json::to_vec(&request_json).unwrap()).unwrap();
+
     let signatures = request_json["signatures"]
         .as_object()
         .unwrap()
@@ -183,25 +155,37 @@ where
                 }
             }
 
+            let status = reqwest_response.status();
+
             let body = reqwest_response
                 .bytes()
                 .await
                 .unwrap_or_else(|e| {
-                    warn!("server error: {}", e);
+                    warn!("server error {}", e);
                     Vec::new().into()
                 }) // TODO: handle timeout
                 .into_iter()
-                .collect();
+                .collect::<Vec<_>>();
+
+            if status != 200 {
+                info!(
+                    "Server returned bad response {} {}\n{}\n{:?}",
+                    destination,
+                    status,
+                    url,
+                    utils::string_from_bytes(&body)
+                );
+            }
 
             let response = T::IncomingResponse::try_from(
                 http_response
                     .body(body)
                     .expect("reqwest body is valid http body"),
             );
-            response.map_err(|e| {
+            response.map_err(|_| {
-                warn!(
+                info!(
-                    "Server returned bad response {} ({}): {:?}",
+                    "Server returned invalid response bytes {}\n{}",
-                    destination, url, e
+                    destination, url
                 );
                 Error::BadServerResponse("Server returned bad response.")
             })
@@ -210,9 +194,135 @@ where
     }
 }
 
+fn get_ip_with_port(destination_str: String) -> Option<String> {
+    if destination_str.parse::<SocketAddr>().is_ok() {
+        Some(destination_str)
+    } else if let Ok(ip_addr) = destination_str.parse::<IpAddr>() {
+        Some(SocketAddr::new(ip_addr, 8448).to_string())
+    } else {
+        None
+    }
+}
+
+fn add_port_to_hostname(destination_str: String) -> String {
+    match destination_str.find(':') {
+        None => destination_str.to_owned() + ":8448",
+        Some(_) => destination_str.to_string(),
+    }
+}
+
+/// Returns: actual_destination, host header
+/// Implemented according to the specification at https://matrix.org/docs/spec/server_server/r0.1.4#resolving-server-names
+/// Numbers in comments below refer to bullet points in linked section of specification
+async fn find_actual_destination(
+    globals: &crate::database::globals::Globals,
+    destination: &Box<ServerName>,
+) -> (String, Option<String>) {
+    let mut host = None;
+
+    let destination_str = destination.as_str().to_owned();
+    let actual_destination = "https://".to_owned()
+        + &match get_ip_with_port(destination_str.clone()) {
+            Some(host_port) => {
+                // 1: IP literal with provided or default port
+                host_port
+            }
+            None => {
+                if destination_str.find(':').is_some() {
+                    // 2: Hostname with included port
+                    destination_str
+                } else {
+                    match request_well_known(globals, &destination.as_str()).await {
+                        // 3: A .well-known file is available
+                        Some(delegated_hostname) => {
+                            match get_ip_with_port(delegated_hostname.clone()) {
+                                Some(host_and_port) => host_and_port, // 3.1: IP literal in .well-known file
+                                None => {
+                                    if destination_str.find(':').is_some() {
+                                        // 3.2: Hostname with port in .well-known file
+                                        destination_str
+                                    } else {
+                                        match query_srv_record(globals, &delegated_hostname).await {
+                                            // 3.3: SRV lookup successful
+                                            Some(hostname) => hostname,
+                                            // 3.4: No SRV records, just use the hostname from .well-known
+                                            None => add_port_to_hostname(delegated_hostname),
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        // 4: No .well-known or an error occured
+                        None => {
+                            match query_srv_record(globals, &destination_str).await {
+                                // 4: SRV record found
+                                Some(hostname) => {
+                                    host = Some(destination_str.to_owned());
+                                    hostname
+                                }
+                                // 5: No SRV record found
+                                None => add_port_to_hostname(destination_str.to_string()),
+                            }
+                        }
+                    }
+                }
+            }
+        };
+
+    (actual_destination, host)
+}
+
+async fn query_srv_record<'a>(
+    globals: &crate::database::globals::Globals,
+    hostname: &'a str,
+) -> Option<String> {
+    if let Ok(Some(host_port)) = globals
+        .dns_resolver()
+        .srv_lookup(format!("_matrix._tcp.{}", hostname))
+        .await
+        .map(|srv| {
+            srv.iter().next().map(|result| {
+                format!(
+                    "{}:{}",
+                    result.target().to_string().trim_end_matches('.'),
+                    result.port().to_string()
+                )
+            })
+        })
+    {
+        Some(host_port)
+    } else {
+        None
+    }
+}
+
+pub async fn request_well_known(
+    globals: &crate::database::globals::Globals,
+    destination: &str,
+) -> Option<String> {
+    let body: serde_json::Value = serde_json::from_str(
+        &globals
+            .reqwest_client()
+            .get(&format!(
+                "https://{}/.well-known/matrix/server",
+                destination
+            ))
+            .send()
+            .await
+            .ok()?
+            .text()
+            .await
+            .ok()?,
+    )
+    .ok()?;
+    Some(body.get("m.server")?.as_str()?.to_owned())
+}
+
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/federation/v1/version"))]
-pub fn get_server_version(db: State<'_, Database>) -> ConduitResult<get_server_version::Response> {
+pub fn get_server_version_route(
-    if !db.globals.federation_enabled() {
+    db: State<'_, Database>,
+) -> ConduitResult<get_server_version::Response> {
+    if !db.globals.allow_federation() {
         return Err(Error::bad_config("Federation is disabled."));
     }
 
@@ -226,22 +336,25 @@ pub fn get_server_version(db: State<'_, Database>) -> ConduitResult<get_server_v
 }
 
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/key/v2/server"))]
-pub fn get_server_keys(db: State<'_, Database>) -> Json<String> {
+pub fn get_server_keys_route(db: State<'_, Database>) -> Json<String> {
-    if !db.globals.federation_enabled() {
+    if !db.globals.allow_federation() {
         // TODO: Use proper types
         return Json("Federation is disabled.".to_owned());
     }
 
     let mut verify_keys = BTreeMap::new();
     verify_keys.insert(
-        format!("ed25519:{}", db.globals.keypair().version()),
+        ServerSigningKeyId::try_from(
+            format!("ed25519:{}", db.globals.keypair().version()).as_str(),
+        )
+        .expect("found invalid server signing keys in DB"),
         VerifyKey {
             key: base64::encode_config(db.globals.keypair().public_key(), base64::STANDARD_NO_PAD),
         },
     );
     let mut response = serde_json::from_slice(
         http::Response::try_from(get_server_keys::v2::Response {
-            server_key: ServerKey {
+            server_key: ServerSigningKeys {
                 server_name: db.globals.server_name().to_owned(),
                 verify_keys,
                 old_verify_keys: BTreeMap::new(),
@@ -253,18 +366,20 @@ pub fn get_server_keys(db: State<'_, Database>) -> Json<String> {
         .body(),
     )
     .unwrap();
+
     ruma::signatures::sign_json(
         db.globals.server_name().as_str(),
         db.globals.keypair(),
         &mut response,
     )
     .unwrap();
-    Json(response.to_string())
+
+    Json(ruma::serde::to_canonical_json_string(&response).expect("JSON is canonical"))
 }
 
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/key/v2/server/<_>"))]
-pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json<String> {
+pub fn get_server_keys_deprecated_route(db: State<'_, Database>) -> Json<String> {
-    get_server_keys(db)
+    get_server_keys_route(db)
 }
 
 #[cfg_attr(
@@ -275,7 +390,7 @@ pub async fn get_public_rooms_filtered_route(
     db: State<'_, Database>,
     body: Ruma<get_public_rooms_filtered::v1::Request<'_>>,
 ) -> ConduitResult<get_public_rooms_filtered::v1::Response> {
-    if !db.globals.federation_enabled() {
+    if !db.globals.allow_federation() {
         return Err(Error::bad_config("Federation is disabled."));
     }
 
@@ -322,7 +437,7 @@ pub async fn get_public_rooms_route(
     db: State<'_, Database>,
     body: Ruma<get_public_rooms::v1::Request<'_>>,
 ) -> ConduitResult<get_public_rooms::v1::Response> {
-    if !db.globals.federation_enabled() {
+    if !db.globals.allow_federation() {
         return Err(Error::bad_config("Federation is disabled."));
     }
 
@@ -365,53 +480,105 @@ pub async fn get_public_rooms_route(
     feature = "conduit_bin",
     put("/_matrix/federation/v1/send/<_>", data = "<body>")
 )]
-pub fn send_transaction_message_route<'a>(
+pub async fn send_transaction_message_route<'a>(
     db: State<'a, Database>,
     body: Ruma<send_transaction_message::v1::Request<'_>>,
 ) -> ConduitResult<send_transaction_message::v1::Response> {
-    if !db.globals.federation_enabled() {
+    if !db.globals.allow_federation() {
         return Err(Error::bad_config("Federation is disabled."));
     }
 
-    //dbg!(&*body);
+    for edu in &body.edus {
-    for pdu in &body.pdus {
+        match serde_json::from_str::<send_transaction_message::v1::Edu>(edu.json().get()) {
-        let mut value = serde_json::from_str(pdu.json().get())
+            Ok(edu) => match edu.edu_type.as_str() {
-            .expect("converting raw jsons to values always works");
+                "m.typing" => {
-
+                    if let Some(typing) = edu.content.get("typing") {
-        let event_id = EventId::try_from(&*format!(
+                        if typing.as_bool().unwrap_or_default() {
-            "${}",
+                            db.rooms.edus.typing_add(
-            ruma::signatures::reference_hash(&value).expect("ruma can calculate reference hashes")
+                                &UserId::try_from(edu.content["user_id"].as_str().unwrap())
-        ))
+                                    .unwrap(),
-        .expect("ruma's reference hashes are valid event ids");
+                                &RoomId::try_from(edu.content["room_id"].as_str().unwrap())
-
+                                    .unwrap(),
-        value
+                                3000 + utils::millis_since_unix_epoch(),
-            .as_object_mut()
+                                &db.globals,
-            .expect("ruma pdus are json objects")
+                            )?;
-            .insert("event_id".to_owned(), event_id.to_string().into());
+                        } else {
-
+                            db.rooms.edus.typing_remove(
-        let pdu = serde_json::from_value::<PduEvent>(value.clone())
+                                &UserId::try_from(edu.content["user_id"].as_str().unwrap())
-            .expect("all ruma pdus are conduit pdus");
+                                    .unwrap(),
-        if db.rooms.exists(&pdu.room_id)? {
+                                &RoomId::try_from(edu.content["room_id"].as_str().unwrap())
-            let count = db.globals.next_count()?;
+                                    .unwrap(),
-            let mut pdu_id = pdu.room_id.as_bytes().to_vec();
+                                &db.globals,
-            pdu_id.push(0xff);
+                            )?;
-            pdu_id.extend_from_slice(&count.to_be_bytes());
+                        }
-            db.rooms.append_to_state(&pdu_id, &pdu)?;
+                    }
-            db.rooms.append_pdu(
+                }
-                &pdu,
+                "m.presence" => {}
-                &value,
+                "m.receipt" => {}
-                count,
+                _ => {}
-                pdu_id.clone().into(),
+            },
-                &db.globals,
+            Err(_err) => {
-                &db.account_data,
+                continue;
-                &db.admin,
+            }
-            )?;
         }
     }
-    Ok(send_transaction_message::v1::Response {
+
-        pdus: BTreeMap::new(),
+    // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we anywhere?
+    // SPEC:
+    // Servers MUST strictly enforce the JSON format specified in the appendices.
+    // This translates to a 400 M_BAD_JSON error on most endpoints, or discarding of
+    // events over federation. For example, the Federation API's /send endpoint would
+    // discard the event whereas the Client Server API's /send/{eventType} endpoint
+    // would return a M_BAD_JSON error.
+    let mut resolved_map = BTreeMap::new();
+    for pdu in &body.pdus {
+        // Ruma/PduEvent/StateEvent satisfies - 1. Is a valid event, otherwise it is dropped.
+
+        // state-res checks signatures - 2. Passes signature checks, otherwise event is dropped.
+
+        // 3. Passes hash checks, otherwise it is redacted before being processed further.
+        // TODO: redact event if hashing fails
+        let (event_id, value) = crate::pdu::process_incoming_pdu(pdu);
+
+        let pdu = serde_json::from_value::<PduEvent>(
+            serde_json::to_value(&value).expect("CanonicalJsonObj is a valid JsonValue"),
+        )
+        .expect("all ruma pdus are conduit pdus");
+        let room_id = &pdu.room_id;
+
+        // If we have no idea about this room skip the PDU
+        if !db.rooms.exists(room_id)? {
+            resolved_map.insert(event_id, Err("Room is unknown to this server".into()));
+            continue;
+        }
+
+        let count = db.globals.next_count()?;
+        let mut pdu_id = room_id.as_bytes().to_vec();
+        pdu_id.push(0xff);
+        pdu_id.extend_from_slice(&count.to_be_bytes());
+
+        let next_room_state = db.rooms.append_to_state(&pdu_id, &pdu, &db.globals)?;
+
+        db.rooms.append_pdu(
+            &pdu,
+            value,
+            count,
+            pdu_id.clone().into(),
+            &db.globals,
+            &db.account_data,
+            &db.admin,
+        )?;
+
+        db.rooms.set_room_state(&room_id, &next_room_state)?;
+
+        for appservice in db.appservice.iter_all().filter_map(|r| r.ok()) {
+            db.sending.send_pdu_appservice(&appservice.0, &pdu_id)?;
+        }
+
+        resolved_map.insert(event_id, Ok::<(), String>(()));
     }
-    .into())
+
+    Ok(send_transaction_message::v1::Response { pdus: resolved_map }.into())
 }
 
 #[cfg_attr(
@@ -422,7 +589,7 @@ pub fn get_missing_events_route<'a>(
     db: State<'a, Database>,
     body: Ruma<get_missing_events::v1::Request<'_>>,
 ) -> ConduitResult<get_missing_events::v1::Response> {
-    if !db.globals.federation_enabled() {
+    if !db.globals.allow_federation() {
         return Err(Error::bad_config("Federation is disabled."));
     }
 
@@ -451,7 +618,7 @@ pub fn get_missing_events_route<'a>(
                 )
                 .map_err(|_| Error::bad_database("Invalid prev_events content in pdu in db."))?,
             );
-            events.push(PduEvent::convert_to_outgoing_federation_event(pdu));
+            events.push(serde_json::from_value(pdu).expect("Raw<..> is always valid"));
         }
         i += 1;
     }
@@ -467,14 +634,16 @@ pub fn get_profile_information_route<'a>(
     db: State<'a, Database>,
     body: Ruma<get_profile_information::v1::Request<'_>>,
 ) -> ConduitResult<get_profile_information::v1::Response> {
-    if !db.globals.federation_enabled() {
+    if !db.globals.allow_federation() {
         return Err(Error::bad_config("Federation is disabled."));
     }
 
     let mut displayname = None;
     let mut avatar_url = None;
 
-    match body.field {
+    match &body.field {
+        // TODO: what to do with custom
+        Some(ProfileField::_Custom(_s)) => {}
         Some(ProfileField::DisplayName) => displayname = db.users.displayname(&body.user_id)?,
         Some(ProfileField::AvatarUrl) => avatar_url = db.users.avatar_url(&body.user_id)?,
         None => {
@@ -499,7 +668,7 @@ pub fn get_user_devices_route<'a>(
     db: State<'a, Database>,
     body: Ruma<membership::v1::Request<'_>>,
 ) -> ConduitResult<get_profile_information::v1::Response> {
-    if !db.globals.federation_enabled() {
+    if !db.globals.allow_federation() {
         return Err(Error::bad_config("Federation is disabled."));
     }
 
@@ -522,3 +691,48 @@ pub fn get_user_devices_route<'a>(
     .into())
 }
 */
+
+#[cfg(test)]
+mod tests {
+    use super::{add_port_to_hostname, get_ip_with_port};
+
+    #[test]
+    fn ips_get_default_ports() {
+        assert_eq!(
+            get_ip_with_port(String::from("1.1.1.1")),
+            Some(String::from("1.1.1.1:8448"))
+        );
+        assert_eq!(
+            get_ip_with_port(String::from("dead:beef::")),
+            Some(String::from("[dead:beef::]:8448"))
+        );
+    }
+
+    #[test]
+    fn ips_keep_custom_ports() {
+        assert_eq!(
+            get_ip_with_port(String::from("1.1.1.1:1234")),
+            Some(String::from("1.1.1.1:1234"))
+        );
+        assert_eq!(
+            get_ip_with_port(String::from("[dead::beef]:8933")),
+            Some(String::from("[dead::beef]:8933"))
+        );
+    }
+
+    #[test]
+    fn hostnames_get_default_ports() {
+        assert_eq!(
+            add_port_to_hostname(String::from("example.com")),
+            "example.com:8448"
+        )
+    }
+
+    #[test]
+    fn hostnames_keep_custom_ports() {
+        assert_eq!(
+            add_port_to_hostname(String::from("example.com:1337")),
+            "example.com:1337"
+        )
+    }
+}

src/utils.rs

@@ -1,6 +1,7 @@
 use argon2::{Config, Variant};
 use cmp::Ordering;
 use rand::prelude::*;
+use ruma::serde::{try_from_json_map, CanonicalJsonError, CanonicalJsonObject};
 use sled::IVec;
 use std::{
     cmp,
@@ -89,9 +90,24 @@ pub fn common_elements(
                         }
                     }
                 }
-
                 false
             })
             .all(|b| b)
     }))
 }
+
+/// Fallible conversion from any value that implements `Serialize` to a `CanonicalJsonObject`.
+///
+/// `value` must serialize to an `serde_json::Value::Object`.
+pub fn to_canonical_object<T: serde::Serialize>(
+    value: T,
+) -> Result<CanonicalJsonObject, CanonicalJsonError> {
+    use serde::ser::Error;
+
+    match serde_json::to_value(value).map_err(CanonicalJsonError::SerDe)? {
+        serde_json::Value::Object(map) => try_from_json_map(map),
+        _ => Err(CanonicalJsonError::SerDe(serde_json::Error::custom(
+            "Value must be an object",
+        ))),
+    }
+}