put media in filesystem

src/client_server/media.rs

@@ -38,6 +38,7 @@ pub async fn create_content_route(
     );
     db.media.create(
         mxc.clone(),
+        &db.globals,
         &body
             .filename
             .as_ref()
@@ -45,7 +46,7 @@ pub async fn create_content_route(
             .as_deref(),
         &body.content_type.as_deref(),
         &body.file,
-    )?;
+    ).await?;
 
     db.flush().await?;
 
@@ -71,7 +72,7 @@ pub async fn get_content_route(
         content_disposition,
         content_type,
         file,
-    }) = db.media.get(&mxc)?
+    }) = db.media.get(&db.globals, &mxc).await?
     {
         Ok(get_content::Response {
             file,
@@ -95,10 +96,11 @@ pub async fn get_content_route(
 
         db.media.create(
             mxc,
+            &db.globals,
             &get_content_response.content_disposition.as_deref(),
             &get_content_response.content_type.as_deref(),
             &get_content_response.file,
-        )?;
+        ).await?;
 
         Ok(get_content_response.into())
     } else {
@@ -121,13 +123,14 @@ pub async fn get_content_thumbnail_route(
         content_type, file, ..
     }) = db.media.get_thumbnail(
         mxc.clone(),
+        &db.globals,
         body.width
             .try_into()
             .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
         body.height
             .try_into()
             .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
-    )? {
+    ).await? {
         Ok(get_content_thumbnail::Response { file, content_type }.into())
     } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
         let get_thumbnail_response = db
@@ -148,12 +151,13 @@ pub async fn get_content_thumbnail_route(
 
         db.media.upload_thumbnail(
             mxc,
+            &db.globals,
             &None,
             &get_thumbnail_response.content_type,
             body.width.try_into().expect("all UInts are valid u32s"),
             body.height.try_into().expect("all UInts are valid u32s"),
             &get_thumbnail_response.file,
-        )?;
+        ).await?;
 
         Ok(get_thumbnail_response.into())
     } else {

src/database.rs

@@ -245,7 +245,7 @@ impl Database {
 
             db.globals.bump_database_version(1)?;
 
-            info!("Migration: 0 -> 1 finished");
+            println!("Migration: 0 -> 1 finished");
         }
 
         if db.globals.database_version()? < 2 {
@@ -262,7 +262,7 @@ impl Database {
 
             db.globals.bump_database_version(2)?;
 
-            info!("Migration: 1 -> 2 finished");
+            println!("Migration: 1 -> 2 finished");
         }
 
         // This data is probably outdated

src/database/abstraction.rs

@@ -47,7 +47,7 @@ pub trait Tree: Send + Sync {
     fn scan_prefix<'a>(
         &'a self,
         prefix: Vec<u8>,
-    ) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a>;
+    ) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + Send + 'a>;
 
     fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
 
@@ -142,7 +142,7 @@ impl Tree for SledEngineTree {
     fn scan_prefix<'a>(
         &'a self,
         prefix: Vec<u8>,
-    ) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a> {
+    ) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + Send + 'a> {
         let iter = self
             .0
             .scan_prefix(prefix)
@@ -279,7 +279,7 @@ impl Tree for RocksDbEngineTree<'_> {
     fn scan_prefix<'a>(
         &'a self,
         prefix: Vec<u8>,
-    ) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a> {
+    ) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + Send + 'a> {
         Box::new(
             self.db
                 .0

src/database/globals.rs

@@ -5,11 +5,7 @@ use ruma::{
     EventId, MilliSecondsSinceUnixEpoch, ServerName, ServerSigningKeyId,
 };
 use rustls::{ServerCertVerifier, WebPKIVerifier};
-use std::{
+use std::{collections::{BTreeMap, HashMap}, path::{PathBuf}, sync::{Arc, RwLock}, time::{Duration, Instant}};
-    collections::{BTreeMap, HashMap},
-    sync::{Arc, RwLock},
-    time::{Duration, Instant},
-};
 use tokio::sync::Semaphore;
 use trust_dns_resolver::TokioAsyncResolver;
 
@@ -275,4 +271,19 @@ impl Globals {
             .insert(b"version", &new_version.to_be_bytes())?;
         Ok(())
     }
+
+    pub fn get_media_folder(&self) -> PathBuf {
+        let mut r = PathBuf::new();
+        r.push(self.config.database_path.clone());
+        r.push("media");
+        r
+    }
+
+    pub fn get_media_file(&self, key: &Vec<u8>) -> PathBuf {
+        let mut r = PathBuf::new();
+        r.push(self.config.database_path.clone());
+        r.push("media");
+        r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD));
+        r
+    }
 }

src/database/media.rs

@@ -1,9 +1,10 @@
 use image::{imageops::FilterType, GenericImageView};
+use crate::database::globals::Globals;
 
 use crate::{utils, Error, Result};
 use std::{mem, sync::Arc};
-
 use super::abstraction::Tree;
+use tokio::{fs::{self, File}, io::AsyncWriteExt, io::AsyncReadExt};
 
 pub struct FileMeta {
     pub content_disposition: Option<String>,
@@ -16,10 +17,11 @@ pub struct Media {
 }
 
 impl Media {
-    /// Uploads or replaces a file.
+    /// Uploads a file.
-    pub fn create(
+    pub async fn create(
         &self,
         mxc: String,
+        globals: &Globals,
         content_disposition: &Option<&str>,
         content_type: &Option<&str>,
         file: &[u8],
@@ -43,15 +45,20 @@ impl Media {
                 .unwrap_or_default(),
         );
 
-        self.mediaid_file.insert(&key, file)?;
+        let path = globals.get_media_file(&key);
+        fs::create_dir_all(path.parent().unwrap()).await?;
+        let mut f = File::create(path).await?;
+        f.write_all(file).await?;
 
+        self.mediaid_file.insert(&key, &[])?;
         Ok(())
     }
 
     /// Uploads or replaces a file thumbnail.
-    pub fn upload_thumbnail(
+    pub async fn upload_thumbnail(
         &self,
         mxc: String,
+        globals: &Globals,
         content_disposition: &Option<String>,
         content_type: &Option<String>,
         width: u32,
@@ -77,20 +84,29 @@ impl Media {
                 .unwrap_or_default(),
         );
 
-        self.mediaid_file.insert(&key, file)?;
+        let path = globals.get_media_file(&key);
+        fs::create_dir_all(path.parent().unwrap()).await?;
+        let mut f = File::create(path).await?;
+        f.write_all(file).await?;
+
+        self.mediaid_file.insert(&key, &[])?;
 
         Ok(())
     }
 
     /// Downloads a file.
-    pub fn get(&self, mxc: &str) -> Result<Option<FileMeta>> {
+    pub async fn get(&self, globals: &Globals, mxc: &str) -> Result<Option<FileMeta>> {
         let mut prefix = mxc.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail
         prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
         prefix.push(0xff);
 
-        if let Some((key, file)) = self.mediaid_file.scan_prefix(prefix).next() {
+        let mut iter = self.mediaid_file.scan_prefix(prefix);
+        if let Some((key, _)) = iter.next() {
+            let path = globals.get_media_file(&key.to_vec());
+            let mut file = vec![];
+            File::open(path).await?.read_to_end(&mut file).await?;
             let mut parts = key.rsplit(|&b| b == 0xff);
 
             let content_type = parts
@@ -121,7 +137,7 @@ impl Media {
             Ok(Some(FileMeta {
                 content_disposition,
                 content_type,
-                file: file.to_vec(),
+                file,
             }))
         } else {
             Ok(None)
@@ -151,7 +167,7 @@ impl Media {
     /// - Server creates the thumbnail and sends it to the user
     ///
     /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
-    pub fn get_thumbnail(&self, mxc: String, width: u32, height: u32) -> Result<Option<FileMeta>> {
+    pub async fn get_thumbnail(&self, mxc: String, globals: &Globals, width: u32, height: u32) -> Result<Option<FileMeta>> {
         let (width, height, crop) = self
             .thumbnail_properties(width, height)
             .unwrap_or((0, 0, false)); // 0, 0 because that's the original file
@@ -169,8 +185,11 @@ impl Media {
         original_prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
         original_prefix.push(0xff);
 
-        if let Some((key, file)) = self.mediaid_file.scan_prefix(thumbnail_prefix).next() {
+        if let Some((key, _)) = self.mediaid_file.scan_prefix(thumbnail_prefix).next() {
             // Using saved thumbnail
+            let path = globals.get_media_file(&key.to_vec());
+            let mut file = vec![];
+            File::open(path).await?.read_to_end(&mut file).await?;
             let mut parts = key.rsplit(|&b| b == 0xff);
 
             let content_type = parts
@@ -201,8 +220,12 @@ impl Media {
                 content_type,
                 file: file.to_vec(),
             }))
-        } else if let Some((key, file)) = self.mediaid_file.scan_prefix(original_prefix).next() {
+        } else if let Some((key, _)) = self.mediaid_file.scan_prefix(original_prefix).next() {
             // Generate a thumbnail
+            let path = globals.get_media_file(&key.to_vec());
+            let mut file = vec![];
+            File::open(path).await?.read_to_end(&mut file).await?;
+            
             let mut parts = key.rsplit(|&b| b == 0xff);
 
             let content_type = parts
@@ -299,19 +322,24 @@ impl Media {
                     widthheight,
                 );
 
-                self.mediaid_file.insert(&thumbnail_key, &thumbnail_bytes)?;
+                let path = globals.get_media_file(&thumbnail_key);
+                fs::create_dir_all(path.parent().unwrap()).await?;
+                let mut f = File::create(path).await?;
+                f.write_all(&thumbnail_bytes).await?;
+        
+                self.mediaid_file.insert(&thumbnail_key, &[])?;
 
                 Ok(Some(FileMeta {
                     content_disposition,
                     content_type,
-                    file: thumbnail_bytes.to_vec(),
+                    file: thumbnail_bytes.to_vec()
                 }))
             } else {
                 // Couldn't parse file to generate thumbnail, send original
                 Ok(Some(FileMeta {
                     content_disposition,
                     content_type,
-                    file: file.to_vec(),
+                    file: file.to_vec()
                 }))
             }
         } else {

src/error.rs

@@ -45,6 +45,11 @@ pub enum Error {
     },
     #[error("{0}")]
     FederationError(Box<ServerName>, RumaError),
+    #[error("Could not do this io: {source}")]
+    IoError {
+        #[from]
+        source: std::io::Error,
+    },
     #[error("{0}")]
     BadServerResponse(&'static str),
     #[error("{0}")]