From 972caacdc2de95183ddddd4084282a069a75b89a Mon Sep 17 00:00:00 2001 From: hamidreza kalbasi Date: Fri, 4 Jun 2021 08:06:12 +0430 Subject: [PATCH] put media in filesystem --- src/client_server/media.rs | 14 +++++---- src/database.rs | 4 +-- src/database/abstraction.rs | 6 ++-- src/database/globals.rs | 21 ++++++++++---- src/database/media.rs | 58 +++++++++++++++++++++++++++---------- src/error.rs | 5 ++++ 6 files changed, 78 insertions(+), 30 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 0673787..14ab6db 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -38,6 +38,7 @@ pub async fn create_content_route( ); db.media.create( mxc.clone(), + &db.globals, &body .filename .as_ref() @@ -45,7 +46,7 @@ pub async fn create_content_route( .as_deref(), &body.content_type.as_deref(), &body.file, - )?; + ).await?; db.flush().await?; @@ -71,7 +72,7 @@ pub async fn get_content_route( content_disposition, content_type, file, - }) = db.media.get(&mxc)? + }) = db.media.get(&db.globals, &mxc).await? { Ok(get_content::Response { file, @@ -95,10 +96,11 @@ pub async fn get_content_route( db.media.create( mxc, + &db.globals, &get_content_response.content_disposition.as_deref(), &get_content_response.content_type.as_deref(), &get_content_response.file, - )?; + ).await?; Ok(get_content_response.into()) } else { @@ -121,13 +123,14 @@ pub async fn get_content_thumbnail_route( content_type, file, .. }) = db.media.get_thumbnail( mxc.clone(), + &db.globals, body.width .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, body.height .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, - )? { + ).await? { Ok(get_content_thumbnail::Response { file, content_type }.into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_thumbnail_response = db @@ -148,12 +151,13 @@ pub async fn get_content_thumbnail_route( db.media.upload_thumbnail( mxc, + &db.globals, &None, &get_thumbnail_response.content_type, body.width.try_into().expect("all UInts are valid u32s"), body.height.try_into().expect("all UInts are valid u32s"), &get_thumbnail_response.file, - )?; + ).await?; Ok(get_thumbnail_response.into()) } else { diff --git a/src/database.rs b/src/database.rs index e3b954e..04d3735 100644 --- a/src/database.rs +++ b/src/database.rs @@ -245,7 +245,7 @@ impl Database { db.globals.bump_database_version(1)?; - info!("Migration: 0 -> 1 finished"); + println!("Migration: 0 -> 1 finished"); } if db.globals.database_version()? < 2 { @@ -262,7 +262,7 @@ impl Database { db.globals.bump_database_version(2)?; - info!("Migration: 1 -> 2 finished"); + println!("Migration: 1 -> 2 finished"); } // This data is probably outdated diff --git a/src/database/abstraction.rs b/src/database/abstraction.rs index 5a2afd5..ad032fb 100644 --- a/src/database/abstraction.rs +++ b/src/database/abstraction.rs @@ -47,7 +47,7 @@ pub trait Tree: Send + Sync { fn scan_prefix<'a>( &'a self, prefix: Vec, - ) -> Box, Box<[u8]>)> + 'a>; + ) -> Box, Box<[u8]>)> + Send + 'a>; fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>>; @@ -142,7 +142,7 @@ impl Tree for SledEngineTree { fn scan_prefix<'a>( &'a self, prefix: Vec, - ) -> Box, Box<[u8]>)> + 'a> { + ) -> Box, Box<[u8]>)> + Send + 'a> { let iter = self .0 .scan_prefix(prefix) @@ -279,7 +279,7 @@ impl Tree for RocksDbEngineTree<'_> { fn scan_prefix<'a>( &'a self, prefix: Vec, - ) -> Box, Box<[u8]>)> + 'a> { + ) -> Box, Box<[u8]>)> + Send + 'a> { Box::new( self.db .0 diff --git a/src/database/globals.rs b/src/database/globals.rs index 37ebf13..1ca64de 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -5,11 +5,7 @@ use ruma::{ EventId, MilliSecondsSinceUnixEpoch, ServerName, ServerSigningKeyId, }; use rustls::{ServerCertVerifier, WebPKIVerifier}; -use std::{ - collections::{BTreeMap, HashMap}, - sync::{Arc, RwLock}, - time::{Duration, Instant}, -}; +use std::{collections::{BTreeMap, HashMap}, path::{PathBuf}, sync::{Arc, RwLock}, time::{Duration, Instant}}; use tokio::sync::Semaphore; use trust_dns_resolver::TokioAsyncResolver; @@ -275,4 +271,19 @@ impl Globals { .insert(b"version", &new_version.to_be_bytes())?; Ok(()) } + + pub fn get_media_folder(&self) -> PathBuf { + let mut r = PathBuf::new(); + r.push(self.config.database_path.clone()); + r.push("media"); + r + } + + pub fn get_media_file(&self, key: &Vec) -> PathBuf { + let mut r = PathBuf::new(); + r.push(self.config.database_path.clone()); + r.push("media"); + r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD)); + r + } } diff --git a/src/database/media.rs b/src/database/media.rs index ca45484..666a494 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -1,9 +1,10 @@ use image::{imageops::FilterType, GenericImageView}; +use crate::database::globals::Globals; use crate::{utils, Error, Result}; use std::{mem, sync::Arc}; - use super::abstraction::Tree; +use tokio::{fs::{self, File}, io::AsyncWriteExt, io::AsyncReadExt}; pub struct FileMeta { pub content_disposition: Option, @@ -16,10 +17,11 @@ pub struct Media { } impl Media { - /// Uploads or replaces a file. - pub fn create( + /// Uploads a file. + pub async fn create( &self, mxc: String, + globals: &Globals, content_disposition: &Option<&str>, content_type: &Option<&str>, file: &[u8], @@ -43,15 +45,20 @@ impl Media { .unwrap_or_default(), ); - self.mediaid_file.insert(&key, file)?; + let path = globals.get_media_file(&key); + fs::create_dir_all(path.parent().unwrap()).await?; + let mut f = File::create(path).await?; + f.write_all(file).await?; + self.mediaid_file.insert(&key, &[])?; Ok(()) } /// Uploads or replaces a file thumbnail. - pub fn upload_thumbnail( + pub async fn upload_thumbnail( &self, mxc: String, + globals: &Globals, content_disposition: &Option, content_type: &Option, width: u32, @@ -77,20 +84,29 @@ impl Media { .unwrap_or_default(), ); - self.mediaid_file.insert(&key, file)?; + let path = globals.get_media_file(&key); + fs::create_dir_all(path.parent().unwrap()).await?; + let mut f = File::create(path).await?; + f.write_all(file).await?; + + self.mediaid_file.insert(&key, &[])?; Ok(()) } /// Downloads a file. - pub fn get(&self, mxc: &str) -> Result> { + pub async fn get(&self, globals: &Globals, mxc: &str) -> Result> { let mut prefix = mxc.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail prefix.push(0xff); - if let Some((key, file)) = self.mediaid_file.scan_prefix(prefix).next() { + let mut iter = self.mediaid_file.scan_prefix(prefix); + if let Some((key, _)) = iter.next() { + let path = globals.get_media_file(&key.to_vec()); + let mut file = vec![]; + File::open(path).await?.read_to_end(&mut file).await?; let mut parts = key.rsplit(|&b| b == 0xff); let content_type = parts @@ -121,7 +137,7 @@ impl Media { Ok(Some(FileMeta { content_disposition, content_type, - file: file.to_vec(), + file, })) } else { Ok(None) @@ -151,7 +167,7 @@ impl Media { /// - Server creates the thumbnail and sends it to the user /// /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards. - pub fn get_thumbnail(&self, mxc: String, width: u32, height: u32) -> Result> { + pub async fn get_thumbnail(&self, mxc: String, globals: &Globals, width: u32, height: u32) -> Result> { let (width, height, crop) = self .thumbnail_properties(width, height) .unwrap_or((0, 0, false)); // 0, 0 because that's the original file @@ -169,8 +185,11 @@ impl Media { original_prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail original_prefix.push(0xff); - if let Some((key, file)) = self.mediaid_file.scan_prefix(thumbnail_prefix).next() { + if let Some((key, _)) = self.mediaid_file.scan_prefix(thumbnail_prefix).next() { // Using saved thumbnail + let path = globals.get_media_file(&key.to_vec()); + let mut file = vec![]; + File::open(path).await?.read_to_end(&mut file).await?; let mut parts = key.rsplit(|&b| b == 0xff); let content_type = parts @@ -201,8 +220,12 @@ impl Media { content_type, file: file.to_vec(), })) - } else if let Some((key, file)) = self.mediaid_file.scan_prefix(original_prefix).next() { + } else if let Some((key, _)) = self.mediaid_file.scan_prefix(original_prefix).next() { // Generate a thumbnail + let path = globals.get_media_file(&key.to_vec()); + let mut file = vec![]; + File::open(path).await?.read_to_end(&mut file).await?; + let mut parts = key.rsplit(|&b| b == 0xff); let content_type = parts @@ -299,19 +322,24 @@ impl Media { widthheight, ); - self.mediaid_file.insert(&thumbnail_key, &thumbnail_bytes)?; + let path = globals.get_media_file(&thumbnail_key); + fs::create_dir_all(path.parent().unwrap()).await?; + let mut f = File::create(path).await?; + f.write_all(&thumbnail_bytes).await?; + + self.mediaid_file.insert(&thumbnail_key, &[])?; Ok(Some(FileMeta { content_disposition, content_type, - file: thumbnail_bytes.to_vec(), + file: thumbnail_bytes.to_vec() })) } else { // Couldn't parse file to generate thumbnail, send original Ok(Some(FileMeta { content_disposition, content_type, - file: file.to_vec(), + file: file.to_vec() })) } } else { diff --git a/src/error.rs b/src/error.rs index 93c67c1..10a48b7 100644 --- a/src/error.rs +++ b/src/error.rs @@ -45,6 +45,11 @@ pub enum Error { }, #[error("{0}")] FederationError(Box, RumaError), + #[error("Could not do this io: {source}")] + IoError { + #[from] + source: std::io::Error, + }, #[error("{0}")] BadServerResponse(&'static str), #[error("{0}")]