put media in filesystem

next
hamidreza kalbasi 2021-06-04 08:06:12 +04:30 committed by Timo Kösters
parent d0ee823254
commit 972caacdc2
No known key found for this signature in database
GPG Key ID: 24DA7517711A2BA4
6 changed files with 78 additions and 30 deletions

View File

@ -38,6 +38,7 @@ pub async fn create_content_route(
); );
db.media.create( db.media.create(
mxc.clone(), mxc.clone(),
&db.globals,
&body &body
.filename .filename
.as_ref() .as_ref()
@ -45,7 +46,7 @@ pub async fn create_content_route(
.as_deref(), .as_deref(),
&body.content_type.as_deref(), &body.content_type.as_deref(),
&body.file, &body.file,
)?; ).await?;
db.flush().await?; db.flush().await?;
@ -71,7 +72,7 @@ pub async fn get_content_route(
content_disposition, content_disposition,
content_type, content_type,
file, file,
}) = db.media.get(&mxc)? }) = db.media.get(&db.globals, &mxc).await?
{ {
Ok(get_content::Response { Ok(get_content::Response {
file, file,
@ -95,10 +96,11 @@ pub async fn get_content_route(
db.media.create( db.media.create(
mxc, mxc,
&db.globals,
&get_content_response.content_disposition.as_deref(), &get_content_response.content_disposition.as_deref(),
&get_content_response.content_type.as_deref(), &get_content_response.content_type.as_deref(),
&get_content_response.file, &get_content_response.file,
)?; ).await?;
Ok(get_content_response.into()) Ok(get_content_response.into())
} else { } else {
@ -121,13 +123,14 @@ pub async fn get_content_thumbnail_route(
content_type, file, .. content_type, file, ..
}) = db.media.get_thumbnail( }) = db.media.get_thumbnail(
mxc.clone(), mxc.clone(),
&db.globals,
body.width body.width
.try_into() .try_into()
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
body.height body.height
.try_into() .try_into()
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
)? { ).await? {
Ok(get_content_thumbnail::Response { file, content_type }.into()) Ok(get_content_thumbnail::Response { file, content_type }.into())
} else if &*body.server_name != db.globals.server_name() && body.allow_remote { } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
let get_thumbnail_response = db let get_thumbnail_response = db
@ -148,12 +151,13 @@ pub async fn get_content_thumbnail_route(
db.media.upload_thumbnail( db.media.upload_thumbnail(
mxc, mxc,
&db.globals,
&None, &None,
&get_thumbnail_response.content_type, &get_thumbnail_response.content_type,
body.width.try_into().expect("all UInts are valid u32s"), body.width.try_into().expect("all UInts are valid u32s"),
body.height.try_into().expect("all UInts are valid u32s"), body.height.try_into().expect("all UInts are valid u32s"),
&get_thumbnail_response.file, &get_thumbnail_response.file,
)?; ).await?;
Ok(get_thumbnail_response.into()) Ok(get_thumbnail_response.into())
} else { } else {

View File

@ -245,7 +245,7 @@ impl Database {
db.globals.bump_database_version(1)?; db.globals.bump_database_version(1)?;
info!("Migration: 0 -> 1 finished"); println!("Migration: 0 -> 1 finished");
} }
if db.globals.database_version()? < 2 { if db.globals.database_version()? < 2 {
@ -262,7 +262,7 @@ impl Database {
db.globals.bump_database_version(2)?; db.globals.bump_database_version(2)?;
info!("Migration: 1 -> 2 finished"); println!("Migration: 1 -> 2 finished");
} }
// This data is probably outdated // This data is probably outdated

View File

@ -47,7 +47,7 @@ pub trait Tree: Send + Sync {
fn scan_prefix<'a>( fn scan_prefix<'a>(
&'a self, &'a self,
prefix: Vec<u8>, prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a>; ) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + Send + 'a>;
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>; fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
@ -142,7 +142,7 @@ impl Tree for SledEngineTree {
fn scan_prefix<'a>( fn scan_prefix<'a>(
&'a self, &'a self,
prefix: Vec<u8>, prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a> { ) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + Send + 'a> {
let iter = self let iter = self
.0 .0
.scan_prefix(prefix) .scan_prefix(prefix)
@ -279,7 +279,7 @@ impl Tree for RocksDbEngineTree<'_> {
fn scan_prefix<'a>( fn scan_prefix<'a>(
&'a self, &'a self,
prefix: Vec<u8>, prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a> { ) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + Send + 'a> {
Box::new( Box::new(
self.db self.db
.0 .0

View File

@ -5,11 +5,7 @@ use ruma::{
EventId, MilliSecondsSinceUnixEpoch, ServerName, ServerSigningKeyId, EventId, MilliSecondsSinceUnixEpoch, ServerName, ServerSigningKeyId,
}; };
use rustls::{ServerCertVerifier, WebPKIVerifier}; use rustls::{ServerCertVerifier, WebPKIVerifier};
use std::{ use std::{collections::{BTreeMap, HashMap}, path::{PathBuf}, sync::{Arc, RwLock}, time::{Duration, Instant}};
collections::{BTreeMap, HashMap},
sync::{Arc, RwLock},
time::{Duration, Instant},
};
use tokio::sync::Semaphore; use tokio::sync::Semaphore;
use trust_dns_resolver::TokioAsyncResolver; use trust_dns_resolver::TokioAsyncResolver;
@ -275,4 +271,19 @@ impl Globals {
.insert(b"version", &new_version.to_be_bytes())?; .insert(b"version", &new_version.to_be_bytes())?;
Ok(()) Ok(())
} }
pub fn get_media_folder(&self) -> PathBuf {
let mut r = PathBuf::new();
r.push(self.config.database_path.clone());
r.push("media");
r
}
pub fn get_media_file(&self, key: &Vec<u8>) -> PathBuf {
let mut r = PathBuf::new();
r.push(self.config.database_path.clone());
r.push("media");
r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD));
r
}
} }

View File

@ -1,9 +1,10 @@
use image::{imageops::FilterType, GenericImageView}; use image::{imageops::FilterType, GenericImageView};
use crate::database::globals::Globals;
use crate::{utils, Error, Result}; use crate::{utils, Error, Result};
use std::{mem, sync::Arc}; use std::{mem, sync::Arc};
use super::abstraction::Tree; use super::abstraction::Tree;
use tokio::{fs::{self, File}, io::AsyncWriteExt, io::AsyncReadExt};
pub struct FileMeta { pub struct FileMeta {
pub content_disposition: Option<String>, pub content_disposition: Option<String>,
@ -16,10 +17,11 @@ pub struct Media {
} }
impl Media { impl Media {
/// Uploads or replaces a file. /// Uploads a file.
pub fn create( pub async fn create(
&self, &self,
mxc: String, mxc: String,
globals: &Globals,
content_disposition: &Option<&str>, content_disposition: &Option<&str>,
content_type: &Option<&str>, content_type: &Option<&str>,
file: &[u8], file: &[u8],
@ -43,15 +45,20 @@ impl Media {
.unwrap_or_default(), .unwrap_or_default(),
); );
self.mediaid_file.insert(&key, file)?; let path = globals.get_media_file(&key);
fs::create_dir_all(path.parent().unwrap()).await?;
let mut f = File::create(path).await?;
f.write_all(file).await?;
self.mediaid_file.insert(&key, &[])?;
Ok(()) Ok(())
} }
/// Uploads or replaces a file thumbnail. /// Uploads or replaces a file thumbnail.
pub fn upload_thumbnail( pub async fn upload_thumbnail(
&self, &self,
mxc: String, mxc: String,
globals: &Globals,
content_disposition: &Option<String>, content_disposition: &Option<String>,
content_type: &Option<String>, content_type: &Option<String>,
width: u32, width: u32,
@ -77,20 +84,29 @@ impl Media {
.unwrap_or_default(), .unwrap_or_default(),
); );
self.mediaid_file.insert(&key, file)?; let path = globals.get_media_file(&key);
fs::create_dir_all(path.parent().unwrap()).await?;
let mut f = File::create(path).await?;
f.write_all(file).await?;
self.mediaid_file.insert(&key, &[])?;
Ok(()) Ok(())
} }
/// Downloads a file. /// Downloads a file.
pub fn get(&self, mxc: &str) -> Result<Option<FileMeta>> { pub async fn get(&self, globals: &Globals, mxc: &str) -> Result<Option<FileMeta>> {
let mut prefix = mxc.as_bytes().to_vec(); let mut prefix = mxc.as_bytes().to_vec();
prefix.push(0xff); prefix.push(0xff);
prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail
prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
prefix.push(0xff); prefix.push(0xff);
if let Some((key, file)) = self.mediaid_file.scan_prefix(prefix).next() { let mut iter = self.mediaid_file.scan_prefix(prefix);
if let Some((key, _)) = iter.next() {
let path = globals.get_media_file(&key.to_vec());
let mut file = vec![];
File::open(path).await?.read_to_end(&mut file).await?;
let mut parts = key.rsplit(|&b| b == 0xff); let mut parts = key.rsplit(|&b| b == 0xff);
let content_type = parts let content_type = parts
@ -121,7 +137,7 @@ impl Media {
Ok(Some(FileMeta { Ok(Some(FileMeta {
content_disposition, content_disposition,
content_type, content_type,
file: file.to_vec(), file,
})) }))
} else { } else {
Ok(None) Ok(None)
@ -151,7 +167,7 @@ impl Media {
/// - Server creates the thumbnail and sends it to the user /// - Server creates the thumbnail and sends it to the user
/// ///
/// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards. /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
pub fn get_thumbnail(&self, mxc: String, width: u32, height: u32) -> Result<Option<FileMeta>> { pub async fn get_thumbnail(&self, mxc: String, globals: &Globals, width: u32, height: u32) -> Result<Option<FileMeta>> {
let (width, height, crop) = self let (width, height, crop) = self
.thumbnail_properties(width, height) .thumbnail_properties(width, height)
.unwrap_or((0, 0, false)); // 0, 0 because that's the original file .unwrap_or((0, 0, false)); // 0, 0 because that's the original file
@ -169,8 +185,11 @@ impl Media {
original_prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail original_prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
original_prefix.push(0xff); original_prefix.push(0xff);
if let Some((key, file)) = self.mediaid_file.scan_prefix(thumbnail_prefix).next() { if let Some((key, _)) = self.mediaid_file.scan_prefix(thumbnail_prefix).next() {
// Using saved thumbnail // Using saved thumbnail
let path = globals.get_media_file(&key.to_vec());
let mut file = vec![];
File::open(path).await?.read_to_end(&mut file).await?;
let mut parts = key.rsplit(|&b| b == 0xff); let mut parts = key.rsplit(|&b| b == 0xff);
let content_type = parts let content_type = parts
@ -201,8 +220,12 @@ impl Media {
content_type, content_type,
file: file.to_vec(), file: file.to_vec(),
})) }))
} else if let Some((key, file)) = self.mediaid_file.scan_prefix(original_prefix).next() { } else if let Some((key, _)) = self.mediaid_file.scan_prefix(original_prefix).next() {
// Generate a thumbnail // Generate a thumbnail
let path = globals.get_media_file(&key.to_vec());
let mut file = vec![];
File::open(path).await?.read_to_end(&mut file).await?;
let mut parts = key.rsplit(|&b| b == 0xff); let mut parts = key.rsplit(|&b| b == 0xff);
let content_type = parts let content_type = parts
@ -299,19 +322,24 @@ impl Media {
widthheight, widthheight,
); );
self.mediaid_file.insert(&thumbnail_key, &thumbnail_bytes)?; let path = globals.get_media_file(&thumbnail_key);
fs::create_dir_all(path.parent().unwrap()).await?;
let mut f = File::create(path).await?;
f.write_all(&thumbnail_bytes).await?;
self.mediaid_file.insert(&thumbnail_key, &[])?;
Ok(Some(FileMeta { Ok(Some(FileMeta {
content_disposition, content_disposition,
content_type, content_type,
file: thumbnail_bytes.to_vec(), file: thumbnail_bytes.to_vec()
})) }))
} else { } else {
// Couldn't parse file to generate thumbnail, send original // Couldn't parse file to generate thumbnail, send original
Ok(Some(FileMeta { Ok(Some(FileMeta {
content_disposition, content_disposition,
content_type, content_type,
file: file.to_vec(), file: file.to_vec()
})) }))
} }
} else { } else {

View File

@ -45,6 +45,11 @@ pub enum Error {
}, },
#[error("{0}")] #[error("{0}")]
FederationError(Box<ServerName>, RumaError), FederationError(Box<ServerName>, RumaError),
#[error("Could not do this io: {source}")]
IoError {
#[from]
source: std::io::Error,
},
#[error("{0}")] #[error("{0}")]
BadServerResponse(&'static str), BadServerResponse(&'static str),
#[error("{0}")] #[error("{0}")]