char/conduit
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

put media in filesystem

Browse source
This commit is contained in:
hamidreza kalbasi 2021-06-04 08:06:12 +04:30 committed by Timo Kösters
parent d0ee823254
commit 972caacdc2
No known key found for this signature in database
GPG key ID: 24DA7517711A2BA4
6 changed files with 78 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/client_server/media.rs
View file

@ -38,6 +38,7 @@ pub async fn create_content_route(
);
db.media.create(
mxc.clone(),
&db.globals,
&body
.filename
.as_ref()
@ -45,7 +46,7 @@ pub async fn create_content_route(
.as_deref(),
&body.content_type.as_deref(),
&body.file,
)?;
).await?;
db.flush().await?;
@ -71,7 +72,7 @@ pub async fn get_content_route(
content_disposition,
content_type,
file,
}) = db.media.get(&mxc)?
}) = db.media.get(&db.globals, &mxc).await?
{
Ok(get_content::Response {
file,
@ -95,10 +96,11 @@ pub async fn get_content_route(
db.media.create(
mxc,
&db.globals,
&get_content_response.content_disposition.as_deref(),
&get_content_response.content_type.as_deref(),
&get_content_response.file,
)?;
).await?;
Ok(get_content_response.into())
} else {
@ -121,13 +123,14 @@ pub async fn get_content_thumbnail_route(
content_type, file, ..
}) = db.media.get_thumbnail(
mxc.clone(),
&db.globals,
body.width
.try_into()
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
body.height
.try_into()
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
)? {
).await? {
Ok(get_content_thumbnail::Response { file, content_type }.into())
} else if &*body.server_name != db.globals.server_name() && body.allow_remote {
let get_thumbnail_response = db
@ -148,12 +151,13 @@ pub async fn get_content_thumbnail_route(
db.media.upload_thumbnail(
mxc,
&db.globals,
&None,
&get_thumbnail_response.content_type,
body.width.try_into().expect("all UInts are valid u32s"),
body.height.try_into().expect("all UInts are valid u32s"),
&get_thumbnail_response.file,
)?;
).await?;
Ok(get_thumbnail_response.into())
} else {

4
src/database.rs
View file

@ -245,7 +245,7 @@ impl Database {
db.globals.bump_database_version(1)?;
info!("Migration: 0 -> 1 finished");
println!("Migration: 0 -> 1 finished");
}
if db.globals.database_version()? < 2 {
@ -262,7 +262,7 @@ impl Database {
db.globals.bump_database_version(2)?;
info!("Migration: 1 -> 2 finished");
println!("Migration: 1 -> 2 finished");
}
// This data is probably outdated

6
src/database/abstraction.rs
View file

@ -47,7 +47,7 @@ pub trait Tree: Send + Sync {
fn scan_prefix<'a>(
&'a self,
prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a>;
) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + Send + 'a>;
fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
@ -142,7 +142,7 @@ impl Tree for SledEngineTree {
fn scan_prefix<'a>(
&'a self,
prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a> {
) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + Send + 'a> {
let iter = self
.0
.scan_prefix(prefix)
@ -279,7 +279,7 @@ impl Tree for RocksDbEngineTree<'_> {
fn scan_prefix<'a>(
&'a self,
prefix: Vec<u8>,
) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + 'a> {
) -> Box<dyn Iterator<Item = (Box<[u8]>, Box<[u8]>)> + Send + 'a> {
Box::new(
self.db
.0

21
src/database/globals.rs
View file

@ -5,11 +5,7 @@ use ruma::{
EventId, MilliSecondsSinceUnixEpoch, ServerName, ServerSigningKeyId,
};
use rustls::{ServerCertVerifier, WebPKIVerifier};
use std::{
collections::{BTreeMap, HashMap},
sync::{Arc, RwLock},
time::{Duration, Instant},
};
use std::{collections::{BTreeMap, HashMap}, path::{PathBuf}, sync::{Arc, RwLock}, time::{Duration, Instant}};
use tokio::sync::Semaphore;
use trust_dns_resolver::TokioAsyncResolver;
@ -275,4 +271,19 @@ impl Globals {
.insert(b"version", &new_version.to_be_bytes())?;
Ok(())
}
pub fn get_media_folder(&self) -> PathBuf {
let mut r = PathBuf::new();
r.push(self.config.database_path.clone());
r.push("media");
r
}
pub fn get_media_file(&self, key: &Vec<u8>) -> PathBuf {
let mut r = PathBuf::new();
r.push(self.config.database_path.clone());
r.push("media");
r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD));
r
}
}

58
src/database/media.rs
View file

@ -1,9 +1,10 @@
use image::{imageops::FilterType, GenericImageView};
use crate::database::globals::Globals;
use crate::{utils, Error, Result};
use std::{mem, sync::Arc};
use super::abstraction::Tree;
use tokio::{fs::{self, File}, io::AsyncWriteExt, io::AsyncReadExt};
pub struct FileMeta {
pub content_disposition: Option<String>,
@ -16,10 +17,11 @@ pub struct Media {
}
impl Media {
/// Uploads or replaces a file.
pub fn create(
/// Uploads a file.
pub async fn create(
&self,
mxc: String,
globals: &Globals,
content_disposition: &Option<&str>,
content_type: &Option<&str>,
file: &[u8],
@ -43,15 +45,20 @@ impl Media {
.unwrap_or_default(),
);
self.mediaid_file.insert(&key, file)?;
let path = globals.get_media_file(&key);
fs::create_dir_all(path.parent().unwrap()).await?;
let mut f = File::create(path).await?;
f.write_all(file).await?;
self.mediaid_file.insert(&key, &[])?;
Ok(())
}
/// Uploads or replaces a file thumbnail.
pub fn upload_thumbnail(
pub async fn upload_thumbnail(
&self,
mxc: String,
globals: &Globals,
content_disposition: &Option<String>,
content_type: &Option<String>,
width: u32,
@ -77,20 +84,29 @@ impl Media {
.unwrap_or_default(),
);
self.mediaid_file.insert(&key, file)?;
let path = globals.get_media_file(&key);
fs::create_dir_all(path.parent().unwrap()).await?;
let mut f = File::create(path).await?;
f.write_all(file).await?;
self.mediaid_file.insert(&key, &[])?;
Ok(())
}
/// Downloads a file.
pub fn get(&self, mxc: &str) -> Result<Option<FileMeta>> {
pub async fn get(&self, globals: &Globals, mxc: &str) -> Result<Option<FileMeta>> {
let mut prefix = mxc.as_bytes().to_vec();
prefix.push(0xff);
prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail
prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
prefix.push(0xff);
if let Some((key, file)) = self.mediaid_file.scan_prefix(prefix).next() {
let mut iter = self.mediaid_file.scan_prefix(prefix);
if let Some((key, _)) = iter.next() {
let path = globals.get_media_file(&key.to_vec());
let mut file = vec![];
File::open(path).await?.read_to_end(&mut file).await?;
let mut parts = key.rsplit(|&b| b == 0xff);
let content_type = parts
@ -121,7 +137,7 @@ impl Media {
Ok(Some(FileMeta {
content_disposition,
content_type,
file: file.to_vec(),
file,
}))
} else {
Ok(None)
@ -151,7 +167,7 @@ impl Media {
/// - Server creates the thumbnail and sends it to the user
///
/// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
pub fn get_thumbnail(&self, mxc: String, width: u32, height: u32) -> Result<Option<FileMeta>> {
pub async fn get_thumbnail(&self, mxc: String, globals: &Globals, width: u32, height: u32) -> Result<Option<FileMeta>> {
let (width, height, crop) = self
.thumbnail_properties(width, height)
.unwrap_or((0, 0, false)); // 0, 0 because that's the original file
@ -169,8 +185,11 @@ impl Media {
original_prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
original_prefix.push(0xff);
if let Some((key, file)) = self.mediaid_file.scan_prefix(thumbnail_prefix).next() {
if let Some((key, _)) = self.mediaid_file.scan_prefix(thumbnail_prefix).next() {
// Using saved thumbnail
let path = globals.get_media_file(&key.to_vec());
let mut file = vec![];
File::open(path).await?.read_to_end(&mut file).await?;
let mut parts = key.rsplit(|&b| b == 0xff);
let content_type = parts
@ -201,8 +220,12 @@ impl Media {
content_type,
file: file.to_vec(),
}))
} else if let Some((key, file)) = self.mediaid_file.scan_prefix(original_prefix).next() {
} else if let Some((key, _)) = self.mediaid_file.scan_prefix(original_prefix).next() {
// Generate a thumbnail
let path = globals.get_media_file(&key.to_vec());
let mut file = vec![];
File::open(path).await?.read_to_end(&mut file).await?;
let mut parts = key.rsplit(|&b| b == 0xff);
let content_type = parts
@ -299,19 +322,24 @@ impl Media {
widthheight,
);
self.mediaid_file.insert(&thumbnail_key, &thumbnail_bytes)?;
let path = globals.get_media_file(&thumbnail_key);
fs::create_dir_all(path.parent().unwrap()).await?;
let mut f = File::create(path).await?;
f.write_all(&thumbnail_bytes).await?;
self.mediaid_file.insert(&thumbnail_key, &[])?;
Ok(Some(FileMeta {
content_disposition,
content_type,
file: thumbnail_bytes.to_vec(),
file: thumbnail_bytes.to_vec()
}))
} else {
// Couldn't parse file to generate thumbnail, send original
Ok(Some(FileMeta {
content_disposition,
content_type,
file: file.to_vec(),
file: file.to_vec()
}))
}
} else {

5
src/error.rs
View file

@ -45,6 +45,11 @@ pub enum Error {
},
#[error("{0}")]
FederationError(Box<ServerName>, RumaError),
#[error("Could not do this io: {source}")]
IoError {
#[from]
source: std::io::Error,
},
#[error("{0}")]
BadServerResponse(&'static str),
#[error("{0}")]