diff --git a/.gitignore b/.gitignore index 3b6f072..678e482 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ postgres/ serveurhttp test .env +uploads/ public/ assets/tailwind.css diff --git a/src/components/upload/create.rs b/src/components/upload/create.rs index cc211fc..3afe1f1 100644 --- a/src/components/upload/create.rs +++ b/src/components/upload/create.rs @@ -29,6 +29,11 @@ use diesel::prelude::*; use httpserver::DB; +use bigdecimal::ToPrimitive; +use bigdecimal::BigDecimal; +#[cfg(feature = "server")] +use diesel::dsl; + use dioxus_primitives::toast::{ ToastOptions, @@ -38,12 +43,24 @@ use dioxus_primitives::toast::{ #[post("/api/upload")] async fn upload_file(mut upload: FileStream) -> Result { use httpserver::schema::file; + let s_config = ServerConfig::load(); let c_config = ClientConfig::load(); let mut total_len: u64 = 0; let mut error: Option<&str> = None; + let hard_limit_margin = match DB.with(|pool| file::table.select(dsl::sum(file::file_size)) + .filter(file::deleted.eq(false)) + .first::>(&mut pool.get().unwrap())) { + Ok(Some(val)) => s_config.upload_storage_limit_hard - val.to_u64().unwrap(), + Ok(None) => s_config.upload_storage_limit_hard, + _ => { + tracing::error!("db request failed"); + return HttpError::bad_request("db request failed"); + } + }; + let filename: String = loop { let cur = random_string::generate(20, "ABCDEFGHIJKLMNOPQRTSTUVWXYZ0123456789"); if !fs::exists(s_config.upload_folder.clone() + &cur).expect("can't check if file exists") { @@ -55,6 +72,9 @@ async fn upload_file(mut upload: FileStream) -> Result { if size > c_config.upload_max_size.try_into().unwrap() { return HttpError::payload_too_large("this file is too large"); } + if size > hard_limit_margin { + return HttpError::payload_too_large("this file exceeds the hard upload size limit, please wait until some space is freed"); + } } if upload.file_name().len() > 255 { @@ -75,6 +95,10 @@ async fn upload_file(mut upload: FileStream) -> Result { error = Some("Uploaded file too large"); break; } + if total_len > c_config.upload_max_size.try_into().unwrap() { + error = Some("Hard limit reached"); + break; + } if file.write(&bytes).is_err() { error = Some("failed write"); break; }; }, diff --git a/src/components/upload/info.rs b/src/components/upload/info.rs index 8d0f2f8..f0f24b9 100644 --- a/src/components/upload/info.rs +++ b/src/components/upload/info.rs @@ -22,7 +22,6 @@ struct FetchedInfo { filename: String, size: i64, created_at: DateTime, - deletion_pos: Option, } @@ -39,12 +38,12 @@ async fn get_file_info(filename: String) -> Result { } let db_file = result.unwrap(); if db_file.len() == 0 { - return HttpError::internal_server_error("The requested file does not exist"); + return HttpError::not_found("The requested file does not exist"); } let db_file = &db_file[0]; - Ok(FetchedInfo { filename: db_file.0.clone(), size: db_file.1, created_at: db_file.2.into(), deletion_pos: None }) + Ok(FetchedInfo { filename: db_file.0.clone(), size: db_file.1, created_at: db_file.2.into()}) } fn show_file_info(file_id: String, file_info: &FetchedInfo) -> Element { @@ -56,12 +55,6 @@ fn show_file_info(file_id: String, file_info: &FetchedInfo) -> Element { p { "File name : {file_info.filename}" } p { "This file was uploaded on the {creation_date}" } p { "File size : {byte_to_human_size(file_info.size.try_into().unwrap())}" } - p { "Position in the deletion queue : ", - match file_info.deletion_pos { - Some(pos) => pos.to_string(), - None => "files aren't deleted for a week after the time of upload".to_string() - } - } a { class: "height-20px text-blue-800 underline", rel: "noopener noreferrer", @@ -80,7 +73,7 @@ pub fn FileInfo(file: String) -> Element { match &*file_info.read_unchecked() { Some(Ok(info)) => { show_file_info(file_id, info) }, - Some(Err(err)) => { rsx! { p {"server retrned an error : {err}"} } }, + Some(Err(err)) => { rsx! { p {"server returned an error {err}"} } }, _ => { rsx! { p {"Loading ..."} } } } } diff --git a/src/components/upload/stats.rs b/src/components/upload/stats.rs index 10908bc..2b18f55 100644 --- a/src/components/upload/stats.rs +++ b/src/components/upload/stats.rs @@ -60,8 +60,8 @@ fn render_stats(stats : &UploadServerStats) -> Element { let mut soft_percentage = stats.used_space as f32 / stats.total_limit_soft as f32; let mut hard_percentage = stats.used_space as f32 / stats.total_limit_hard as f32; - soft_percentage = (soft_percentage * 1000f32).round() / 1000f32; - hard_percentage = (hard_percentage * 1000f32).round() / 1000f32; + soft_percentage = (soft_percentage * 1000f32).round() / 10f32; + hard_percentage = (hard_percentage * 1000f32).round() / 10f32; rsx! { diff --git a/src/config.rs b/src/config.rs index 078fd71..c86f83d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -14,8 +14,8 @@ impl ServerConfig { upload_folder: "./uploads/".to_string(), db_url: std::env::var("HTTPSERVER_DATABASE_URL").expect("missing HTTPSERVER_DATABASE_URL"), - upload_storage_limit_soft: 1024 * 1024 * 1024 * 200, - upload_storage_limit_hard: 1024 * 1024 * 1024 * 300, + upload_storage_limit_soft: 1024 * 1024 * 1024 * 3, + upload_storage_limit_hard: 1024 * 1024 * 1024 * 5, } } } diff --git a/src/main.rs b/src/main.rs index 4faab2d..b8ba1aa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -60,7 +60,7 @@ fn main() { #[cfg(feature = "server")] task::spawn(scheduled_tasks_loop()); - dioxus::logger::init(Level::WARN).expect("failed to init logger"); + dioxus::logger::init(Level::INFO).expect("failed to init logger"); dioxus::launch(App); } diff --git a/src/tasks/register.rs b/src/tasks/register.rs index 21d8234..0a3a10b 100644 --- a/src/tasks/register.rs +++ b/src/tasks/register.rs @@ -1,12 +1,13 @@ use super::scheduled_tasks::ScheduledTaskInfo; mod upload; -use upload::upload_cleanup; +use upload::{ upload_cleanup, upload_detect }; pub fn register_tasks() -> Vec { let mut res : Vec = vec![]; ScheduledTaskInfo::every_fifteen_minutes(&mut res, "/upload cleanup".to_owned(), upload_cleanup); + ScheduledTaskInfo::on_load(&mut res, "/upload detect".to_owned(), upload_detect); res } diff --git a/src/tasks/register/upload.rs b/src/tasks/register/upload.rs index 49b0fd2..dcdf39e 100644 --- a/src/tasks/register/upload.rs +++ b/src/tasks/register/upload.rs @@ -33,7 +33,7 @@ fn upload_delete_files(files: Vec, space_needed: u64) { match DB.with(|pool| diesel::update(&files[i]).set(file::deleted.eq(true)) .execute(&mut pool.get().unwrap())) { Ok(_) => { tracing::info!("successfully set the file as deleted"); }, - Err(err) => { tracing::error!("failed to set the file as deleted"); } + Err(_) => { tracing::error!("failed to set the file as deleted"); } }; tracing::info!("{}% freed", (freed as f64 / space_needed as f64) * 100.0); @@ -76,3 +76,32 @@ pub fn upload_cleanup(_: bool) { upload_delete_files(files, space_needed); } } + +pub fn upload_detect(_: bool) { + let s_config = ServerConfig::load(); + + let files = match DB.with(|pool| GetFile::query() + .filter(schema::file::deleted.eq(false)) + .load(&mut pool.get().unwrap())) { + Ok(files) => files, + Err(_) => { tracing::error!("failed to get a file list"); return; } + }; + + for file in files { + let exists = match fs::exists(String::new() + &s_config.upload_folder + &file.stored_filename) { + Ok(val) => { val } + Err(err) => { tracing::error!("can't check file existence, err : {}", err); true } + }; + + if exists == false { + tracing::info!("file {} doesn't exist anymore, setting as deleted", file.stored_filename); + + match DB.with(|pool| diesel::update(&file).set(schema::file::deleted.eq(true)) + .execute(&mut pool.get().unwrap())) { + Ok(_) => { tracing::info!("successfully set the file as deleted"); }, + Err(_) => { tracing::error!("failed to set the file as deleted"); } + }; + } + } + +} diff --git a/src/tasks/scheduled_tasks.rs b/src/tasks/scheduled_tasks.rs index 9b33fca..40e80c9 100644 --- a/src/tasks/scheduled_tasks.rs +++ b/src/tasks/scheduled_tasks.rs @@ -72,10 +72,18 @@ pub async fn scheduled_tasks_loop() { for task in &mut tasks { let mut first_exec = false; + + if let Some(last_exec) = task.last_exec { + if(task.interval == Duration::from_secs(0)) { + continue ; + } + } + let should_exec = match task.last_exec { Some(time) => SystemTime::now().duration_since(time).expect("how ???") >= task.interval, None => { first_exec = true; true } }; + if should_exec { tracing::info!("running task {}", task.name); (task.f)(first_exec); diff --git a/todo b/todo deleted file mode 100644 index 61dc33d..0000000 --- a/todo +++ /dev/null @@ -1,17 +0,0 @@ - -/upload - - can upload file DONE - - show current server space usage - - direct download on /upload/dl/ - - can upload up to 100GB - - if video, set mime - - deletion rules: - - older than 1 month - - more than 300GB used, delete oldest until under 200GB used, unless it's less than 7 days old. hard limit on 500GB - - file info on /upload/ - - file deletion queue position - - show file deletion info - -/status (check status of dependent devices, maybe useless) - - /boot (later, previous program is bad)