hard limit enforcement, detect deleted files on load

.gitignore

@@ -9,6 +9,7 @@ postgres/
 serveurhttp
 test
 .env
+uploads/
 
 public/
 assets/tailwind.css

src/components/upload/create.rs

@@ -29,6 +29,11 @@ use diesel::prelude::*;
 use httpserver::DB;
 
 
+use bigdecimal::ToPrimitive;
+use bigdecimal::BigDecimal;
+#[cfg(feature = "server")]
+use diesel::dsl;
+
 
 use dioxus_primitives::toast::{
     ToastOptions,
@@ -38,12 +43,24 @@ use dioxus_primitives::toast::{
 #[post("/api/upload")]
 async fn upload_file(mut upload: FileStream) -> Result<String, HttpError> {
     use httpserver::schema::file;
+
     let s_config = ServerConfig::load();
     let c_config = ClientConfig::load();
 
     let mut total_len: u64 = 0;
     let mut error: Option<&str> = None;
 
+    let hard_limit_margin = match DB.with(|pool| file::table.select(dsl::sum(file::file_size))
+        .filter(file::deleted.eq(false))
+        .first::<Option<BigDecimal>>(&mut pool.get().unwrap())) {
+            Ok(Some(val)) => s_config.upload_storage_limit_hard - val.to_u64().unwrap(),
+            Ok(None) => s_config.upload_storage_limit_hard,
+            _ => {
+                tracing::error!("db request failed");
+                return HttpError::bad_request("db request failed");
+            }
+        };
+
     let filename: String = loop {
         let cur = random_string::generate(20, "ABCDEFGHIJKLMNOPQRTSTUVWXYZ0123456789");
         if !fs::exists(s_config.upload_folder.clone() + &cur).expect("can't check if file exists") {
@@ -55,6 +72,9 @@ async fn upload_file(mut upload: FileStream) -> Result<String, HttpError> {
         if size > c_config.upload_max_size.try_into().unwrap() {
             return HttpError::payload_too_large("this file is too large");
         }
+        if size > hard_limit_margin {
+            return HttpError::payload_too_large("this file exceeds the hard upload size limit, please wait until some space is freed");
+        }
     }
 
     if upload.file_name().len() > 255 {
@@ -75,6 +95,10 @@ async fn upload_file(mut upload: FileStream) -> Result<String, HttpError> {
                     error = Some("Uploaded file too large");
                     break;
                 }
+                if total_len > c_config.upload_max_size.try_into().unwrap() {
+                    error = Some("Hard limit reached");
+                    break;
+                }
 
                 if file.write(&bytes).is_err() { error = Some("failed write"); break; };
             },

src/components/upload/info.rs

@@ -22,7 +22,6 @@ struct FetchedInfo {
     filename: String,
     size: i64,
     created_at: DateTime<Utc>,
-    deletion_pos: Option<u32>,
 }
 
 
@@ -39,12 +38,12 @@ async fn get_file_info(filename: String) -> Result<FetchedInfo, HttpError> {
     }
     let db_file = result.unwrap();
     if db_file.len() == 0 {
-        return HttpError::internal_server_error("The requested file does not exist");
+        return HttpError::not_found("The requested file does not exist");
     }
 
     let db_file = &db_file[0];
 
-    Ok(FetchedInfo { filename: db_file.0.clone(), size: db_file.1, created_at: db_file.2.into(), deletion_pos: None })
+    Ok(FetchedInfo { filename: db_file.0.clone(), size: db_file.1, created_at: db_file.2.into()})
 }
 
 fn show_file_info(file_id: String, file_info: &FetchedInfo) -> Element {
@@ -56,12 +55,6 @@ fn show_file_info(file_id: String, file_info: &FetchedInfo) -> Element {
                 p { "File name : {file_info.filename}" }
                 p { "This file was uploaded on the {creation_date}" }
                 p { "File size : {byte_to_human_size(file_info.size.try_into().unwrap())}" }
-                p { "Position in the deletion queue : ",
-                    match file_info.deletion_pos {
-                        Some(pos) => pos.to_string(),
-                        None => "files aren't deleted for a week after the time of upload".to_string()
-                    }
-                }
                 a {
                     class: "height-20px text-blue-800 underline",
                     rel: "noopener noreferrer",
@@ -80,7 +73,7 @@ pub fn FileInfo(file: String) -> Element {
     
     match &*file_info.read_unchecked() {
         Some(Ok(info)) => { show_file_info(file_id, info) },
-        Some(Err(err)) => { rsx! { p {"server retrned an error : {err}"} } },
+        Some(Err(err)) => { rsx! { p {"server returned an error {err}"} } },
         _ => { rsx! { p {"Loading ..."} } }
     }
 }

src/components/upload/stats.rs

@@ -60,8 +60,8 @@ fn render_stats(stats : &UploadServerStats) -> Element {
     let mut soft_percentage = stats.used_space as f32 / stats.total_limit_soft as f32;
     let mut hard_percentage = stats.used_space as f32 / stats.total_limit_hard as f32;
 
-    soft_percentage = (soft_percentage * 1000f32).round() / 1000f32;
+    soft_percentage = (soft_percentage * 1000f32).round() / 10f32;
-    hard_percentage = (hard_percentage * 1000f32).round() / 1000f32;
+    hard_percentage = (hard_percentage * 1000f32).round() / 10f32;
 
 
     rsx! { 

src/config.rs

@@ -14,8 +14,8 @@ impl ServerConfig {
             upload_folder: "./uploads/".to_string(),
             db_url: std::env::var("HTTPSERVER_DATABASE_URL").expect("missing HTTPSERVER_DATABASE_URL"),
 
-            upload_storage_limit_soft: 1024 * 1024 * 1024 * 200,
+            upload_storage_limit_soft: 1024 * 1024 * 1024 * 3,
-            upload_storage_limit_hard: 1024 * 1024 * 1024 * 300,
+            upload_storage_limit_hard: 1024 * 1024 * 1024 * 5,
         }
     }
 }

src/main.rs

@@ -60,7 +60,7 @@ fn main() {
     #[cfg(feature = "server")]
     task::spawn(scheduled_tasks_loop());
 
-    dioxus::logger::init(Level::WARN).expect("failed to init logger");
+    dioxus::logger::init(Level::INFO).expect("failed to init logger");
     dioxus::launch(App);
 }
 

src/tasks/register.rs

@@ -1,12 +1,13 @@
 use super::scheduled_tasks::ScheduledTaskInfo;
 
 mod upload;
-use upload::upload_cleanup;
+use upload::{ upload_cleanup, upload_detect };
 
 pub fn register_tasks() -> Vec<ScheduledTaskInfo> {
     let mut res : Vec<ScheduledTaskInfo> = vec![];
 
     ScheduledTaskInfo::every_fifteen_minutes(&mut res, "/upload cleanup".to_owned(), upload_cleanup);
+    ScheduledTaskInfo::on_load(&mut res, "/upload detect".to_owned(), upload_detect);
 
     res
 }

src/tasks/register/upload.rs

@@ -33,7 +33,7 @@ fn upload_delete_files(files: Vec<GetFile>, space_needed: u64) {
         match DB.with(|pool| diesel::update(&files[i]).set(file::deleted.eq(true))
             .execute(&mut pool.get().unwrap())) {
             Ok(_) => { tracing::info!("successfully set the file as deleted"); },
-            Err(err) => { tracing::error!("failed to set the file as deleted"); }
+            Err(_) => { tracing::error!("failed to set the file as deleted"); }
         };
 
         tracing::info!("{}% freed", (freed as f64 / space_needed as f64) * 100.0);
@@ -76,3 +76,32 @@ pub fn upload_cleanup(_: bool) {
         upload_delete_files(files, space_needed);
     }
 }
+
+pub fn upload_detect(_: bool) {
+    let s_config = ServerConfig::load();
+
+    let files = match DB.with(|pool| GetFile::query()
+        .filter(schema::file::deleted.eq(false))
+        .load(&mut pool.get().unwrap())) {
+            Ok(files) => files,
+            Err(_) => { tracing::error!("failed to get a file list"); return; }
+        };
+
+    for file in files {
+        let exists = match fs::exists(String::new() + &s_config.upload_folder + &file.stored_filename) {
+            Ok(val) => { val }
+            Err(err) => { tracing::error!("can't check file existence, err : {}", err); true }
+        };
+
+        if exists == false {
+            tracing::info!("file {} doesn't exist anymore, setting as deleted", file.stored_filename);
+
+            match DB.with(|pool| diesel::update(&file).set(schema::file::deleted.eq(true))
+                .execute(&mut pool.get().unwrap())) {
+                Ok(_) => { tracing::info!("successfully set the file as deleted"); },
+                Err(_) => { tracing::error!("failed to set the file as deleted"); }
+            };
+        }
+    }
+    
+}

src/tasks/scheduled_tasks.rs

@@ -72,10 +72,18 @@ pub async fn scheduled_tasks_loop() {
 
         for task in &mut tasks {
             let mut first_exec = false;
+
+            if let Some(last_exec) = task.last_exec {
+                if(task.interval == Duration::from_secs(0)) {
+                    continue ;
+                }
+            }
+
             let should_exec = match task.last_exec {
                 Some(time) => SystemTime::now().duration_since(time).expect("how ???") >= task.interval,
                 None => { first_exec = true; true }
             };
+
             if should_exec {
                 tracing::info!("running task {}", task.name);
                 (task.f)(first_exec);

todo

@@ -1,17 +0,0 @@
-
-/upload
- - can upload file DONE
- - show current server space usage
- - direct download on /upload/dl/<file id>
- - can upload up to 100GB
- - if video, set mime
- - deletion rules:
-   - older than 1 month
-   - more than 300GB used, delete oldest until under 200GB used, unless  it's less than 7 days old. hard limit on 500GB
- - file info on /upload/<file id>
- 	- file deletion queue position
- 	- show file deletion info
-
-/status (check status of dependent devices, maybe useless)
-
- /boot (later, previous program is bad)