auto delete automatically

src/config.rs

@@ -14,9 +14,8 @@ impl ServerConfig {
             upload_folder: "./uploads/".to_string(),
             db_url: std::env::var("HTTPSERVER_DATABASE_URL").expect("missing HTTPSERVER_DATABASE_URL"),
 
-            upload_storage_limit_soft: 1024 * 1024 * 1024 * 200, 
+            upload_storage_limit_soft: 1024 * 1024 * 1024 * 200,
             upload_storage_limit_hard: 1024 * 1024 * 1024 * 300,
-
         }
     }
 }

src/main.rs

@@ -2,12 +2,11 @@ use dioxus::prelude::*;
 
 use crate::dioxus_fullstack::FullstackContext;
 
+
 use tracing::Level;
 
 pub mod config;
 
-use async_std::task;
-use std::time::Duration;
 
 use crate::components::toast::ToastProvider;
 
@@ -21,6 +20,14 @@ use crate::views::{
 #[cfg(feature = "server")]
 use crate::config::ServerConfig;
 
+#[cfg(feature = "server")]
+use async_std::task;
+
+#[cfg(feature = "server")]
+mod tasks;
+#[cfg(feature = "server")]
+use tasks::scheduled_tasks_loop;
+
 mod components;
 mod views;
 
@@ -51,18 +58,12 @@ fn main() {
     let _ = ServerConfig::load();
 
     #[cfg(feature = "server")]
-    task::spawn(scheduled_tasks());
+    task::spawn(scheduled_tasks_loop());
 
-    dioxus::logger::init(Level::INFO).expect("failed to init logger");
+    dioxus::logger::init(Level::WARN).expect("failed to init logger");
     dioxus::launch(App);
 }
 
-async fn scheduled_tasks() {
-    loop {
-        task::sleep(Duration::from_secs(1)).await;
-//        tracing::info!("patate douce");
-    }
-}
 
 #[component]
 fn ErrorLayout() -> Element {

src/models/t_file.rs

@@ -4,9 +4,8 @@ use diesel::prelude::*;
 
 use crate::schema::file;
 
-#[derive(Queryable, Selectable)]
+#[derive(HasQuery, Identifiable, AsChangeset)]
 #[diesel(table_name = file)]
-#[diesel(check_for_backend(diesel::pg::Pg))]
 pub struct GetFile {
     pub id: i64,
     pub created_at: SystemTime,

src/tasks/mod.rs

@@ -0,0 +1,4 @@
+mod scheduled_tasks;
+pub use scheduled_tasks::scheduled_tasks_loop;
+
+mod register;

src/tasks/register.rs

@@ -0,0 +1,12 @@
+use super::scheduled_tasks::ScheduledTaskInfo;
+
+mod upload;
+use upload::upload_cleanup;
+
+pub fn register_tasks() -> Vec<ScheduledTaskInfo> {
+    let mut res : Vec<ScheduledTaskInfo> = vec![];
+
+    ScheduledTaskInfo::every_fifteen_minutes(&mut res, "/upload cleanup".to_owned(), upload_cleanup);
+
+    res
+}

src/tasks/register/upload.rs

@@ -0,0 +1,78 @@
+use std::time::Duration;
+use std::time::SystemTime;
+
+use bigdecimal::BigDecimal;
+use httpserver::DB;
+
+use diesel::prelude::*;
+use diesel::dsl;
+
+use httpserver::config::ServerConfig;
+use httpserver::schema;
+use httpserver::utils::byte_to_human_size;
+
+use std::fs;
+
+use bigdecimal::ToPrimitive;
+use httpserver::models::GetFile;
+
+fn upload_delete_files(files: Vec<GetFile>, space_needed: u64) {
+    use schema::file;
+
+    let mut freed: u64 = 0;
+    let s_config = ServerConfig::load();
+    let mut i = 0;
+
+    while freed < space_needed && i < files.len() {
+        match fs::remove_file(String::new() + &s_config.upload_folder + &files[i].stored_filename) {
+            Ok(_) => { tracing::info!("delted {}", files[i].stored_filename); },
+            Err(err) => { tracing::error!("failed to delete, reason : {:?}", err); }
+        };
+
+        freed += files[i].file_size as u64;
+        match DB.with(|pool| diesel::update(&files[i]).set(file::deleted.eq(true))
+            .execute(&mut pool.get().unwrap())) {
+            Ok(_) => { tracing::info!("successfully set the file as deleted"); },
+            Err(err) => { tracing::error!("failed to set the file as deleted"); }
+        };
+
+        tracing::info!("{}% freed", (freed as f64 / space_needed as f64) * 100.0);
+        i += 1;
+    }
+}
+
+pub fn upload_cleanup(_: bool) {
+    use schema::file;
+
+    let s_config = ServerConfig::load();
+
+    let current_space_used: u64 = match DB.with(|pool| file::table.select(dsl::sum(file::file_size))
+        .filter(file::deleted.eq(false))
+        .first::<Option<BigDecimal>>(&mut pool.get().unwrap())) {
+            Ok(Some(val)) => val.to_u64().unwrap(),
+            Ok(None) => 0,
+            _ => { tracing::error!("db request failed"); return; }
+        };
+
+    if current_space_used >= s_config.upload_storage_limit_soft {
+        tracing::info!("soft limit reached, searching for files to delete");
+
+        let space_needed = current_space_used - s_config.upload_storage_limit_soft;
+        tracing::info!("{} need to be freed", byte_to_human_size(space_needed));
+
+        let files = match DB.with(|pool| GetFile::query()
+            .filter(file::deleted.eq(false))
+            .filter(file::created_at.lt(SystemTime::now() - Duration::from_hours(7 * 24)))
+            .order_by(file::id.desc())
+            .load(&mut pool.get().unwrap())) {
+                Ok(files) => files,
+                Err(_) => { tracing::error!("failed to get a file list"); return; }
+            };
+
+        if files.len() == 0 {
+            tracing::info!("no files were found");
+        }
+
+        upload_delete_files(files, space_needed);
+    }
+}

src/tasks/scheduled_tasks.rs

@@ -0,0 +1,86 @@
+use async_std::task;
+use std::time::{Duration, SystemTime};
+
+use super::register::register_tasks;
+
+// Usage : 
+//
+// interval:
+//   - time interval between executions
+//   - 0 seconds to only execute on startup
+//
+// last_exec:
+//   - time of last execution, should be 0 by default
+//
+// name : 
+//   - name of the task that will be displayed in the logs
+//
+// f :
+//   - function that will be executed
+
+
+pub struct ScheduledTaskInfo {
+    interval: Duration,
+    last_exec: Option<SystemTime>,
+    name : String,
+    f: fn(bool)
+}
+
+#[allow(dead_code)]
+impl ScheduledTaskInfo {
+    pub fn every_second(task_list: &mut Vec<ScheduledTaskInfo>, name: String, function: fn(bool)) {
+        task_list.push( ScheduledTaskInfo { interval:  Duration::from_secs(1) , last_exec : None, name: name, f : function });
+    }
+    pub fn every_ten_seconds(task_list: &mut Vec<ScheduledTaskInfo>, name: String, function: fn(bool)) {
+        task_list.push( ScheduledTaskInfo { interval:  Duration::from_secs(10) , last_exec : None, name: name, f : function });
+    }
+    pub fn every_half_minute(task_list: &mut Vec<ScheduledTaskInfo>, name: String, function: fn(bool)) {
+        task_list.push( ScheduledTaskInfo { interval:  Duration::from_secs(30) , last_exec : None, name: name, f : function });
+    }
+    pub fn every_minute(task_list: &mut Vec<ScheduledTaskInfo>, name: String, function: fn(bool)) {
+        task_list.push( ScheduledTaskInfo { interval:  Duration::from_mins(1) , last_exec : None, name: name, f : function });
+    }
+    pub fn every_fifteen_minutes(task_list: &mut Vec<ScheduledTaskInfo>, name: String, function: fn(bool)) {
+        task_list.push( ScheduledTaskInfo { interval:  Duration::from_mins(15) , last_exec : None, name: name, f : function });
+    }
+    pub fn every_half_hour(task_list: &mut Vec<ScheduledTaskInfo>, name: String,  function: fn(bool)) {
+        task_list.push( ScheduledTaskInfo { interval:  Duration::from_mins(30) , last_exec : None, name: name, f : function });
+    }
+    pub fn every_hour(task_list: &mut Vec<ScheduledTaskInfo>, name: String, function: fn(bool)) {
+        task_list.push( ScheduledTaskInfo { interval:  Duration::from_hours(1) , last_exec : None, name: name, f : function });
+    }
+    pub fn every_day(task_list: &mut Vec<ScheduledTaskInfo>, name: String, function: fn(bool)) {
+        task_list.push( ScheduledTaskInfo { interval:  Duration::from_hours(24) , last_exec : None, name: name, f : function });
+    }
+    pub fn every_week(task_list: &mut Vec<ScheduledTaskInfo>, name: String, function: fn(bool)) {
+        task_list.push( ScheduledTaskInfo { interval:  Duration::from_hours(24 * 7) , last_exec : None, name: name, f : function });
+    }
+    pub fn every_interval(task_list: &mut Vec<ScheduledTaskInfo>, name: String,  function: fn(bool), interval: Duration) {
+        task_list.push( ScheduledTaskInfo { interval:  interval , last_exec : None, name: name, f : function });
+    }
+    
+    pub fn on_load(task_list: &mut Vec<ScheduledTaskInfo>, name: String, function: fn(bool)) {
+        task_list.push( ScheduledTaskInfo { interval: Duration::from_secs(0), last_exec: None, name, f: function } );
+    }
+}
+
+pub async fn scheduled_tasks_loop() {
+    let mut tasks = register_tasks();
+    loop {
+        task::sleep(Duration::from_secs(1)).await;
+
+
+        for task in &mut tasks {
+            let mut first_exec = false;
+            let should_exec = match task.last_exec {
+                Some(time) => SystemTime::now().duration_since(time).expect("how ???") >= task.interval,
+                None => { first_exec = true; true }
+            };
+            if should_exec {
+                tracing::info!("running task {}", task.name);
+                (task.f)(first_exec);
+                task.last_exec = Some(SystemTime::now());
+            }
+        }
+    }
+}