diff --git a/src/config.rs b/src/config.rs index 5d6285b..078fd71 100644 --- a/src/config.rs +++ b/src/config.rs @@ -14,9 +14,8 @@ impl ServerConfig { upload_folder: "./uploads/".to_string(), db_url: std::env::var("HTTPSERVER_DATABASE_URL").expect("missing HTTPSERVER_DATABASE_URL"), - upload_storage_limit_soft: 1024 * 1024 * 1024 * 200, + upload_storage_limit_soft: 1024 * 1024 * 1024 * 200, upload_storage_limit_hard: 1024 * 1024 * 1024 * 300, - } } } diff --git a/src/main.rs b/src/main.rs index b164802..4faab2d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,12 +2,11 @@ use dioxus::prelude::*; use crate::dioxus_fullstack::FullstackContext; + use tracing::Level; pub mod config; -use async_std::task; -use std::time::Duration; use crate::components::toast::ToastProvider; @@ -21,6 +20,14 @@ use crate::views::{ #[cfg(feature = "server")] use crate::config::ServerConfig; +#[cfg(feature = "server")] +use async_std::task; + +#[cfg(feature = "server")] +mod tasks; +#[cfg(feature = "server")] +use tasks::scheduled_tasks_loop; + mod components; mod views; @@ -51,18 +58,12 @@ fn main() { let _ = ServerConfig::load(); #[cfg(feature = "server")] - task::spawn(scheduled_tasks()); + task::spawn(scheduled_tasks_loop()); - dioxus::logger::init(Level::INFO).expect("failed to init logger"); + dioxus::logger::init(Level::WARN).expect("failed to init logger"); dioxus::launch(App); } -async fn scheduled_tasks() { - loop { - task::sleep(Duration::from_secs(1)).await; -// tracing::info!("patate douce"); - } -} #[component] fn ErrorLayout() -> Element { diff --git a/src/models/t_file.rs b/src/models/t_file.rs index 8cc61ea..7ebb2fe 100644 --- a/src/models/t_file.rs +++ b/src/models/t_file.rs @@ -4,9 +4,8 @@ use diesel::prelude::*; use crate::schema::file; -#[derive(Queryable, Selectable)] +#[derive(HasQuery, Identifiable, AsChangeset)] #[diesel(table_name = file)] -#[diesel(check_for_backend(diesel::pg::Pg))] pub struct GetFile { pub id: i64, pub created_at: SystemTime, diff --git a/src/tasks/mod.rs b/src/tasks/mod.rs new file mode 100644 index 0000000..3708e40 --- /dev/null +++ b/src/tasks/mod.rs @@ -0,0 +1,4 @@ +mod scheduled_tasks; +pub use scheduled_tasks::scheduled_tasks_loop; + +mod register; diff --git a/src/tasks/register.rs b/src/tasks/register.rs new file mode 100644 index 0000000..21d8234 --- /dev/null +++ b/src/tasks/register.rs @@ -0,0 +1,12 @@ +use super::scheduled_tasks::ScheduledTaskInfo; + +mod upload; +use upload::upload_cleanup; + +pub fn register_tasks() -> Vec { + let mut res : Vec = vec![]; + + ScheduledTaskInfo::every_fifteen_minutes(&mut res, "/upload cleanup".to_owned(), upload_cleanup); + + res +} diff --git a/src/tasks/register/upload.rs b/src/tasks/register/upload.rs new file mode 100644 index 0000000..49b0fd2 --- /dev/null +++ b/src/tasks/register/upload.rs @@ -0,0 +1,78 @@ +use std::time::Duration; +use std::time::SystemTime; + +use bigdecimal::BigDecimal; +use httpserver::DB; + +use diesel::prelude::*; +use diesel::dsl; + +use httpserver::config::ServerConfig; +use httpserver::schema; +use httpserver::utils::byte_to_human_size; + +use std::fs; + +use bigdecimal::ToPrimitive; +use httpserver::models::GetFile; + +fn upload_delete_files(files: Vec, space_needed: u64) { + use schema::file; + + let mut freed: u64 = 0; + let s_config = ServerConfig::load(); + let mut i = 0; + + while freed < space_needed && i < files.len() { + match fs::remove_file(String::new() + &s_config.upload_folder + &files[i].stored_filename) { + Ok(_) => { tracing::info!("delted {}", files[i].stored_filename); }, + Err(err) => { tracing::error!("failed to delete, reason : {:?}", err); } + }; + + freed += files[i].file_size as u64; + match DB.with(|pool| diesel::update(&files[i]).set(file::deleted.eq(true)) + .execute(&mut pool.get().unwrap())) { + Ok(_) => { tracing::info!("successfully set the file as deleted"); }, + Err(err) => { tracing::error!("failed to set the file as deleted"); } + }; + + tracing::info!("{}% freed", (freed as f64 / space_needed as f64) * 100.0); + i += 1; + } +} + +pub fn upload_cleanup(_: bool) { + use schema::file; + + let s_config = ServerConfig::load(); + + let current_space_used: u64 = match DB.with(|pool| file::table.select(dsl::sum(file::file_size)) + .filter(file::deleted.eq(false)) + .first::>(&mut pool.get().unwrap())) { + Ok(Some(val)) => val.to_u64().unwrap(), + Ok(None) => 0, + _ => { tracing::error!("db request failed"); return; } + }; + + if current_space_used >= s_config.upload_storage_limit_soft { + tracing::info!("soft limit reached, searching for files to delete"); + + let space_needed = current_space_used - s_config.upload_storage_limit_soft; + tracing::info!("{} need to be freed", byte_to_human_size(space_needed)); + + let files = match DB.with(|pool| GetFile::query() + .filter(file::deleted.eq(false)) + .filter(file::created_at.lt(SystemTime::now() - Duration::from_hours(7 * 24))) + .order_by(file::id.desc()) + .load(&mut pool.get().unwrap())) { + Ok(files) => files, + Err(_) => { tracing::error!("failed to get a file list"); return; } + }; + + if files.len() == 0 { + tracing::info!("no files were found"); + } + + upload_delete_files(files, space_needed); + } +} diff --git a/src/tasks/scheduled_tasks.rs b/src/tasks/scheduled_tasks.rs new file mode 100644 index 0000000..9b33fca --- /dev/null +++ b/src/tasks/scheduled_tasks.rs @@ -0,0 +1,86 @@ +use async_std::task; +use std::time::{Duration, SystemTime}; + +use super::register::register_tasks; + +// Usage : +// +// interval: +// - time interval between executions +// - 0 seconds to only execute on startup +// +// last_exec: +// - time of last execution, should be 0 by default +// +// name : +// - name of the task that will be displayed in the logs +// +// f : +// - function that will be executed + + +pub struct ScheduledTaskInfo { + interval: Duration, + last_exec: Option, + name : String, + f: fn(bool) +} + +#[allow(dead_code)] +impl ScheduledTaskInfo { + pub fn every_second(task_list: &mut Vec, name: String, function: fn(bool)) { + task_list.push( ScheduledTaskInfo { interval: Duration::from_secs(1) , last_exec : None, name: name, f : function }); + } + pub fn every_ten_seconds(task_list: &mut Vec, name: String, function: fn(bool)) { + task_list.push( ScheduledTaskInfo { interval: Duration::from_secs(10) , last_exec : None, name: name, f : function }); + } + pub fn every_half_minute(task_list: &mut Vec, name: String, function: fn(bool)) { + task_list.push( ScheduledTaskInfo { interval: Duration::from_secs(30) , last_exec : None, name: name, f : function }); + } + pub fn every_minute(task_list: &mut Vec, name: String, function: fn(bool)) { + task_list.push( ScheduledTaskInfo { interval: Duration::from_mins(1) , last_exec : None, name: name, f : function }); + } + pub fn every_fifteen_minutes(task_list: &mut Vec, name: String, function: fn(bool)) { + task_list.push( ScheduledTaskInfo { interval: Duration::from_mins(15) , last_exec : None, name: name, f : function }); + } + pub fn every_half_hour(task_list: &mut Vec, name: String, function: fn(bool)) { + task_list.push( ScheduledTaskInfo { interval: Duration::from_mins(30) , last_exec : None, name: name, f : function }); + } + pub fn every_hour(task_list: &mut Vec, name: String, function: fn(bool)) { + task_list.push( ScheduledTaskInfo { interval: Duration::from_hours(1) , last_exec : None, name: name, f : function }); + } + pub fn every_day(task_list: &mut Vec, name: String, function: fn(bool)) { + task_list.push( ScheduledTaskInfo { interval: Duration::from_hours(24) , last_exec : None, name: name, f : function }); + } + pub fn every_week(task_list: &mut Vec, name: String, function: fn(bool)) { + task_list.push( ScheduledTaskInfo { interval: Duration::from_hours(24 * 7) , last_exec : None, name: name, f : function }); + } + pub fn every_interval(task_list: &mut Vec, name: String, function: fn(bool), interval: Duration) { + task_list.push( ScheduledTaskInfo { interval: interval , last_exec : None, name: name, f : function }); + } + + pub fn on_load(task_list: &mut Vec, name: String, function: fn(bool)) { + task_list.push( ScheduledTaskInfo { interval: Duration::from_secs(0), last_exec: None, name, f: function } ); + } +} + +pub async fn scheduled_tasks_loop() { + let mut tasks = register_tasks(); + loop { + task::sleep(Duration::from_secs(1)).await; + + + for task in &mut tasks { + let mut first_exec = false; + let should_exec = match task.last_exec { + Some(time) => SystemTime::now().duration_since(time).expect("how ???") >= task.interval, + None => { first_exec = true; true } + }; + if should_exec { + tracing::info!("running task {}", task.name); + (task.f)(first_exec); + task.last_exec = Some(SystemTime::now()); + } + } + } +}