feat: dashboard-databases

docker-compose.yml

@@ -21,7 +21,7 @@ services:
       LOG: debug
       TZ: "Europe/Paris"
 #      TMPDIR: /scratch
-      EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiODY1Mjk2NDgtYmQ0Zi00MWMxLWFmNDItNGM1MzE3ZDEzY2JhIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
+      EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMGNjODEyYzktM2M4Ny00NmM3LWJlNjctOTI4MDljMmQ3N2NlIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
       #CHUNK_SIZE_MB: "1"
       #POOLING: 1
       #DATABASES_CONFIG_FILE: "config.toml"

src/core/agent.rs

@@ -2,13 +2,16 @@
 
 use crate::core::context::Context;
 use crate::services::backup::BackupService;
-use crate::services::config::ConfigService;
+use crate::services::config::{ConfigService, DatabaseConfig};
 use crate::services::cron::CronService;
+use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache};
 use crate::services::restore::RestoreService;
 use crate::services::status::StatusService;
+use crate::settings::CONFIG;
 use crate::utils::common::BackupMethod;
+use std::path::PathBuf;
 use std::sync::Arc;
-use tracing::info;
+use tracing::{error, info, warn};
 
 pub struct Agent {
     ctx: Arc<Context>,
@@ -17,6 +20,8 @@ pub struct Agent {
     cron_service: CronService,
     backup_service: BackupService,
     restore_service: RestoreService,
+    dashboard_cache: Vec<DatabaseConfig>,
+    cache_path: PathBuf,
 }
 
 impl Agent {
@@ -28,26 +33,43 @@ impl Agent {
         let backup_service = BackupService::new(ctx.clone());
         let restore_service = RestoreService::new(ctx.clone());
 
+        let cache_path = PathBuf::from(&CONFIG.data_path).join("dashboard_databases.json");
+        let dashboard_cache = load_cache(&cache_path);
+
         Agent {
             ctx,
             config_service,
             status_service,
             cron_service,
             backup_service,
             restore_service,
+            dashboard_cache,
+            cache_path,
         }
     }
 
     pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box<dyn std::error::Error>> {
-        let config = self.config_service.load(None)?;
-        let ping_result = self.status_service.ping(&config.databases).await?;
+        let local = self.config_service.load_optional(None);
+
+        let merged_in = merge(&local.databases, &self.dashboard_cache);
+        let ping_result = self.status_service.ping(&merged_in.databases).await?;
+
+        self.dashboard_cache = collect_configs(&ping_result);
+        if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) {
+            error!("Failed to persist dashboard cache: {e}");
+        }
+
+        let merged = merge(&local.databases, &self.dashboard_cache);
 
         for db in ping_result.databases.iter() {
-            let database = config
+            let Some(database) = merged
                 .databases
                 .iter()
                 .find(|cfg_db| cfg_db.generated_id == db.generated_id)
-                .unwrap();
+            else {
+                warn!("No config for returned database {}; skipping", db.generated_id);
+                continue;
+            };
             info!(
                 "Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
                 db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
@@ -59,14 +81,14 @@ impl Agent {
                     .backup_service
                     .dispatch(
                         &db.generated_id,
-                        &config,
+                        &merged,
                         method.clone(),
                         &db.storages,
                         db.encrypt,
                     )
                     .await;
             } else if db.data.restore.action {
-                let _ = self.restore_service.dispatch(db, &config).await;
+                let _ = self.restore_service.dispatch(db, &merged).await;
             }
         }
 

src/main.rs

@@ -22,7 +22,6 @@ async fn main() {
         eprintln!("Failed to clean locks on startup: {:?}", e);
     }
 
-    // Best-effort cleanup of ephemeral helper containers orphaned by a crash.
     match crate::domain::docker_volume::docker::client() {
         Ok(docker) => match crate::domain::docker_volume::docker::sweep_ephemeral(&docker).await {
             Ok(n) if n > 0 => tracing::info!("Removed {n} orphaned ephemeral helper container(s)"),

src/services/api/models/agent/status.rs

@@ -1,5 +1,6 @@
 #![allow(dead_code)]
 
+use crate::services::config::DatabaseConfig;
 use crate::utils::deserializer::{deserialize_snake_case, string_or_number_to_string};
 use serde::{Deserialize, Serialize};
 use toml::Value;
@@ -39,6 +40,13 @@ pub struct DatabaseStatus {
     pub storages_encrypted: Option<bool>,
     #[serde(default)]
     pub storages_ciphertext: Option<String>,
+    #[serde(default)]
+    pub config_encrypted: Option<bool>,
+    #[serde(default)]
+    pub config_ciphertext: Option<String>,
+    /// Filled in memory after decrypting `config_ciphertext`; never on the wire.
+    #[serde(skip)]
+    pub resolved_config: Option<DatabaseConfig>,
     pub encrypt: bool,
     pub data: DatabaseData,
 }

src/services/config.rs

@@ -1,7 +1,7 @@
 #![allow(dead_code)]
 
 use crate::core::context::Context;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use serde_json;
 use std::collections::HashMap;
 use std::fs::File;
@@ -12,7 +12,7 @@ use toml;
 use tracing::info;
 use uuid::Uuid;
 
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename_all = "lowercase")]
 pub enum DbType {
     Mysql,
@@ -49,7 +49,7 @@ impl DbType {
 }
 
 #[allow(dead_code)]
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct DatabaseConfig {
     pub name: String,
     pub database: String,
@@ -68,7 +68,7 @@ pub struct DatabaseConfig {
 }
 
 #[allow(dead_code)]
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct DatabasesConfig {
     pub databases: Vec<DatabaseConfig>,
 }
@@ -98,6 +98,86 @@ pub struct InputDatabasesConfig {
     pub databases: Vec<InputDatabaseConfig>,
 }
 
+fn required<T: Clone>(opt: &Option<T>, db_name: &str, field_name: &str) -> Result<T, String> {
+    match opt {
+        Some(v) => Ok(v.clone()),
+        None => Err(format!(
+            "Missing required field '{}' for database '{}'",
+            field_name, db_name
+        )),
+    }
+}
+
+fn optional<T: Clone + Default>(opt: &Option<T>) -> T {
+    opt.clone().unwrap_or_default()
+}
+
+pub fn build_config(db: InputDatabaseConfig) -> Result<DatabaseConfig, String> {
+    if Uuid::parse_str(&db.generated_id).is_err() {
+        return Err(format!("Invalid UUID for database '{}'", db.name));
+    }
+
+    let username = match db.db_type {
+        DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb
+        | DbType::Mssql => required(&db.username, &db.name, "username")?,
+        _ => optional(&db.username),
+    };
+    let password = match db.db_type {
+        DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb
+        | DbType::Mssql => required(&db.password, &db.name, "password")?,
+        _ => optional(&db.password),
+    };
+    let host = match db.db_type {
+        DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb
+        | DbType::MongoDB | DbType::Redis | DbType::Firebird | DbType::Valkey | DbType::Mssql => {
+            required(&db.host, &db.name, "host")?
+        }
+        DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
+    };
+    let port = match db.db_type {
+        DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb
+        | DbType::MongoDB | DbType::Redis | DbType::Firebird | DbType::Valkey | DbType::Mssql => {
+            required(&db.port, &db.name, "port")?
+        }
+        DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
+    };
+    let database_name = match db.db_type {
+        DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
+            optional(&db.database)
+        }
+        DbType::PostgresqlCluster => db.database.clone().unwrap_or_else(|| "postgres".to_string()),
+        _ => required(&db.database, &db.name, "database")?,
+    };
+    let path_val = match db.db_type {
+        DbType::Sqlite => required(&db.path, &db.name, "path")?,
+        _ => optional(&db.path),
+    };
+    let max_packet_size = match db.db_type {
+        DbType::Mysql | DbType::Mariadb => db.max_packet_size.unwrap_or_else(|| "512M".to_string()),
+        _ => String::new(),
+    };
+    let volume_name = match db.db_type {
+        DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
+        _ => optional(&db.volume_name),
+    };
+
+    Ok(DatabaseConfig {
+        name: db.name,
+        database: database_name,
+        db_type: db.db_type,
+        username,
+        password,
+        host,
+        port,
+        generated_id: db.generated_id,
+        path: path_val,
+        max_packet_size,
+        volume_name,
+        container_name: db.container_name.clone(),
+        options: db.options.unwrap_or_default(),
+    })
+}
+
 pub struct ConfigService {
     ctx: Arc<Context>,
 }
@@ -150,128 +230,21 @@ impl ConfigService {
             _ => return Err("Unsupported config file format. Use .json or .toml".to_string()),
         };
 
-        fn required<T: Clone>(
-            opt: &Option<T>,
-            db_name: &str,
-            field_name: &str,
-        ) -> Result<T, String> {
-            match opt {
-                Some(v) => Ok(v.clone()),
-                None => {
-                    let msg = format!(
-                        "Missing required field '{}' for database '{}'",
-                        field_name, db_name
-                    );
-                    Err(msg)
-                }
-            }
-        }
-
-        fn optional<T: Clone>(opt: &Option<T>) -> T
-        where
-            T: Default,
-        {
-            opt.clone().unwrap_or_default()
-        }
-
         let mut databases = Vec::with_capacity(input_config.databases.len());
-
         for db in input_config.databases {
-            if Uuid::parse_str(&db.generated_id).is_err() {
-                return Err(format!("Invalid UUID for database '{}'", db.name));
-            }
-
-            let username = match db.db_type {
-                DbType::Postgresql
-                | DbType::PostgresqlCluster
-                | DbType::Mysql
-                | DbType::Mariadb
-                | DbType::Mssql => required(&db.username, &db.name, "username")?,
-                _ => optional(&db.username),
-            };
-
-            let password = match db.db_type {
-                DbType::Postgresql
-                | DbType::PostgresqlCluster
-                | DbType::Mysql
-                | DbType::Mariadb
-                | DbType::Mssql => required(&db.password, &db.name, "password")?,
-                _ => optional(&db.password),
-            };
-
-            let host = match db.db_type {
-                DbType::Postgresql
-                | DbType::PostgresqlCluster
-                | DbType::Mysql
-                | DbType::Mariadb
-                | DbType::MongoDB
-                | DbType::Redis
-                | DbType::Firebird
-                | DbType::Valkey
-                | DbType::Mssql => required(&db.host, &db.name, "host")?,
-                DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
-            };
-
-            let port = match db.db_type {
-                DbType::Postgresql
-                | DbType::PostgresqlCluster
-                | DbType::Mysql
-                | DbType::Mariadb
-                | DbType::MongoDB
-                | DbType::Redis
-                | DbType::Firebird
-                | DbType::Valkey
-                | DbType::Mssql => required(&db.port, &db.name, "port")?,
-                DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
-            };
-
-            let database_name = match db.db_type {
-                DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
-                    optional(&db.database)
-                }
-                DbType::PostgresqlCluster => db
-                    .database
-                    .clone()
-                    .unwrap_or_else(|| "postgres".to_string()),
-                _ => required(&db.database, &db.name, "database")?,
-            };
-
-            let path_val = match db.db_type {
-                DbType::Sqlite => required(&db.path, &db.name, "path")?,
-                _ => optional(&db.path),
-            };
-
-            let max_packet_size = match db.db_type {
-                DbType::Mysql | DbType::Mariadb => {
-                    db.max_packet_size.unwrap_or_else(|| "512M".to_string())
-                }
-                _ => String::new(),
-            };
-
-            let volume_name = match db.db_type {
-                DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
-                _ => optional(&db.volume_name),
-            };
-            let container_name = db.container_name.clone();
-
-            databases.push(DatabaseConfig {
-                name: db.name,
-                database: database_name,
-                db_type: db.db_type,
-                username,
-                password,
-                host,
-                port,
-                generated_id: db.generated_id,
-                path: path_val,
-                max_packet_size,
-                volume_name,
-                container_name,
-                options: db.options.unwrap_or_default(),
-            });
+            databases.push(build_config(db)?);
         }
 
         info!("Databases: {} instances loaded", databases.len());
         Ok(DatabasesConfig { databases })
     }
+
+    pub fn load_optional(&self, file_path: Option<&str>) -> DatabasesConfig {
+        self.load(file_path).unwrap_or_else(|e| {
+            tracing::warn!(
+                    "Local databases config unavailable ({e}); continuing with dashboard-defined databases only"
+                );
+            DatabasesConfig { databases: Vec::new() }
+        })
+    }
 }