Add a Vacuum option ton SQLite connection builder

to limit duplication between aggregator & signer

Author: Alenar

internal/mithril-persistence/src/sqlite/connection_builder.rs

@@ -2,12 +2,13 @@ use std::ops::Not;
 use std::path::{Path, PathBuf};
 
 use anyhow::Context;
-use slog::Logger;
+use slog::{info, Logger};
 use sqlite::{Connection, ConnectionThreadSafe};
 
 use mithril_common::StdResult;
 
 use crate::database::{ApplicationNodeType, DatabaseVersionChecker, SqlMigration};
+use crate::sqlite::vacuum_database;
 
 /// Builder of SQLite connection
 pub struct ConnectionBuilder {
@@ -31,6 +32,11 @@ pub enum ConnectionOptions {
     ///
     /// This option take priority over [ConnectionOptions::EnableForeignKeys] if both are enabled.
     ForceDisableForeignKeys,
+
+    /// Run a VACUUM operation on the database after the connection is opened
+    ///
+    /// ⚠ This operation can be very slow on large databases ⚠
+    Vacuum,
 }
 
 impl ConnectionBuilder {
@@ -86,6 +92,23 @@ impl ConnectionBuilder {
                 )
             })?;
 
+        if self.options.contains(&ConnectionOptions::Vacuum) {
+            info!(
+                self.logger,
+                "Vacuuming SQLite database, this may take a while...";
+                "database" => self.connection_path.display()
+            );
+
+            vacuum_database(&connection)
+                .with_context(|| "SQLite initialization: database VACUUM error")?;
+
+            info!(
+                self.logger,
+                "SQLite database vacuumed successfully";
+                "database" => self.connection_path.display()
+            );
+        }
+
         if self
             .options
             .contains(&ConnectionOptions::EnableWriteAheadLog)
@@ -130,11 +153,14 @@ impl ConnectionBuilder {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use crate::sqlite::ConnectionOptions::ForceDisableForeignKeys;
-    use mithril_common::test_utils::TempDir;
     use sqlite::Value;
 
+    use mithril_common::test_utils::TempDir;
+
+    use crate::sqlite::ConnectionOptions::ForceDisableForeignKeys;
+
+    use super::*;
+
     // see: https://www.sqlite.org/pragma.html#pragma_journal_mode
     const DEFAULT_SQLITE_JOURNAL_MODE: &str = "delete";
     // see: https://www.sqlite.org/pragma.html#pragma_synchronous

mithril-aggregator/src/dependency_injection/builder.rs

@@ -1,6 +1,6 @@
 use anyhow::Context;
 use semver::Version;
-use slog::{info, Logger};
+use slog::Logger;
 use std::sync::Arc;
 use tokio::{
     sync::{
@@ -40,10 +40,7 @@ use mithril_common::{
 };
 use mithril_persistence::{
     database::{repository::CardanoTransactionRepository, ApplicationNodeType, SqlMigration},
-    sqlite::{
-        vacuum_database, ConnectionBuilder, ConnectionOptions, SqliteConnection,
-        SqliteConnectionPool,
-    },
+    sqlite::{ConnectionBuilder, ConnectionOptions, SqliteConnection, SqliteConnectionPool},
     store::adapter::{MemoryAdapter, SQLiteAdapter, StoreAdapter},
 };
 
@@ -286,6 +283,7 @@ impl DependenciesBuilder {
         migrations: Vec<SqlMigration>,
         do_vacuum_database: bool,
     ) -> Result<SqliteConnection> {
+        let logger = self.get_logger()?;
         let connection_builder = match self.configuration.environment {
             ExecutionEnvironment::Production => ConnectionBuilder::open_file(
                 &self.configuration.get_sqlite_dir().join(sqlite_file_name),
@@ -301,13 +299,21 @@ impl DependenciesBuilder {
             ),
         };
 
-        let logger = self.get_logger()?;
-        let connection = connection_builder
-            .with_node_type(ApplicationNodeType::Aggregator)
-            .with_options(&[
+        let connection_options = {
+            let mut options = vec![
                 ConnectionOptions::EnableForeignKeys,
                 ConnectionOptions::EnableWriteAheadLog,
-            ])
+            ];
+            if do_vacuum_database {
+                options.push(ConnectionOptions::Vacuum);
+            }
+
+            options
+        };
+
+        let connection = connection_builder
+            .with_node_type(ApplicationNodeType::Aggregator)
+            .with_options(&connection_options)
             .with_logger(logger.clone())
             .with_migrations(migrations)
             .build()
@@ -316,16 +322,6 @@ impl DependenciesBuilder {
                 error: Some(e),
             })?;
 
-        if do_vacuum_database {
-            info!(
-                logger,
-                "Vacuuming '{sqlite_file_name}', this may take a while..."
-            );
-            vacuum_database(&connection)
-                .with_context(|| "SQLite initialization: database vacuum error")?;
-            info!(logger, "SQLite '{sqlite_file_name}' vacuumed successfully");
-        }
-
         Ok(connection)
     }
 

mithril-signer/src/runtime/signer_services.rs

@@ -1,6 +1,5 @@
 use anyhow::{anyhow, Context};
 use async_trait::async_trait;
-use slog::info;
 use std::{fs, sync::Arc, time::Duration};
 
 use mithril_common::{
@@ -25,7 +24,7 @@ use mithril_common::{
 };
 use mithril_persistence::{
     database::{repository::CardanoTransactionRepository, ApplicationNodeType, SqlMigration},
-    sqlite::{vacuum_database, ConnectionBuilder, SqliteConnection, SqliteConnectionPool},
+    sqlite::{ConnectionBuilder, ConnectionOptions, SqliteConnection, SqliteConnectionPool},
     store::{adapter::SQLiteAdapter, StakeStore},
 };
 
@@ -173,19 +172,12 @@ impl<'a> ProductionServiceBuilder<'a> {
         let logger = slog_scope::logger();
         let connection = ConnectionBuilder::open_file(&sqlite_db_path)
             .with_node_type(ApplicationNodeType::Signer)
+            .with_options(&[ConnectionOptions::Vacuum])
             .with_migrations(migrations)
             .with_logger(logger.clone())
             .build()
             .with_context(|| "Database connection initialisation error")?;
 
-        info!(
-            logger,
-            "Vacuuming '{sqlite_file_name}', this may take a while..."
-        );
-        vacuum_database(&connection)
-            .with_context(|| "SQLite initialization: database vacuum error")?;
-        info!(logger, "SQLite '{sqlite_file_name}' vacuumed successfully");
-
         Ok(connection)
     }
 }