feat(core)!: Replace YAML config with CLI args and env vars for metadata DB (resolves #1146).

components/core/CMakeLists.txt

@@ -716,6 +716,7 @@ set(SOURCE_FILES_unitTest
         tests/test-ffi_KeyValuePairLogEvent.cpp
         tests/test-ffi_SchemaTree.cpp
         tests/test-FileDescriptorReader.cpp
+        tests/test-GlobalMetadataDBConfig.cpp
         tests/test-Grep.cpp
         tests/test-hash_utils.cpp
         tests/test-ir_encoding_methods.cpp
@@ -774,7 +775,6 @@ if(CLP_BUILD_TESTING)
             ${STD_FS_LIBS}
             clp::regex_utils
             clp::string_utils
-            yaml-cpp
             ystdlib::containers
             ystdlib::error_handling
             ${LIBLZMA_LIBRARIES}

components/core/src/clp/GlobalMetadataDBConfig.cpp

@@ -1,110 +1,97 @@
 #include "GlobalMetadataDBConfig.hpp"
 
+#include <cstdlib>
+#include <stdexcept>
+
 #include <fmt/core.h>
-#include <yaml-cpp/yaml.h>
 
 using std::exception;
 using std::invalid_argument;
 using std::string;
 
-static exception get_yaml_missing_key_exception(string const& key_name) {
-    throw invalid_argument(fmt::format("Missing key '{}'", key_name));
+namespace clp {
+auto GlobalMetadataDBConfig::add_command_line_options(
+        boost::program_options::options_description& options_description
+) -> void {
+    options_description.add_options()
+            ("db-type",
+             boost::program_options::value<string>()->default_value("sqlite"),
+             "Database type [sqlite | mysql]")
+            ("db-host",
+             boost::program_options::value<string>()->default_value("127.0.0.1"),
+             "[db-type=mysql] Database host")
+            ("db-port",
+             boost::program_options::value<int>()->default_value(cDefaultMetadataDbPort),
+             "[db-type=mysql] Database port")
+            ("db-name",
+             boost::program_options::value<string>()->default_value("clp-db"),
+             "[db-type=mysql] Database name")
+            ("db-table-prefix",
+             boost::program_options::value<string>()->default_value("clp_"),
+             "[db-type=mysql] Database table prefix");
 }
 
-static exception
-get_yaml_unconvertable_value_exception(string const& key_name, string const& destination_type) {
-    throw invalid_argument(
-            fmt::format("'{}' could not be converted to type '{}'", key_name, destination_type)
-    );
-}
+auto GlobalMetadataDBConfig::init_from_parsed_options(
+        boost::program_options::variables_map const& parsed_options
+) -> void {
+    if (auto const db_type_string{parsed_options["db-type"].as<string>()};
+        "sqlite" == db_type_string)
+    {
+        m_metadata_db_type = MetadataDBType::SQLite;
+    } else if ("mysql" == db_type_string) {
+        m_metadata_db_type = MetadataDBType::MySQL;
+    } else {
+        throw invalid_argument("Unknown database type: " + db_type_string);
+    }
 
-namespace clp {
-void GlobalMetadataDBConfig::parse_config_file(string const& config_file_path) {
-    YAML::Node config = YAML::LoadFile(config_file_path);
+    if (m_metadata_db_type == MetadataDBType::MySQL) {
+        m_metadata_db_host = parsed_options["db-host"].as<string>();
+        m_metadata_db_port = parsed_options["db-port"].as<int>();
+        m_metadata_db_name = parsed_options["db-name"].as<string>();
+        m_metadata_table_prefix = parsed_options["db-table-prefix"].as<string>();
+    }
+}
 
-    if (!config["type"]) {
-        throw get_yaml_missing_key_exception("type");
+auto GlobalMetadataDBConfig::read_credentials_from_env() -> void {
+    auto const* db_username{std::getenv("CLP_DB_USER")};
+    if (nullptr != db_username) {
+        m_metadata_db_username = db_username;
     }
 
-    auto db_type_string = config["type"].as<string>();
-    if ("sqlite" == db_type_string) {
-        m_metadata_db_type = MetadataDBType::SQLite;
-    } else if ("mysql" == db_type_string) {
-        m_metadata_db_type = MetadataDBType::MySQL;
+    auto const* db_password{std::getenv("CLP_DB_PASS")};
+    if (nullptr != db_password) {
+        m_metadata_db_password = db_password;
+    }
+}
 
-        if (!config["host"]) {
-            throw get_yaml_missing_key_exception("host");
-        }
-        try {
-            m_metadata_db_host = config["host"].as<string>();
-        } catch (YAML::BadConversion& e) {
-            throw get_yaml_unconvertable_value_exception("host", "string");
-        }
+auto GlobalMetadataDBConfig::validate() const -> void {
+    if (m_metadata_db_type == MetadataDBType::MySQL) {
         if (m_metadata_db_host.empty()) {
-            throw invalid_argument("Database 'host' not specified or empty.");
+            throw invalid_argument("Database '--db-host' is empty.");
         }
 
-        if (!config["port"]) {
-            throw get_yaml_missing_key_exception("port");
-        }
-        try {
-            m_metadata_db_port = config["port"].as<int>();
-        } catch (YAML::BadConversion& e) {
-            throw get_yaml_unconvertable_value_exception("port", "int");
-        }
-        if (m_metadata_db_port < 0) {
-            throw invalid_argument("Database 'port' cannot be negative.");
+        if (0 > m_metadata_db_port || 65'535 < m_metadata_db_port) {
+            throw invalid_argument(
+                    "Database '--db-port' is out of range [0, 65535]: "
+                    + std::to_string(m_metadata_db_port)
+            );
         }
 
-        if (!config["name"]) {
-            throw get_yaml_missing_key_exception("name");
-        }
-        try {
-            m_metadata_db_name = config["name"].as<string>();
-        } catch (YAML::BadConversion& e) {
-            throw get_yaml_unconvertable_value_exception("name", "string");
-        }
         if (m_metadata_db_name.empty()) {
-            throw invalid_argument("Database 'name' not specified or empty.");
+            throw invalid_argument("Database '--db-name' is empty.");
         }
 
-        if (!config["username"]) {
-            throw get_yaml_missing_key_exception("username");
-        }
-        try {
-            m_metadata_db_username = config["username"].as<string>();
-        } catch (YAML::BadConversion& e) {
-            throw get_yaml_unconvertable_value_exception("username", "string");
+        if (m_metadata_table_prefix.empty()) {
+            throw invalid_argument("Database '--db-table_prefix' is empty.");
         }
+
         if (m_metadata_db_username.empty()) {
-            throw invalid_argument("Database 'username' not specified or empty.");
+            throw invalid_argument("Database 'CLP_DB_USER' not specified or empty.");
         }
 
-        if (!config["password"]) {
-            throw get_yaml_missing_key_exception("password");
-        }
-        try {
-            m_metadata_db_password = config["password"].as<string>();
-        } catch (YAML::BadConversion& e) {
-            throw get_yaml_unconvertable_value_exception("password", "string");
-        }
         if (m_metadata_db_password.empty()) {
-            throw invalid_argument("Database 'password' not specified or empty.");
-        }
-
-        if (!config["table_prefix"]) {
-            throw get_yaml_missing_key_exception("table_prefix");
+            throw invalid_argument("Database 'CLP_DB_PASS' not specified or empty.");
         }
-        try {
-            m_metadata_table_prefix = config["table_prefix"].as<string>();
-        } catch (YAML::BadConversion& e) {
-            throw get_yaml_unconvertable_value_exception("table_prefix", "string");
-        }
-        if (m_metadata_table_prefix.empty()) {
-            throw invalid_argument("Database 'table_prefix' not specified or empty.");
-        }
-    } else {
-        throw invalid_argument("Unknown type");
     }
 }
 }  // namespace clp

components/core/src/clp/GlobalMetadataDBConfig.hpp

@@ -4,6 +4,8 @@
 #include <cstdint>
 #include <string>
 
+#include <boost/program_options.hpp>
+
 namespace clp {
 /**
  * Class encapsulating the global metadata database's configuration details
@@ -16,11 +18,37 @@ class GlobalMetadataDBConfig {
         MySQL,
     };
 
+    // Static functions
+    /**
+     * Adds database-related command-line options to the given options_description.
+     * @param options_description
+     */
+    static auto add_command_line_options(
+            boost::program_options::options_description& options_description
+    ) -> void;
+
     // Constructors
     GlobalMetadataDBConfig() = default;
 
     // Methods
-    void parse_config_file(std::string const& config_file_path);
+    /**
+     * Initializes the configuration from parsed command-line options.
+     * @param parsed_options
+     */
+    auto init_from_parsed_options(boost::program_options::variables_map const& parsed_options)
+            -> void;
+
+    /**
+     * Reads database credentials from environment variables:
+     * - CLP_DB_USER for database username
+     * - CLP_DB_PASS for database password
+     */
+    auto read_credentials_from_env() -> void;
+
+    /**
+     * Validates that all required parameters are provided
+     */
+    auto validate() const -> void;
 
     [[nodiscard]] auto get_metadata_db_type() const -> MetadataDBType { return m_metadata_db_type; }
 
@@ -52,7 +80,7 @@ class GlobalMetadataDBConfig {
 
     MetadataDBType m_metadata_db_type{MetadataDBType::SQLite};
 
-    std::string m_metadata_db_host{"localhost"};
+    std::string m_metadata_db_host{"127.0.0.1"};
     int m_metadata_db_port{cDefaultMetadataDbPort};
     std::string m_metadata_db_name;
 

components/core/src/clp/clg/CMakeLists.txt

@@ -143,7 +143,6 @@ if(CLP_BUILD_EXECUTABLES)
                 ${sqlite_LIBRARY_DEPENDENCIES}
                 ${STD_FS_LIBS}
                 clp::string_utils
-                yaml-cpp
                 ystdlib::containers
                 ystdlib::error_handling
                 ZStd::ZStd

components/core/src/clp/clg/CommandLineArguments.cpp

@@ -41,21 +41,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) {
         config_file_path += '/';
     }
     config_file_path += cDefaultConfigFilename;
-    string global_metadata_db_config_file_path;
-    options_general.add_options()
-            ("help,h", "Print help")
-            ("version,V", "Print version")
-            (
-                    "config-file",
-                    po::value<string>(&config_file_path)->value_name("FILE")
-                            ->default_value(config_file_path),
-                    "Use configuration options from FILE"
-            )(
-                    "db-config-file",
-                    po::value<string>(&global_metadata_db_config_file_path)->value_name("FILE")
-                            ->default_value(global_metadata_db_config_file_path),
-                    "Global metadata DB YAML config"
-            );
+    options_general.add_options()("help,h", "Print help")("version,V", "Print version")(
+            "config-file",
+            po::value<string>(&config_file_path)
+                    ->value_name("FILE")
+                    ->default_value(config_file_path),
+            "Use configuration options from FILE"
+    );
+    GlobalMetadataDBConfig::add_command_line_options(options_general);
 
     // Define input options
     po::options_description options_input("Input Options");
@@ -195,14 +188,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) {
             return ParsingResult::InfoCommand;
         }
 
-        // Parse and validate global metadata DB config
-        if (false == global_metadata_db_config_file_path.empty()) {
-            try {
-                m_metadata_db_config.parse_config_file(global_metadata_db_config_file_path);
-            } catch (std::exception& e) {
-                SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what());
-                return ParsingResult::Failure;
-            }
+        // Initialize and validate global metadata DB config
+        try {
+            m_metadata_db_config.init_from_parsed_options(parsed_command_line_options);
+            m_metadata_db_config.read_credentials_from_env();
+            m_metadata_db_config.validate();
+        } catch (std::exception& e) {
+            SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what());
+            return ParsingResult::Failure;
         }
 
         // Validate archive path was specified

components/core/src/clp/clp/CMakeLists.txt

@@ -186,7 +186,6 @@ if(CLP_BUILD_EXECUTABLES)
                 nlohmann_json::nlohmann_json
                 ${STD_FS_LIBS}
                 clp::string_utils
-                yaml-cpp
                 ystdlib::containers
                 ystdlib::error_handling
                 ZStd::ZStd

components/core/src/clp/clp/CommandLineArguments.cpp

@@ -41,24 +41,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) {
         config_file_path += '/';
     }
     config_file_path += cDefaultConfigFilename;
-    string global_metadata_db_config_file_path;
-    options_general.add_options()
-            ("help,h", "Print help")
-            ("version,V", "Print version")
-            (
-                    "config-file",
-                    po::value<string>(&config_file_path)
-                            ->value_name("FILE")
-                            ->default_value(config_file_path),
-                    "Use configuration options from FILE"
-            )
-            (
-                    "db-config-file",
-                    po::value<string>(&global_metadata_db_config_file_path)
-                            ->value_name("FILE")
-                            ->default_value(global_metadata_db_config_file_path),
-                    "Global metadata DB YAML config"
-            );
+    options_general.add_options()("help,h", "Print help")("version,V", "Print version")(
+            "config-file",
+            po::value<string>(&config_file_path)
+                    ->value_name("FILE")
+                    ->default_value(config_file_path),
+            "Use configuration options from FILE"
+    );
+    GlobalMetadataDBConfig::add_command_line_options(options_general);
 
     // Define functional options
     po::options_description options_functional("Input Options");
@@ -121,14 +111,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) {
             return ParsingResult::InfoCommand;
         }
 
-        // Parse and validate global metadata DB config
-        if (false == global_metadata_db_config_file_path.empty()) {
-            try {
-                m_metadata_db_config.parse_config_file(global_metadata_db_config_file_path);
-            } catch (std::exception& e) {
-                SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what());
-                return ParsingResult::Failure;
-            }
+        // Initialize and validate global metadata DB config
+        try {
+            m_metadata_db_config.init_from_parsed_options(parsed_command_line_options);
+            m_metadata_db_config.read_credentials_from_env();
+            m_metadata_db_config.validate();
+        } catch (std::exception& e) {
+            SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what());
+            return ParsingResult::Failure;
         }
 
         // Validate command

components/core/src/clp_s/indexer/CMakeLists.txt

@@ -95,7 +95,6 @@ if(CLP_BUILD_EXECUTABLES)
                 OpenSSL::Crypto
                 simdjson::simdjson
                 spdlog::spdlog
-                yaml-cpp
                 ystdlib::containers
                 ZStd::ZStd
         )