From 21944b9eb74fc4d388c35f92cad5df728fd9b2b1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 31 Jul 2025 19:45:36 -0400 Subject: [PATCH 001/108] feat(metadata-db): Replace YAML config with env vars and CLI args (resolves #1146). --- components/core/CMakeLists.txt | 2 +- components/core/config/metadata-db.yml | 10 -- .../core/src/clp/GlobalMetadataDBConfig.cpp | 141 ++++++++---------- .../core/src/clp/GlobalMetadataDBConfig.hpp | 32 +++- components/core/src/clp/clg/CMakeLists.txt | 1 - .../core/src/clp/clg/CommandLineArguments.cpp | 39 ++--- components/core/src/clp/clp/CMakeLists.txt | 1 - .../core/src/clp/clp/CommandLineArguments.cpp | 42 ++---- .../core/src/clp_s/indexer/CMakeLists.txt | 1 - .../clp_s/indexer/CommandLineArguments.cpp | 45 +++--- .../tests/test-GlobalMetadataDBConfig.cpp | 136 +++++++++++++++++ components/core/tools/scripts/db/init-db.py | 42 +++--- .../executor/compress/compression_task.py | 62 ++++++-- docs/src/user-guide/core-unstructured/clp.md | 26 ++-- 14 files changed, 366 insertions(+), 214 deletions(-) delete mode 100644 components/core/config/metadata-db.yml create mode 100644 components/core/tests/test-GlobalMetadataDBConfig.cpp diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 7c2f408a02..488fa56d88 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -716,6 +716,7 @@ set(SOURCE_FILES_unitTest tests/test-ffi_KeyValuePairLogEvent.cpp tests/test-ffi_SchemaTree.cpp tests/test-FileDescriptorReader.cpp + tests/test-GlobalMetadataDBConfig.cpp tests/test-Grep.cpp tests/test-hash_utils.cpp tests/test-ir_encoding_methods.cpp @@ -774,7 +775,6 @@ if(CLP_BUILD_TESTING) ${STD_FS_LIBS} clp::regex_utils clp::string_utils - yaml-cpp ystdlib::containers ystdlib::error_handling ${LIBLZMA_LIBRARIES} diff --git a/components/core/config/metadata-db.yml b/components/core/config/metadata-db.yml deleted file mode 100644 index 76eec2780f..0000000000 --- a/components/core/config/metadata-db.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Database type [sqlite | mysql] -type: "sqlite" - -# MySQL configuration -#host: localhost -#port: 3306 -#name: clp_metadata -#username: ... -#password: ... -#table_prefix: "clp_" diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 833c72d6df..30b667540e 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -1,110 +1,95 @@ #include "GlobalMetadataDBConfig.hpp" +#include +#include + #include -#include using std::exception; using std::invalid_argument; using std::string; -static exception get_yaml_missing_key_exception(string const& key_name) { - throw invalid_argument(fmt::format("Missing key '{}'", key_name)); +namespace clp { +auto GlobalMetadataDBConfig::add_command_line_options( + boost::program_options::options_description& options_description +) -> void { + options_description.add_options() + ("db-type", + boost::program_options::value()->default_value("sqlite"), + "Database type [sqlite | mysql]") + ("db-host", + boost::program_options::value()->default_value("127.0.0.1"), + "[db-type=mysql] Database host") + ("db-port", + boost::program_options::value()->default_value(cDefaultMetadataDbPort), + "[db-type=mysql] Database port") + ("db-name", + boost::program_options::value()->default_value("clp-db"), + "[db-type=mysql] Database name") + ("db-table-prefix", + boost::program_options::value()->default_value("clp_"), + "[db-type=mysql] Database table prefix"); } -static exception -get_yaml_unconvertable_value_exception(string const& key_name, string const& destination_type) { - throw invalid_argument( - fmt::format("'{}' could not be converted to type '{}'", key_name, destination_type) - ); -} +auto GlobalMetadataDBConfig::init_from_parsed_options( + boost::program_options::variables_map const& parsed_options +) -> void { + if (auto const db_type_string{parsed_options["db-type"].as()}; + "sqlite" == db_type_string) + { + m_metadata_db_type = MetadataDBType::SQLite; + } else if ("mysql" == db_type_string) { + m_metadata_db_type = MetadataDBType::MySQL; + } else { + throw invalid_argument("Unknown database type: " + db_type_string); + } -namespace clp { -void GlobalMetadataDBConfig::parse_config_file(string const& config_file_path) { - YAML::Node config = YAML::LoadFile(config_file_path); + if (m_metadata_db_type == MetadataDBType::MySQL) { + m_metadata_db_host = parsed_options["db-host"].as(); + m_metadata_db_port = parsed_options["db-port"].as(); + m_metadata_db_name = parsed_options["db-name"].as(); + m_metadata_table_prefix = parsed_options["db-table-prefix"].as(); + } +} - if (!config["type"]) { - throw get_yaml_missing_key_exception("type"); +auto GlobalMetadataDBConfig::read_credentials_from_env() -> void { + auto const* db_username{std::getenv("CLP_DB_USER")}; + if (nullptr != db_username) { + m_metadata_db_username = db_username; } - auto db_type_string = config["type"].as(); - if ("sqlite" == db_type_string) { - m_metadata_db_type = MetadataDBType::SQLite; - } else if ("mysql" == db_type_string) { - m_metadata_db_type = MetadataDBType::MySQL; + auto const* db_password{std::getenv("CLP_DB_PASS")}; + if (nullptr != db_password) { + m_metadata_db_password = db_password; + } +} - if (!config["host"]) { - throw get_yaml_missing_key_exception("host"); - } - try { - m_metadata_db_host = config["host"].as(); - } catch (YAML::BadConversion& e) { - throw get_yaml_unconvertable_value_exception("host", "string"); - } +auto GlobalMetadataDBConfig::validate() const -> void { + if (m_metadata_db_type == MetadataDBType::MySQL) { if (m_metadata_db_host.empty()) { - throw invalid_argument("Database 'host' not specified or empty."); + throw invalid_argument("Database '--db-host' is empty."); } - if (!config["port"]) { - throw get_yaml_missing_key_exception("port"); - } - try { - m_metadata_db_port = config["port"].as(); - } catch (YAML::BadConversion& e) { - throw get_yaml_unconvertable_value_exception("port", "int"); - } - if (m_metadata_db_port < 0) { - throw invalid_argument("Database 'port' cannot be negative."); + if (0 > m_metadata_db_port || 65535 < m_metadata_db_port) { + throw invalid_argument("Database '--db-port' is out of range [0, 65535]: " + + std::to_string(m_metadata_db_port)); } - if (!config["name"]) { - throw get_yaml_missing_key_exception("name"); - } - try { - m_metadata_db_name = config["name"].as(); - } catch (YAML::BadConversion& e) { - throw get_yaml_unconvertable_value_exception("name", "string"); - } if (m_metadata_db_name.empty()) { - throw invalid_argument("Database 'name' not specified or empty."); + throw invalid_argument("Database '--db--name' is empty."); } - if (!config["username"]) { - throw get_yaml_missing_key_exception("username"); - } - try { - m_metadata_db_username = config["username"].as(); - } catch (YAML::BadConversion& e) { - throw get_yaml_unconvertable_value_exception("username", "string"); + if (m_metadata_table_prefix.empty()) { + throw invalid_argument("Database '--db-table_prefix' is empty."); } + if (m_metadata_db_username.empty()) { - throw invalid_argument("Database 'username' not specified or empty."); + throw invalid_argument("Database 'CLP_DB_USER' not specified or empty."); } - if (!config["password"]) { - throw get_yaml_missing_key_exception("password"); - } - try { - m_metadata_db_password = config["password"].as(); - } catch (YAML::BadConversion& e) { - throw get_yaml_unconvertable_value_exception("password", "string"); - } if (m_metadata_db_password.empty()) { - throw invalid_argument("Database 'password' not specified or empty."); - } - - if (!config["table_prefix"]) { - throw get_yaml_missing_key_exception("table_prefix"); + throw invalid_argument("Database 'CLP_DB_PASS' not specified or empty."); } - try { - m_metadata_table_prefix = config["table_prefix"].as(); - } catch (YAML::BadConversion& e) { - throw get_yaml_unconvertable_value_exception("table_prefix", "string"); - } - if (m_metadata_table_prefix.empty()) { - throw invalid_argument("Database 'table_prefix' not specified or empty."); - } - } else { - throw invalid_argument("Unknown type"); } } } // namespace clp diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index 6e304ca624..76e8d7b69f 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -4,6 +4,8 @@ #include #include +#include + namespace clp { /** * Class encapsulating the global metadata database's configuration details @@ -16,11 +18,37 @@ class GlobalMetadataDBConfig { MySQL, }; + // Static functions + /** + * Adds database-related command-line options to the given options_description. + * @param options_description + */ + static auto add_command_line_options( + boost::program_options::options_description& options_description + ) -> void; + // Constructors GlobalMetadataDBConfig() = default; // Methods - void parse_config_file(std::string const& config_file_path); + /** + * Initializes the configuration from parsed command-line options. + * @param parsed_options + */ + auto init_from_parsed_options(boost::program_options::variables_map const& parsed_options) + -> void; + + /** + * Reads database credentials from environment variables: + * - CLP_DB_USER for database username + * - CLP_DB_PASS for database password + */ + auto read_credentials_from_env() -> void; + + /** + * Validates that all required parameters are provided + */ + auto validate() const -> void; [[nodiscard]] auto get_metadata_db_type() const -> MetadataDBType { return m_metadata_db_type; } @@ -52,7 +80,7 @@ class GlobalMetadataDBConfig { MetadataDBType m_metadata_db_type{MetadataDBType::SQLite}; - std::string m_metadata_db_host{"localhost"}; + std::string m_metadata_db_host{"127.0.0.1"}; int m_metadata_db_port{cDefaultMetadataDbPort}; std::string m_metadata_db_name; diff --git a/components/core/src/clp/clg/CMakeLists.txt b/components/core/src/clp/clg/CMakeLists.txt index 816173bdb0..889525db4f 100644 --- a/components/core/src/clp/clg/CMakeLists.txt +++ b/components/core/src/clp/clg/CMakeLists.txt @@ -143,7 +143,6 @@ if(CLP_BUILD_EXECUTABLES) ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} clp::string_utils - yaml-cpp ystdlib::containers ystdlib::error_handling ZStd::ZStd diff --git a/components/core/src/clp/clg/CommandLineArguments.cpp b/components/core/src/clp/clg/CommandLineArguments.cpp index 3cd58dc9f0..7ed035b48a 100644 --- a/components/core/src/clp/clg/CommandLineArguments.cpp +++ b/components/core/src/clp/clg/CommandLineArguments.cpp @@ -41,21 +41,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { config_file_path += '/'; } config_file_path += cDefaultConfigFilename; - string global_metadata_db_config_file_path; - options_general.add_options() - ("help,h", "Print help") - ("version,V", "Print version") - ( - "config-file", - po::value(&config_file_path)->value_name("FILE") - ->default_value(config_file_path), - "Use configuration options from FILE" - )( - "db-config-file", - po::value(&global_metadata_db_config_file_path)->value_name("FILE") - ->default_value(global_metadata_db_config_file_path), - "Global metadata DB YAML config" - ); + options_general.add_options()("help,h", "Print help")("version,V", "Print version")( + "config-file", + po::value(&config_file_path) + ->value_name("FILE") + ->default_value(config_file_path), + "Use configuration options from FILE" + ); + GlobalMetadataDBConfig::add_command_line_options(options_general); // Define input options po::options_description options_input("Input Options"); @@ -195,14 +188,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { return ParsingResult::InfoCommand; } - // Parse and validate global metadata DB config - if (false == global_metadata_db_config_file_path.empty()) { - try { - m_metadata_db_config.parse_config_file(global_metadata_db_config_file_path); - } catch (std::exception& e) { - SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what()); - return ParsingResult::Failure; - } + // Initialize and validate global metadata DB config + try { + m_metadata_db_config.init_from_parsed_options(parsed_command_line_options); + m_metadata_db_config.read_credentials_from_env(); + m_metadata_db_config.validate(); + } catch (std::exception& e) { + SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what()); + return ParsingResult::Failure; } // Validate archive path was specified diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index e9b35c296e..092ce83991 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -186,7 +186,6 @@ if(CLP_BUILD_EXECUTABLES) nlohmann_json::nlohmann_json ${STD_FS_LIBS} clp::string_utils - yaml-cpp ystdlib::containers ystdlib::error_handling ZStd::ZStd diff --git a/components/core/src/clp/clp/CommandLineArguments.cpp b/components/core/src/clp/clp/CommandLineArguments.cpp index e5dd6c1110..c87a2c153c 100644 --- a/components/core/src/clp/clp/CommandLineArguments.cpp +++ b/components/core/src/clp/clp/CommandLineArguments.cpp @@ -41,24 +41,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { config_file_path += '/'; } config_file_path += cDefaultConfigFilename; - string global_metadata_db_config_file_path; - options_general.add_options() - ("help,h", "Print help") - ("version,V", "Print version") - ( - "config-file", - po::value(&config_file_path) - ->value_name("FILE") - ->default_value(config_file_path), - "Use configuration options from FILE" - ) - ( - "db-config-file", - po::value(&global_metadata_db_config_file_path) - ->value_name("FILE") - ->default_value(global_metadata_db_config_file_path), - "Global metadata DB YAML config" - ); + options_general.add_options()("help,h", "Print help")("version,V", "Print version")( + "config-file", + po::value(&config_file_path) + ->value_name("FILE") + ->default_value(config_file_path), + "Use configuration options from FILE" + ); + GlobalMetadataDBConfig::add_command_line_options(options_general); // Define functional options po::options_description options_functional("Input Options"); @@ -121,14 +111,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { return ParsingResult::InfoCommand; } - // Parse and validate global metadata DB config - if (false == global_metadata_db_config_file_path.empty()) { - try { - m_metadata_db_config.parse_config_file(global_metadata_db_config_file_path); - } catch (std::exception& e) { - SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what()); - return ParsingResult::Failure; - } + // Initialize and validate global metadata DB config + try { + m_metadata_db_config.init_from_parsed_options(parsed_command_line_options); + m_metadata_db_config.read_credentials_from_env(); + m_metadata_db_config.validate(); + } catch (std::exception& e) { + SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what()); + return ParsingResult::Failure; } // Validate command diff --git a/components/core/src/clp_s/indexer/CMakeLists.txt b/components/core/src/clp_s/indexer/CMakeLists.txt index 20e9e9a1e0..6682a5e133 100644 --- a/components/core/src/clp_s/indexer/CMakeLists.txt +++ b/components/core/src/clp_s/indexer/CMakeLists.txt @@ -95,7 +95,6 @@ if(CLP_BUILD_EXECUTABLES) OpenSSL::Crypto simdjson::simdjson spdlog::spdlog - yaml-cpp ystdlib::containers ZStd::ZStd ) diff --git a/components/core/src/clp_s/indexer/CommandLineArguments.cpp b/components/core/src/clp_s/indexer/CommandLineArguments.cpp index da94ef0c30..b00f2d5cd5 100644 --- a/components/core/src/clp_s/indexer/CommandLineArguments.cpp +++ b/components/core/src/clp_s/indexer/CommandLineArguments.cpp @@ -22,19 +22,14 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { // Define output options po::options_description output_options("Output Options"); - std::string metadata_db_config_file_path; // clang-format off output_options.add_options()( - "db-config-file", - po::value(&metadata_db_config_file_path)->value_name("FILE") - ->default_value(metadata_db_config_file_path), - "Path to the YAML DB config file for metadata storage" - )( "create-table", po::bool_switch(&m_should_create_table), "Create the column metadata table if it doesn't exist" ); // clang-format on + clp::GlobalMetadataDBConfig::add_command_line_options(output_options); // Define visible options po::options_description visible_options; @@ -96,27 +91,29 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { throw std::invalid_argument("Archive path not specified or empty."); } m_archive_path = get_path_object_for_raw_path(archive_path); - if (false == metadata_db_config_file_path.empty()) { - clp::GlobalMetadataDBConfig metadata_db_config; - try { - metadata_db_config.parse_config_file(metadata_db_config_file_path); - } catch (std::exception& e) { - SPDLOG_ERROR("Failed to validate metadata database config - {}.", e.what()); - return ParsingResult::Failure; - } - if (clp::GlobalMetadataDBConfig::MetadataDBType::MySQL - != metadata_db_config.get_metadata_db_type()) - { - SPDLOG_ERROR( - "Invalid metadata database type for {}; only supported type is MySQL.", - m_program_name - ); - return ParsingResult::Failure; - } + // Initialize and validate global metadata DB config + clp::GlobalMetadataDBConfig metadata_db_config; + try { + metadata_db_config.init_from_parsed_options(parsed_command_line_options); + metadata_db_config.read_credentials_from_env(); + metadata_db_config.validate(); + } catch (std::exception& e) { + SPDLOG_ERROR("Failed to validate metadata database config - {}.", e.what()); + return ParsingResult::Failure; + } - m_metadata_db_config = std::move(metadata_db_config); + if (clp::GlobalMetadataDBConfig::MetadataDBType::MySQL + != metadata_db_config.get_metadata_db_type()) + { + SPDLOG_ERROR( + "Invalid metadata database type for {}; only supported type is MySQL.", + m_program_name + ); + return ParsingResult::Failure; } + + m_metadata_db_config = std::move(metadata_db_config); } catch (std::exception& e) { SPDLOG_ERROR("{}", e.what()); print_basic_usage(); diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp new file mode 100644 index 0000000000..5b6c3d7a18 --- /dev/null +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -0,0 +1,136 @@ +#include + +#include + +#include "../src/clp/GlobalMetadataDBConfig.hpp" + +using namespace clp; + +namespace { +void set_env_var(char const* name, char const* value) { + setenv(name, value, 1); +} + +void unset_env_var(char const* name) { + unsetenv(name); +} + +template +auto parse_args(std::array const& argv) -> boost::program_options::variables_map { + constexpr auto argc{static_cast(N)}; + boost::program_options::options_description options_desc; + GlobalMetadataDBConfig::add_command_line_options(options_desc); + + boost::program_options::variables_map vm; + boost::program_options::store( + boost::program_options::parse_command_line( + argc, + const_cast(argv.data()), + options_desc + ), + vm + ); + boost::program_options::notify(vm); + + return vm; +} +} // namespace + +TEST_CASE( + "Test adding command line options for GlobalMetadataDBConfig", + "[GlobalMetadataDBConfig]" +) { + constexpr std::array argv{ + "test", + "--db-type", + "mysql", + "--db-host", + "test-host", + "--db-port", + "8888", + "--db-name", + "test-db", + "--db-table-prefix", + "test_prefix_" + }; + auto const vm = parse_args(argv); + + GlobalMetadataDBConfig config; + config.init_from_parsed_options(vm); + + REQUIRE(config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL); + REQUIRE(config.get_metadata_db_host() == "test-host"); + REQUIRE(config.get_metadata_db_port() == 8888); + REQUIRE(config.get_metadata_db_name() == "test-db"); + REQUIRE(config.get_metadata_table_prefix() == "test_prefix_"); +} + +TEST_CASE("Test reading credentials from environment variables", "[GlobalMetadataDBConfig]") { + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", "test-pass"); + + GlobalMetadataDBConfig config; + config.read_credentials_from_env(); + + REQUIRE(config.get_metadata_db_username() == "test-user"); + REQUIRE(config.get_metadata_db_password() == "test-pass"); + + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); +} + +TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBConfig]") { + constexpr std::array argv{ + "test", + "--db-type", + "mysql", + "--db-host", + "test-host", + "--db-port", + "8888", + "--db-name", + "test-db", + "--db-table-prefix", + "test_prefix_" + }; + auto const vm = parse_args(argv); + + GlobalMetadataDBConfig config; + config.init_from_parsed_options(vm); + + SECTION("With valid credentials") { + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", "test-pass"); + config.read_credentials_from_env(); + REQUIRE_NOTHROW(config.validate()); + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); + } + + SECTION("With missing credentials") { + // Neither username nor password set + SECTION("Neither username nor password") { + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + } + SECTION("Username set but password missing") { + set_env_var("CLP_DB_USER", "test-user"); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + unset_env_var("CLP_DB_USER"); + } + SECTION("Password set but username missing") { + set_env_var("CLP_DB_PASS", "test-pass"); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + unset_env_var("CLP_DB_PASS"); + } + } +} + +TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { + constexpr std::array argv{"test", "--db-type", "sqlite"}; + auto const vm = parse_args(argv); + + GlobalMetadataDBConfig config; + config.init_from_parsed_options(vm); + + REQUIRE_NOTHROW(config.validate()); +} diff --git a/components/core/tools/scripts/db/init-db.py b/components/core/tools/scripts/db/init-db.py index ccd81115f9..a4abc628f2 100644 --- a/components/core/tools/scripts/db/init-db.py +++ b/components/core/tools/scripts/db/init-db.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 import argparse import logging +import os + import mariadb -import yaml import sys # Setup logging @@ -18,30 +19,27 @@ def main(argv): args_parser = argparse.ArgumentParser(description="Setup a global MySQL metadata database for CLP.") - args_parser.add_argument("--config-file", required=True, help="Metadata database basic config file.") - parsed_args = args_parser.parse_args(argv[1:]) - - config_file_path = parsed_args.config_file + args_parser.add_argument( "--db-host", default="127.0.0.1", help="Database host" ) + args_parser.add_argument( "--db-port", type=int, default=3306, help="Database port" ) + args_parser.add_argument( "--db-name", default="clp-db", help="Database name" ) + args_parser.add_argument( "--db-table-prefix", default="clp_", help="Database table prefix" ) - with open(config_file_path, 'r') as f: - config = yaml.safe_load(f) - if config is None: - raise Exception(f"Unable to parse configuration from {config_file_path}.") + parsed_args = args_parser.parse_args(argv[1:]) - required_keys = ["host", "port", "username", "password", "name"] - for key in required_keys: - if key not in config: - raise Exception(f"'{key}' missing from config file.") + host = parsed_args.db_host + port = parsed_args.db_port + db_name = parsed_args.db_name + table_prefix = parsed_args.db_table_prefix - host = config["host"] - port = config["port"] - username = config["username"] - password = config["password"] - db_name = config["name"] - table_prefix = config["table_prefix"] + try: + username = os.environ["CLP_DB_USER"] + password = os.environ["CLP_DB_PASS"] + except KeyError as e: + key = e.args[0] + raise Exception(f"'{key}' is missing from environment variables.") try: - mysql_conn = mariadb.connect(host=host, port=port, username=username, password=password) + mysql_conn = mariadb.connect(host=host, port=port, username=username, password=password, ) mysql_cursor = mysql_conn.cursor() except mariadb.Error as err: logger.error("Failed to connect - {}".format(err.msg)) @@ -50,14 +48,14 @@ def main(argv): try: # Create database try: - mysql_cursor.execute("CREATE DATABASE IF NOT EXISTS {} DEFAULT CHARACTER SET 'utf8'".format(db_name)) + mysql_cursor.execute("CREATE DATABASE IF NOT EXISTS `{}` DEFAULT CHARACTER SET 'utf8'".format(db_name)) except mariadb.Error as err: logger.error("Failed to create database - {}".format(err.msg)) return -1 # Use database try: - mysql_cursor.execute("USE {}".format(db_name)) + mysql_cursor.execute("USE `{}`".format(db_name)) except mariadb.Error as err: logger.error("Failed to use database - {}".format(err.msg)) return -1 diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index f5d23f4c74..409e0a9d14 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -6,7 +6,6 @@ from contextlib import closing from typing import Any, Dict, List, Optional, Tuple -import yaml from celery.app.task import Task from celery.utils.log import get_task_logger from clp_py_utils.clp_config import ( @@ -183,18 +182,47 @@ def _upload_archive_to_s3( s3_put(s3_config, archive_src_path, dest_path) +def _get_db_connection_args( + clp_metadata_db_connection_config: Dict[str, Any] +) -> List[str]: + """ + :param clp_metadata_db_connection_config: + :return: List of database connection arguments for the clp command. + """ + return [ + "--db-type", clp_metadata_db_connection_config["type"], + "--db-host", clp_metadata_db_connection_config["host"], + "--db-port", str(clp_metadata_db_connection_config["port"]), + "--db-name", clp_metadata_db_connection_config["name"], + "--db-table-prefix", clp_metadata_db_connection_config["table_prefix"], + ] + + +def _get_db_connection_env_vars( + clp_metadata_db_connection_config: Dict[str, Any] +) -> Dict[str, str]: + """ + :param clp_metadata_db_connection_config: + :return: Dictionary of database connection environment variables for the clp command. + """ + return { + "CLP_DB_USER": clp_metadata_db_connection_config["username"], + "CLP_DB_PASS": clp_metadata_db_connection_config["password"], + } + + def _make_clp_command_and_env( clp_home: pathlib.Path, archive_output_dir: pathlib.Path, clp_config: ClpIoConfig, - db_config_file_path: pathlib.Path, + clp_metadata_db_connection_config: Dict[str, Any], ) -> Tuple[List[str], Optional[Dict[str, str]]]: """ Generates the command and environment variables for a clp compression job. :param clp_home: :param archive_output_dir: :param clp_config: - :param db_config_file_path: + :param clp_metadata_db_connection_config: :return: Tuple of (compression_command, compression_env_vars) """ @@ -209,7 +237,6 @@ def _make_clp_command_and_env( "--target-segment-size", str(clp_config.output.target_segment_size), "--target-encoded-file-size", str(clp_config.output.target_encoded_file_size), "--compression-level", str(clp_config.output.compression_level), - "--db-config-file", str(db_config_file_path), ] # fmt: on if path_prefix_to_remove: @@ -222,7 +249,12 @@ def _make_clp_command_and_env( compression_cmd.append("--schema-path") compression_cmd.append(str(schema_path)) - return compression_cmd, None + # Set database connection parameters + compression_cmd.extend(_get_db_connection_args(clp_metadata_db_connection_config)) + compression_env_vars = dict(os.environ) + compression_env_vars.update(_get_db_connection_env_vars(clp_metadata_db_connection_config)) + + return compression_cmd, compression_env_vars def _make_clp_s_command_and_env( @@ -302,12 +334,6 @@ def run_clp( data_dir = worker_config.data_directory archive_output_dir = worker_config.archive_output.get_directory() - # Generate database config file for clp - db_config_file_path = data_dir / f"{instance_id_str}-db-config.yml" - db_config_file = open(db_config_file_path, "w") - yaml.safe_dump(clp_metadata_db_connection_config, db_config_file) - db_config_file.close() - # Get S3 config s3_config: S3Config enable_s3_write = False @@ -321,14 +347,13 @@ def run_clp( s3_config = worker_config.archive_output.storage.s3_config enable_s3_write = True - table_prefix = clp_metadata_db_connection_config["table_prefix"] dataset = clp_config.input.dataset if StorageEngine.CLP == clp_storage_engine: compression_cmd, compression_env = _make_clp_command_and_env( clp_home=clp_home, archive_output_dir=archive_output_dir, clp_config=clp_config, - db_config_file_path=db_config_file_path, + clp_metadata_db_connection_config=clp_metadata_db_connection_config, ) elif StorageEngine.CLP_S == clp_storage_engine: archive_output_dir = archive_output_dir / dataset @@ -403,6 +428,7 @@ def run_clp( # which case the process will still return success. proc.terminate() + table_prefix = clp_metadata_db_connection_config["table_prefix"] if s3_error is None: # We've started a new archive so add the previous archive's last reported size to # the total @@ -428,17 +454,22 @@ def run_clp( if StorageEngine.CLP_S == clp_storage_engine: indexer_cmd = [ str(clp_home / "bin" / "indexer"), - "--db-config-file", - str(db_config_file_path), + *_get_db_connection_args(clp_metadata_db_connection_config), dataset, archive_path, ] + + # Set database credentials in environment variables for indexer + indexer_env = dict(os.environ) + indexer_env.update(_get_db_connection_env_vars(clp_metadata_db_connection_config)) + try: subprocess.run( indexer_cmd, stdout=subprocess.DEVNULL, stderr=stderr_log_file, check=True, + env=indexer_env, ) except subprocess.CalledProcessError: logger.exception("Failed to index archive.") @@ -458,7 +489,6 @@ def run_clp( # Remove generated temporary files if logs_list_path: logs_list_path.unlink() - db_config_file_path.unlink() logger.debug("Compressed.") # Close stderr log file diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index ddcace4d70..3a257f7e6b 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -105,8 +105,10 @@ Currently, timestamps must be specified as milliseconds since the UNIX epoch. # Parallel Compression -By default, `clp` uses an embedded SQLite database, so each directory containing archives can only -be accessed by a single `clp` instance. +:::{warning} +By default, `clp` uses an embedded SQLite database. Running multiple clp instances with SQLite +can fail due to "database is locked" because SQLite does not support concurrent writes. +::: To enable parallel compression to the same archives directory, `clp`/`clg` can be configured to use a MySQL-type database (e.g., MariaDB) as follows: @@ -114,15 +116,21 @@ a MySQL-type database (e.g., MariaDB) as follows: * Install and configure MariaDB using the instructions for your platform * Create a user that has privileges to create databases, create tables, insert records, and delete records. -* Copy and change `config/metadata-db.yml`, setting the type to `mysql` and uncommenting the MySQL - parameters. -* Install the MariaDB and PyYAML Python packages `pip3 install mariadb PyYAML` +* Install the MariaDB Python package `pip3 install mariadb` * This is necessary to run the database initialization script. If you prefer, you can run the SQL statements in `tools/scripts/db/init-db.py` directly. -* Run `tools/scripts/db/init-db.py` with the updated config file. This will initialize the database - CLP requires. -* Run `clp` or `clg` as before, with the addition of the `--db-config-file` option pointing at the - updated config file. +* Run `tools/scripts/db/init-db.py`. This will initialize the database CLP requires. Use + command-line options to configure MySQL connection parameters and set environment variables + for database credentials: + * `--db-host ` to specify the database host (default: `"localhost"`) + * `--db-port ` to specify the database port (default: `3306`) + * `--db-name ` to specify the database name (default: `"clp-db"`) + * `--db-table-prefix ` to specify the table prefix (default: `"clp_"`) + * Set `CLP_DB_USER` environment variable for the database username + * Set `CLP_DB_PASS` environment variable for the database password +* Run `clp` or `clg` with the same command options and environment variables, with the addition of + the database type command-line option: + * `--db-type mysql` to specify MySQL as the database type * To compress in parallel, simply run another instance of `clp` concurrently. Note that currently, decompression (`clp x`) and search (`clg`) can only be run with a single From bbce2fc54810aa716309a0233b1cd2e11551de26 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 31 Jul 2025 19:58:32 -0400 Subject: [PATCH 002/108] fix lint --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 30b667540e..460fc843a4 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -70,9 +70,11 @@ auto GlobalMetadataDBConfig::validate() const -> void { throw invalid_argument("Database '--db-host' is empty."); } - if (0 > m_metadata_db_port || 65535 < m_metadata_db_port) { - throw invalid_argument("Database '--db-port' is out of range [0, 65535]: " - + std::to_string(m_metadata_db_port)); + if (0 > m_metadata_db_port || 65'535 < m_metadata_db_port) { + throw invalid_argument( + "Database '--db-port' is out of range [0, 65535]: " + + std::to_string(m_metadata_db_port) + ); } if (m_metadata_db_name.empty()) { From 849b6b42c1fbac729ce2ccae03c7d8e662e3c147 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 31 Jul 2025 20:17:46 -0400 Subject: [PATCH 003/108] fix lint --- .../executor/compress/compression_task.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index 409e0a9d14..bcc8d6492a 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -182,24 +182,27 @@ def _upload_archive_to_s3( s3_put(s3_config, archive_src_path, dest_path) -def _get_db_connection_args( - clp_metadata_db_connection_config: Dict[str, Any] -) -> List[str]: +def _get_db_connection_args(clp_metadata_db_connection_config: Dict[str, Any]) -> List[str]: """ :param clp_metadata_db_connection_config: :return: List of database connection arguments for the clp command. """ return [ - "--db-type", clp_metadata_db_connection_config["type"], - "--db-host", clp_metadata_db_connection_config["host"], - "--db-port", str(clp_metadata_db_connection_config["port"]), - "--db-name", clp_metadata_db_connection_config["name"], - "--db-table-prefix", clp_metadata_db_connection_config["table_prefix"], + "--db-type", + clp_metadata_db_connection_config["type"], + "--db-host", + clp_metadata_db_connection_config["host"], + "--db-port", + str(clp_metadata_db_connection_config["port"]), + "--db-name", + clp_metadata_db_connection_config["name"], + "--db-table-prefix", + clp_metadata_db_connection_config["table_prefix"], ] def _get_db_connection_env_vars( - clp_metadata_db_connection_config: Dict[str, Any] + clp_metadata_db_connection_config: Dict[str, Any], ) -> Dict[str, str]: """ :param clp_metadata_db_connection_config: @@ -458,10 +461,12 @@ def run_clp( dataset, archive_path, ] - + # Set database credentials in environment variables for indexer indexer_env = dict(os.environ) - indexer_env.update(_get_db_connection_env_vars(clp_metadata_db_connection_config)) + indexer_env.update( + _get_db_connection_env_vars(clp_metadata_db_connection_config) + ) try: subprocess.run( From 580678550ed7a46829ca37e2a4f5fd4cf404996c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 31 Jul 2025 20:17:51 -0400 Subject: [PATCH 004/108] fix typo --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 460fc843a4..5a8e8ef99f 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -78,7 +78,7 @@ auto GlobalMetadataDBConfig::validate() const -> void { } if (m_metadata_db_name.empty()) { - throw invalid_argument("Database '--db--name' is empty."); + throw invalid_argument("Database '--db-name' is empty."); } if (m_metadata_table_prefix.empty()) { From 144f74a173dd40f740e35ecd80bfdfe51553f637 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 31 Jul 2025 20:18:21 -0400 Subject: [PATCH 005/108] remove trailing space --- docs/src/user-guide/core-unstructured/clp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index 3a257f7e6b..b562e06c16 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -106,7 +106,7 @@ Currently, timestamps must be specified as milliseconds since the UNIX epoch. # Parallel Compression :::{warning} -By default, `clp` uses an embedded SQLite database. Running multiple clp instances with SQLite +By default, `clp` uses an embedded SQLite database. Running multiple clp instances with SQLite can fail due to "database is locked" because SQLite does not support concurrent writes. ::: From eece9aa9f28fa735f98b201062f0d555743e9d4c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 31 Jul 2025 20:18:49 -0400 Subject: [PATCH 006/108] remove trailing spaces --- docs/src/user-guide/core-unstructured/clp.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index b562e06c16..0ce7075d81 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -119,7 +119,7 @@ a MySQL-type database (e.g., MariaDB) as follows: * Install the MariaDB Python package `pip3 install mariadb` * This is necessary to run the database initialization script. If you prefer, you can run the SQL statements in `tools/scripts/db/init-db.py` directly. -* Run `tools/scripts/db/init-db.py`. This will initialize the database CLP requires. Use +* Run `tools/scripts/db/init-db.py`. This will initialize the database CLP requires. Use command-line options to configure MySQL connection parameters and set environment variables for database credentials: * `--db-host ` to specify the database host (default: `"localhost"`) @@ -128,7 +128,7 @@ a MySQL-type database (e.g., MariaDB) as follows: * `--db-table-prefix ` to specify the table prefix (default: `"clp_"`) * Set `CLP_DB_USER` environment variable for the database username * Set `CLP_DB_PASS` environment variable for the database password -* Run `clp` or `clg` with the same command options and environment variables, with the addition of +* Run `clp` or `clg` with the same command options and environment variables, with the addition of the database type command-line option: * `--db-type mysql` to specify MySQL as the database type * To compress in parallel, simply run another instance of `clp` concurrently. From 6d534b3f418d2b2a37a99d4b6a650f64e7c269df Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 31 Jul 2025 22:42:02 -0400 Subject: [PATCH 007/108] Apply rabbit suggestion on - docs/src/user-guide/core-unstructured/clp.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- docs/src/user-guide/core-unstructured/clp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index 0ce7075d81..f36abb9204 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -114,7 +114,7 @@ To enable parallel compression to the same archives directory, `clp`/`clg` can b a MySQL-type database (e.g., MariaDB) as follows: * Install and configure MariaDB using the instructions for your platform -* Create a user that has privileges to create databases, create tables, insert records, and delete +* Create a user who has privileges to create databases, create tables, insert records, and delete records. * Install the MariaDB Python package `pip3 install mariadb` * This is necessary to run the database initialization script. If you prefer, you can run the SQL From 0cd45e37a24a5cbccfefb19717ea8fc00d975936 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 10:09:46 -0400 Subject: [PATCH 008/108] rabbit suggestion - Update docs/src/user-guide/core-unstructured/clp.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- docs/src/user-guide/core-unstructured/clp.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index f36abb9204..5275955273 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -106,10 +106,9 @@ Currently, timestamps must be specified as milliseconds since the UNIX epoch. # Parallel Compression :::{warning} -By default, `clp` uses an embedded SQLite database. Running multiple clp instances with SQLite +By default, `clp` uses an embedded SQLite database. Running multiple `clp` instances with SQLite can fail due to "database is locked" because SQLite does not support concurrent writes. ::: - To enable parallel compression to the same archives directory, `clp`/`clg` can be configured to use a MySQL-type database (e.g., MariaDB) as follows: From cb43f628c8a8603a3811e192e18699c6e3ec6900 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 15:04:50 -0400 Subject: [PATCH 009/108] Remove unused `using std::exception` - Apply suggestion from @gibber9809 Co-authored-by: Devin Gibson --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 5a8e8ef99f..86315f5039 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -5,7 +5,6 @@ #include -using std::exception; using std::invalid_argument; using std::string; From 8ea2f0a075919f6db9723ea600960cb6fe55b16d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 15:07:16 -0400 Subject: [PATCH 010/108] Docs - Apply suggestion from @gibber9809 Co-authored-by: Devin Gibson --- components/core/src/clp/GlobalMetadataDBConfig.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index 76e8d7b69f..c299f51d1a 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -46,7 +46,7 @@ class GlobalMetadataDBConfig { auto read_credentials_from_env() -> void; /** - * Validates that all required parameters are provided + * Validates that all required parameters are available and well-formed. */ auto validate() const -> void; From 90353c142af3002201e9caac6f7fe9007d4ac7ae Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 15:09:05 -0400 Subject: [PATCH 011/108] Remove trailing comma in arg list - Apply suggestion from @gibber9809 Co-authored-by: Devin Gibson --- components/core/tools/scripts/db/init-db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tools/scripts/db/init-db.py b/components/core/tools/scripts/db/init-db.py index a4abc628f2..fd1746d561 100644 --- a/components/core/tools/scripts/db/init-db.py +++ b/components/core/tools/scripts/db/init-db.py @@ -39,7 +39,7 @@ def main(argv): raise Exception(f"'{key}' is missing from environment variables.") try: - mysql_conn = mariadb.connect(host=host, port=port, username=username, password=password, ) + mysql_conn = mariadb.connect(host=host, port=port, username=username, password=password) mysql_cursor = mysql_conn.cursor() except mariadb.Error as err: logger.error("Failed to connect - {}".format(err.msg)) From d4c17f427b1f5c841edb890289d4b6a79745a956 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 16:42:02 -0400 Subject: [PATCH 012/108] refactor: Change add_command_line_options from static to instance method to initialize member values. --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 47 +++++++++---------- .../core/src/clp/GlobalMetadataDBConfig.hpp | 37 ++++++++------- .../core/src/clp/clg/CommandLineArguments.cpp | 3 +- .../core/src/clp/clp/CommandLineArguments.cpp | 3 +- .../clp_s/indexer/CommandLineArguments.cpp | 5 +- .../tests/test-GlobalMetadataDBConfig.cpp | 23 ++++----- 6 files changed, 56 insertions(+), 62 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 86315f5039..e9f20dd0ee 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -9,48 +9,43 @@ using std::invalid_argument; using std::string; namespace clp { +auto operator>> (std::istream &in, GlobalMetadataDBConfig::MetadataDBType &metadata_db_type) -> std::istream& { + string db_type_string; + in >> db_type_string; + + if ("sqlite" == db_type_string) + { + metadata_db_type = GlobalMetadataDBConfig::MetadataDBType::SQLite; + } else if ("mysql" == db_type_string) { + metadata_db_type = GlobalMetadataDBConfig::MetadataDBType::MySQL; + } else { + throw invalid_argument("Unknown database type: " + db_type_string); + } + + return in; +} + auto GlobalMetadataDBConfig::add_command_line_options( boost::program_options::options_description& options_description ) -> void { options_description.add_options() ("db-type", - boost::program_options::value()->default_value("sqlite"), + boost::program_options::value(&m_metadata_db_type)->default_value(cDefaultMetadataDbType, cDefaultMetadataDbTypeName.data()), "Database type [sqlite | mysql]") ("db-host", - boost::program_options::value()->default_value("127.0.0.1"), + boost::program_options::value(&m_metadata_db_host)->default_value(cDefaultMetadataDbHost.data()), "[db-type=mysql] Database host") ("db-port", - boost::program_options::value()->default_value(cDefaultMetadataDbPort), + boost::program_options::value(&m_metadata_db_port)->default_value(cDefaultMetadataDbPort), "[db-type=mysql] Database port") ("db-name", - boost::program_options::value()->default_value("clp-db"), + boost::program_options::value(&m_metadata_db_name)->default_value(cDefaultMetadataDbName.data()), "[db-type=mysql] Database name") ("db-table-prefix", - boost::program_options::value()->default_value("clp_"), + boost::program_options::value(&m_metadata_table_prefix)->default_value(cDefaultMetadataTablePrefix.data()), "[db-type=mysql] Database table prefix"); } -auto GlobalMetadataDBConfig::init_from_parsed_options( - boost::program_options::variables_map const& parsed_options -) -> void { - if (auto const db_type_string{parsed_options["db-type"].as()}; - "sqlite" == db_type_string) - { - m_metadata_db_type = MetadataDBType::SQLite; - } else if ("mysql" == db_type_string) { - m_metadata_db_type = MetadataDBType::MySQL; - } else { - throw invalid_argument("Unknown database type: " + db_type_string); - } - - if (m_metadata_db_type == MetadataDBType::MySQL) { - m_metadata_db_host = parsed_options["db-host"].as(); - m_metadata_db_port = parsed_options["db-port"].as(); - m_metadata_db_name = parsed_options["db-name"].as(); - m_metadata_table_prefix = parsed_options["db-table-prefix"].as(); - } -} - auto GlobalMetadataDBConfig::read_credentials_from_env() -> void { auto const* db_username{std::getenv("CLP_DB_USER")}; if (nullptr != db_username) { diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index c299f51d1a..6335db90bd 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -18,25 +18,25 @@ class GlobalMetadataDBConfig { MySQL, }; - // Static functions - /** - * Adds database-related command-line options to the given options_description. - * @param options_description - */ - static auto add_command_line_options( - boost::program_options::options_description& options_description - ) -> void; + /** + * Overloads operator >> to read MetadataDBType from an input stream. + * @param in + * @param metadata_db_type + * @return std::istream& reference to the input stream + */ + friend auto operator>> (std::istream &in, MetadataDBType &metadata_db_type) -> std::istream&; // Constructors GlobalMetadataDBConfig() = default; // Methods /** - * Initializes the configuration from parsed command-line options. - * @param parsed_options + * Adds database-related command-line options to the given options_description. + * @param options_description */ - auto init_from_parsed_options(boost::program_options::variables_map const& parsed_options) - -> void; + auto add_command_line_options( + boost::program_options::options_description& options_description + ) -> void; /** * Reads database credentials from environment variables: @@ -76,18 +76,23 @@ class GlobalMetadataDBConfig { private: // Variables + static constexpr std::string_view cDefaultMetadataDbTypeName{"sqlite"}; + static constexpr MetadataDBType cDefaultMetadataDbType{MetadataDBType::SQLite}; + static constexpr std::string_view cDefaultMetadataDbHost{"127.0.0.1"}; static constexpr int cDefaultMetadataDbPort{3306}; + static constexpr std::string_view cDefaultMetadataDbName{"clp-db"}; + static constexpr std::string_view cDefaultMetadataTablePrefix{"clp_"}; - MetadataDBType m_metadata_db_type{MetadataDBType::SQLite}; + MetadataDBType m_metadata_db_type{cDefaultMetadataDbType}; - std::string m_metadata_db_host{"127.0.0.1"}; + std::string m_metadata_db_host{cDefaultMetadataDbHost}; int m_metadata_db_port{cDefaultMetadataDbPort}; - std::string m_metadata_db_name; + std::string m_metadata_db_name{cDefaultMetadataDbName}; std::string m_metadata_db_username; std::string m_metadata_db_password; - std::string m_metadata_table_prefix; + std::string m_metadata_table_prefix{cDefaultMetadataTablePrefix}; }; } // namespace clp diff --git a/components/core/src/clp/clg/CommandLineArguments.cpp b/components/core/src/clp/clg/CommandLineArguments.cpp index 7ed035b48a..920be8dadd 100644 --- a/components/core/src/clp/clg/CommandLineArguments.cpp +++ b/components/core/src/clp/clg/CommandLineArguments.cpp @@ -48,7 +48,7 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { ->default_value(config_file_path), "Use configuration options from FILE" ); - GlobalMetadataDBConfig::add_command_line_options(options_general); + m_metadata_db_config.add_command_line_options(options_general); // Define input options po::options_description options_input("Input Options"); @@ -190,7 +190,6 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { // Initialize and validate global metadata DB config try { - m_metadata_db_config.init_from_parsed_options(parsed_command_line_options); m_metadata_db_config.read_credentials_from_env(); m_metadata_db_config.validate(); } catch (std::exception& e) { diff --git a/components/core/src/clp/clp/CommandLineArguments.cpp b/components/core/src/clp/clp/CommandLineArguments.cpp index c87a2c153c..20f72849a6 100644 --- a/components/core/src/clp/clp/CommandLineArguments.cpp +++ b/components/core/src/clp/clp/CommandLineArguments.cpp @@ -48,7 +48,7 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { ->default_value(config_file_path), "Use configuration options from FILE" ); - GlobalMetadataDBConfig::add_command_line_options(options_general); + m_metadata_db_config.add_command_line_options(options_general); // Define functional options po::options_description options_functional("Input Options"); @@ -113,7 +113,6 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { // Initialize and validate global metadata DB config try { - m_metadata_db_config.init_from_parsed_options(parsed_command_line_options); m_metadata_db_config.read_credentials_from_env(); m_metadata_db_config.validate(); } catch (std::exception& e) { diff --git a/components/core/src/clp_s/indexer/CommandLineArguments.cpp b/components/core/src/clp_s/indexer/CommandLineArguments.cpp index b00f2d5cd5..8b1452db93 100644 --- a/components/core/src/clp_s/indexer/CommandLineArguments.cpp +++ b/components/core/src/clp_s/indexer/CommandLineArguments.cpp @@ -29,7 +29,8 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { "Create the column metadata table if it doesn't exist" ); // clang-format on - clp::GlobalMetadataDBConfig::add_command_line_options(output_options); + clp::GlobalMetadataDBConfig metadata_db_config; + metadata_db_config.add_command_line_options(output_options); // Define visible options po::options_description visible_options; @@ -93,9 +94,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { m_archive_path = get_path_object_for_raw_path(archive_path); // Initialize and validate global metadata DB config - clp::GlobalMetadataDBConfig metadata_db_config; try { - metadata_db_config.init_from_parsed_options(parsed_command_line_options); metadata_db_config.read_credentials_from_env(); metadata_db_config.validate(); } catch (std::exception& e) { diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 5b6c3d7a18..a391221ad7 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -16,12 +16,15 @@ void unset_env_var(char const* name) { } template -auto parse_args(std::array const& argv) -> boost::program_options::variables_map { - constexpr auto argc{static_cast(N)}; +sauto parse_args( + std::array const& argv, + GlobalMetadataDBConfig& config +) -> boost::program_options::variables_map { boost::program_options::options_description options_desc; - GlobalMetadataDBConfig::add_command_line_options(options_desc); + config.add_command_line_options(options_desc); boost::program_options::variables_map vm; + constexpr auto argc{static_cast(N)}; boost::program_options::store( boost::program_options::parse_command_line( argc, @@ -37,7 +40,7 @@ auto parse_args(std::array const& argv) -> boost::program_option } // namespace TEST_CASE( - "Test adding command line options for GlobalMetadataDBConfig", + "Test parsing command line arguments for GlobalMetadataDBConfig", "[GlobalMetadataDBConfig]" ) { constexpr std::array argv{ @@ -53,10 +56,8 @@ TEST_CASE( "--db-table-prefix", "test_prefix_" }; - auto const vm = parse_args(argv); - GlobalMetadataDBConfig config; - config.init_from_parsed_options(vm); + auto const vm = parse_args(argv, config); REQUIRE(config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL); REQUIRE(config.get_metadata_db_host() == "test-host"); @@ -93,10 +94,8 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - auto const vm = parse_args(argv); - GlobalMetadataDBConfig config; - config.init_from_parsed_options(vm); + auto const vm = parse_args(argv, config); SECTION("With valid credentials") { set_env_var("CLP_DB_USER", "test-user"); @@ -127,10 +126,8 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { constexpr std::array argv{"test", "--db-type", "sqlite"}; - auto const vm = parse_args(argv); - GlobalMetadataDBConfig config; - config.init_from_parsed_options(vm); + auto const vm = parse_args(argv, config); REQUIRE_NOTHROW(config.validate()); } From d3d2146fc1d90c044f4a2b80d8adfeef932d0580 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 16:45:41 -0400 Subject: [PATCH 013/108] fix lint --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 6 +++--- .../core/src/clp/GlobalMetadataDBConfig.hpp | 19 +++++++++---------- .../tests/test-GlobalMetadataDBConfig.cpp | 6 ++---- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index e9f20dd0ee..830bcb3ff8 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -9,12 +9,12 @@ using std::invalid_argument; using std::string; namespace clp { -auto operator>> (std::istream &in, GlobalMetadataDBConfig::MetadataDBType &metadata_db_type) -> std::istream& { +auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metadata_db_type) + -> std::istream& { string db_type_string; in >> db_type_string; - if ("sqlite" == db_type_string) - { + if ("sqlite" == db_type_string) { metadata_db_type = GlobalMetadataDBConfig::MetadataDBType::SQLite; } else if ("mysql" == db_type_string) { metadata_db_type = GlobalMetadataDBConfig::MetadataDBType::MySQL; diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index 6335db90bd..cc12378d7b 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -18,13 +18,13 @@ class GlobalMetadataDBConfig { MySQL, }; - /** - * Overloads operator >> to read MetadataDBType from an input stream. - * @param in - * @param metadata_db_type - * @return std::istream& reference to the input stream - */ - friend auto operator>> (std::istream &in, MetadataDBType &metadata_db_type) -> std::istream&; + /** + * Overloads operator >> to read MetadataDBType from an input stream. + * @param in + * @param metadata_db_type + * @return std::istream& reference to the input stream + */ + friend auto operator>>(std::istream& in, MetadataDBType& metadata_db_type) -> std::istream&; // Constructors GlobalMetadataDBConfig() = default; @@ -34,9 +34,8 @@ class GlobalMetadataDBConfig { * Adds database-related command-line options to the given options_description. * @param options_description */ - auto add_command_line_options( - boost::program_options::options_description& options_description - ) -> void; + auto add_command_line_options(boost::program_options::options_description& options_description) + -> void; /** * Reads database credentials from environment variables: diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index a391221ad7..4556e906de 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -16,10 +16,8 @@ void unset_env_var(char const* name) { } template -sauto parse_args( - std::array const& argv, - GlobalMetadataDBConfig& config -) -> boost::program_options::variables_map { +auto parse_args(std::array const& argv, GlobalMetadataDBConfig& config) + -> boost::program_options::variables_map { boost::program_options::options_description options_desc; config.add_command_line_options(options_desc); From a0687a51bfc790584633044d8a6b9d44429c9762 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 16:49:26 -0400 Subject: [PATCH 014/108] test: Add credential reading from environment in validation tests --- components/core/tests/test-GlobalMetadataDBConfig.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 4556e906de..c77e50e7fe 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -107,15 +107,18 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With missing credentials") { // Neither username nor password set SECTION("Neither username nor password") { + config.read_credentials_from_env(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Username set but password missing") { set_env_var("CLP_DB_USER", "test-user"); + config.read_credentials_from_env(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); unset_env_var("CLP_DB_USER"); } SECTION("Password set but username missing") { set_env_var("CLP_DB_PASS", "test-pass"); + config.read_credentials_from_env(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); unset_env_var("CLP_DB_PASS"); } From 1bc050282b6247a8c1cd875f75f52ed002065320 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 16:52:15 -0400 Subject: [PATCH 015/108] refactor: Introduce constants for minimum and maximum port values in GlobalMetadataDBConfig --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 830bcb3ff8..caec362785 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -9,6 +9,10 @@ using std::invalid_argument; using std::string; namespace clp { +// Constants +constexpr int cMinPort{0}; +constexpr int cMaxPort{65'535}; + auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metadata_db_type) -> std::istream& { string db_type_string; @@ -64,7 +68,7 @@ auto GlobalMetadataDBConfig::validate() const -> void { throw invalid_argument("Database '--db-host' is empty."); } - if (0 > m_metadata_db_port || 65'535 < m_metadata_db_port) { + if (cMinPort > m_metadata_db_port || cMaxPort < m_metadata_db_port) { throw invalid_argument( "Database '--db-port' is out of range [0, 65535]: " + std::to_string(m_metadata_db_port) From 690e0350089dfdde2ecf7e8cf9d5c662de8c990a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 16:55:32 -0400 Subject: [PATCH 016/108] lint: Remove GlobalMetadataDBConfig.cpp exception from lint configuration --- taskfiles/lint.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 61c1371169..27be7b0b21 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -270,7 +270,6 @@ tasks: - "{{.G_CORE_COMPONENT_DIR}}/src/clp/FileWriter.cpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp/FileWriter.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp/GlobalMetadataDB.hpp" - - "{{.G_CORE_COMPONENT_DIR}}/src/clp/GlobalMetadataDBConfig.cpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp/GlobalMySQLMetadataDB.cpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp/GlobalMySQLMetadataDB.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp/GlobalSQLiteMetadataDB.cpp" From e3a24b023e1437223c7686d4631b8ec27528ed14 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 17:17:35 -0400 Subject: [PATCH 017/108] refactor: Improve error message formatting for database port range validation --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index caec362785..32b037439e 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -70,8 +70,12 @@ auto GlobalMetadataDBConfig::validate() const -> void { if (cMinPort > m_metadata_db_port || cMaxPort < m_metadata_db_port) { throw invalid_argument( - "Database '--db-port' is out of range [0, 65535]: " - + std::to_string(m_metadata_db_port) + fmt::format( + "Database '--db-port' is out of range [{}, {}]: {}", + cMinPort, + cMaxPort, + m_metadata_db_port + ) ); } From de6800786ea73bc6806928db562ccaa504208ab2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 1 Aug 2025 17:17:44 -0400 Subject: [PATCH 018/108] refactor: Update minimum port value in GlobalMetadataDBConfig to 1 --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 32b037439e..79b969988a 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -10,7 +10,7 @@ using std::string; namespace clp { // Constants -constexpr int cMinPort{0}; +constexpr int cMinPort{1}; constexpr int cMaxPort{65'535}; auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metadata_db_type) From 1647e0b04548a361d43b68e2c90c817a7582435a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 3 Aug 2025 02:15:16 -0400 Subject: [PATCH 019/108] Remove redundant spaces in arg list - Apply suggestion from @coderabbitai[bot] Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- components/core/tools/scripts/db/init-db.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/core/tools/scripts/db/init-db.py b/components/core/tools/scripts/db/init-db.py index fd1746d561..8bc7e98092 100644 --- a/components/core/tools/scripts/db/init-db.py +++ b/components/core/tools/scripts/db/init-db.py @@ -19,10 +19,10 @@ def main(argv): args_parser = argparse.ArgumentParser(description="Setup a global MySQL metadata database for CLP.") - args_parser.add_argument( "--db-host", default="127.0.0.1", help="Database host" ) - args_parser.add_argument( "--db-port", type=int, default=3306, help="Database port" ) - args_parser.add_argument( "--db-name", default="clp-db", help="Database name" ) - args_parser.add_argument( "--db-table-prefix", default="clp_", help="Database table prefix" ) + args_parser.add_argument("--db-host", default="127.0.0.1", help="Database host") + args_parser.add_argument("--db-port", type=int, default=3306, help="Database port") + args_parser.add_argument("--db-name", default="clp-db", help="Database name") + args_parser.add_argument("--db-table-prefix", default="clp_", help="Database table prefix") parsed_args = args_parser.parse_args(argv[1:]) From 04923b8f1637eeda15f02991bb8a4682a2072e8f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 3 Aug 2025 02:43:13 -0400 Subject: [PATCH 020/108] Use fmt for log messages - Update components/core/src/clp/GlobalMetadataDBConfig.cpp Co-authored-by: haiqi96 <14502009+haiqi96@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 79b969988a..c7216d9292 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -23,7 +23,7 @@ auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metada } else if ("mysql" == db_type_string) { metadata_db_type = GlobalMetadataDBConfig::MetadataDBType::MySQL; } else { - throw invalid_argument("Unknown database type: " + db_type_string); + throw invalid_argument(fmt::format("Unknown database type: {}",db_type_string)); } return in; From aedebe179862f95de5e12b6d6054bb85205709b0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 3 Aug 2025 02:20:53 -0400 Subject: [PATCH 021/108] perform mysql db type check before further parsing from env --- .../src/clp_s/indexer/CommandLineArguments.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/core/src/clp_s/indexer/CommandLineArguments.cpp b/components/core/src/clp_s/indexer/CommandLineArguments.cpp index 8b1452db93..49c7f1a186 100644 --- a/components/core/src/clp_s/indexer/CommandLineArguments.cpp +++ b/components/core/src/clp_s/indexer/CommandLineArguments.cpp @@ -94,14 +94,6 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { m_archive_path = get_path_object_for_raw_path(archive_path); // Initialize and validate global metadata DB config - try { - metadata_db_config.read_credentials_from_env(); - metadata_db_config.validate(); - } catch (std::exception& e) { - SPDLOG_ERROR("Failed to validate metadata database config - {}.", e.what()); - return ParsingResult::Failure; - } - if (clp::GlobalMetadataDBConfig::MetadataDBType::MySQL != metadata_db_config.get_metadata_db_type()) { @@ -112,6 +104,14 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { return ParsingResult::Failure; } + try { + metadata_db_config.read_credentials_from_env(); + metadata_db_config.validate(); + } catch (std::exception& e) { + SPDLOG_ERROR("Failed to validate metadata database config - {}.", e.what()); + return ParsingResult::Failure; + } + m_metadata_db_config = std::move(metadata_db_config); } catch (std::exception& e) { SPDLOG_ERROR("{}", e.what()); From 5dc87d1b2c3556441bdc1afd4440b784a23f46ac Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 3 Aug 2025 02:24:46 -0400 Subject: [PATCH 022/108] refactor: Skip credential reading for SQLite database type --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index c7216d9292..26ac3dd851 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -51,6 +51,11 @@ auto GlobalMetadataDBConfig::add_command_line_options( } auto GlobalMetadataDBConfig::read_credentials_from_env() -> void { + if (MetadataDBType::SQLite == m_metadata_db_type) { + // SQLite does not require credentials + return; + } + auto const* db_username{std::getenv("CLP_DB_USER")}; if (nullptr != db_username) { m_metadata_db_username = db_username; From 968221c4da1ec27dba436e1d40863d63c0ccddf6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 3 Aug 2025 02:29:47 -0400 Subject: [PATCH 023/108] rename read_credentials_from_env -> read_credentials_from_env_if_needed --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 2 +- components/core/src/clp/GlobalMetadataDBConfig.hpp | 4 ++-- components/core/src/clp/clg/CommandLineArguments.cpp | 2 +- components/core/src/clp/clp/CommandLineArguments.cpp | 2 +- .../core/src/clp_s/indexer/CommandLineArguments.cpp | 2 +- components/core/tests/test-GlobalMetadataDBConfig.cpp | 10 +++++----- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 26ac3dd851..1765a87255 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -50,7 +50,7 @@ auto GlobalMetadataDBConfig::add_command_line_options( "[db-type=mysql] Database table prefix"); } -auto GlobalMetadataDBConfig::read_credentials_from_env() -> void { +auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { if (MetadataDBType::SQLite == m_metadata_db_type) { // SQLite does not require credentials return; diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index cc12378d7b..b5c8b8d7cc 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -38,11 +38,11 @@ class GlobalMetadataDBConfig { -> void; /** - * Reads database credentials from environment variables: + * Reads database credentials from environment variables if required by the database type. * - CLP_DB_USER for database username * - CLP_DB_PASS for database password */ - auto read_credentials_from_env() -> void; + auto read_credentials_from_env_if_needed() -> void; /** * Validates that all required parameters are available and well-formed. diff --git a/components/core/src/clp/clg/CommandLineArguments.cpp b/components/core/src/clp/clg/CommandLineArguments.cpp index 920be8dadd..33f9325c6a 100644 --- a/components/core/src/clp/clg/CommandLineArguments.cpp +++ b/components/core/src/clp/clg/CommandLineArguments.cpp @@ -190,7 +190,7 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { // Initialize and validate global metadata DB config try { - m_metadata_db_config.read_credentials_from_env(); + m_metadata_db_config.read_credentials_from_env_if_needed(); m_metadata_db_config.validate(); } catch (std::exception& e) { SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what()); diff --git a/components/core/src/clp/clp/CommandLineArguments.cpp b/components/core/src/clp/clp/CommandLineArguments.cpp index 20f72849a6..bf927e1af0 100644 --- a/components/core/src/clp/clp/CommandLineArguments.cpp +++ b/components/core/src/clp/clp/CommandLineArguments.cpp @@ -113,7 +113,7 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { // Initialize and validate global metadata DB config try { - m_metadata_db_config.read_credentials_from_env(); + m_metadata_db_config.read_credentials_from_env_if_needed(); m_metadata_db_config.validate(); } catch (std::exception& e) { SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what()); diff --git a/components/core/src/clp_s/indexer/CommandLineArguments.cpp b/components/core/src/clp_s/indexer/CommandLineArguments.cpp index 49c7f1a186..fa1b72d1e9 100644 --- a/components/core/src/clp_s/indexer/CommandLineArguments.cpp +++ b/components/core/src/clp_s/indexer/CommandLineArguments.cpp @@ -105,7 +105,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { } try { - metadata_db_config.read_credentials_from_env(); + metadata_db_config.read_credentials_from_env_if_needed(); metadata_db_config.validate(); } catch (std::exception& e) { SPDLOG_ERROR("Failed to validate metadata database config - {}.", e.what()); diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index c77e50e7fe..f471b1658c 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -69,7 +69,7 @@ TEST_CASE("Test reading credentials from environment variables", "[GlobalMetadat set_env_var("CLP_DB_PASS", "test-pass"); GlobalMetadataDBConfig config; - config.read_credentials_from_env(); + config.read_credentials_from_env_if_needed(); REQUIRE(config.get_metadata_db_username() == "test-user"); REQUIRE(config.get_metadata_db_password() == "test-pass"); @@ -98,7 +98,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With valid credentials") { set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); - config.read_credentials_from_env(); + config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); unset_env_var("CLP_DB_USER"); unset_env_var("CLP_DB_PASS"); @@ -107,18 +107,18 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With missing credentials") { // Neither username nor password set SECTION("Neither username nor password") { - config.read_credentials_from_env(); + config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Username set but password missing") { set_env_var("CLP_DB_USER", "test-user"); - config.read_credentials_from_env(); + config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); unset_env_var("CLP_DB_USER"); } SECTION("Password set but username missing") { set_env_var("CLP_DB_PASS", "test-pass"); - config.read_credentials_from_env(); + config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); unset_env_var("CLP_DB_PASS"); } From cae4ce51f4d737b5b3ddd81134c318e81c46feaf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 3 Aug 2025 02:32:59 -0400 Subject: [PATCH 024/108] early return --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 67 ++++++++++--------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 1765a87255..2d38a07cd1 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -52,7 +52,7 @@ auto GlobalMetadataDBConfig::add_command_line_options( auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { if (MetadataDBType::SQLite == m_metadata_db_type) { - // SQLite does not require credentials + // SQLite does not require credentials. return; } @@ -68,37 +68,40 @@ auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { } auto GlobalMetadataDBConfig::validate() const -> void { - if (m_metadata_db_type == MetadataDBType::MySQL) { - if (m_metadata_db_host.empty()) { - throw invalid_argument("Database '--db-host' is empty."); - } - - if (cMinPort > m_metadata_db_port || cMaxPort < m_metadata_db_port) { - throw invalid_argument( - fmt::format( - "Database '--db-port' is out of range [{}, {}]: {}", - cMinPort, - cMaxPort, - m_metadata_db_port - ) - ); - } - - if (m_metadata_db_name.empty()) { - throw invalid_argument("Database '--db-name' is empty."); - } - - if (m_metadata_table_prefix.empty()) { - throw invalid_argument("Database '--db-table_prefix' is empty."); - } - - if (m_metadata_db_username.empty()) { - throw invalid_argument("Database 'CLP_DB_USER' not specified or empty."); - } - - if (m_metadata_db_password.empty()) { - throw invalid_argument("Database 'CLP_DB_PASS' not specified or empty."); - } + if (m_metadata_db_type == MetadataDBType::SQLite) { + // Currently, SQLite does not require any additional parameters. + return; + } + + if (m_metadata_db_host.empty()) { + throw invalid_argument("Database '--db-host' is empty."); + } + + if (cMinPort > m_metadata_db_port || cMaxPort < m_metadata_db_port) { + throw invalid_argument( + fmt::format( + "Database '--db-port' is out of range [{}, {}]: {}", + cMinPort, + cMaxPort, + m_metadata_db_port + ) + ); + } + + if (m_metadata_db_name.empty()) { + throw invalid_argument("Database '--db-name' is empty."); + } + + if (m_metadata_table_prefix.empty()) { + throw invalid_argument("Database '--db-table_prefix' is empty."); + } + + if (m_metadata_db_username.empty()) { + throw invalid_argument("Database 'CLP_DB_USER' not specified or empty."); + } + + if (m_metadata_db_password.empty()) { + throw invalid_argument("Database 'CLP_DB_PASS' not specified or empty."); } } } // namespace clp From aedca2b45cdf26fd601a9ff3610c772e46320872 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 3 Aug 2025 02:33:39 -0400 Subject: [PATCH 025/108] reorder m_metadata_table_prefix --- components/core/src/clp/GlobalMetadataDBConfig.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index b5c8b8d7cc..9a6f6ddbb3 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -87,11 +87,10 @@ class GlobalMetadataDBConfig { std::string m_metadata_db_host{cDefaultMetadataDbHost}; int m_metadata_db_port{cDefaultMetadataDbPort}; std::string m_metadata_db_name{cDefaultMetadataDbName}; + std::string m_metadata_table_prefix{cDefaultMetadataTablePrefix}; std::string m_metadata_db_username; std::string m_metadata_db_password; - - std::string m_metadata_table_prefix{cDefaultMetadataTablePrefix}; }; } // namespace clp From 9c63298f28d819ea073045c14694695388bd39f9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 3 Aug 2025 02:34:21 -0400 Subject: [PATCH 026/108] docs: add exception detail to validate method documentation --- components/core/src/clp/GlobalMetadataDBConfig.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index 9a6f6ddbb3..c69dddb4a1 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -46,6 +46,7 @@ class GlobalMetadataDBConfig { /** * Validates that all required parameters are available and well-formed. + * @throw std::invalid_argument if validation fails */ auto validate() const -> void; From 47fb5d2d66613ba93cf48480f6faa2b6885ed4d5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 3 Aug 2025 02:39:35 -0400 Subject: [PATCH 027/108] refactor: enhance validation for SQLite database parameters --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 2d38a07cd1..b0e9a36d04 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -69,7 +69,16 @@ auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { auto GlobalMetadataDBConfig::validate() const -> void { if (m_metadata_db_type == MetadataDBType::SQLite) { - // Currently, SQLite does not require any additional parameters. + // SQLite does not require extra parameters. + if (false == m_metadata_db_host.empty() || cDefaultMetadataDbPort != m_metadata_db_port + || false == m_metadata_db_name.empty() || false == m_metadata_table_prefix.empty() + || false == m_metadata_db_username.empty() || false == m_metadata_db_password.empty()) + { + throw invalid_argument( + "MySQL-specific parameters were provided when '--db-type' is 'SQLite'. " + "Please remove them or set '--db-type=mysql'." + ); + } return; } From 54e348369541d3556240f8d856ac182baf39b345 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 4 Aug 2025 02:08:09 -0400 Subject: [PATCH 028/108] move consts to anonymous namespace; use consts for metadata db type strings. --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 76 +++++++++++++------ .../core/src/clp/GlobalMetadataDBConfig.hpp | 9 ++- 2 files changed, 61 insertions(+), 24 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index b0e9a36d04..358c735b6d 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -5,49 +5,79 @@ #include +#include "type_utils.hpp" using std::invalid_argument; using std::string; -namespace clp { +namespace { // Constants constexpr int cMinPort{1}; constexpr int cMaxPort{65'535}; +} // namespace +namespace clp { auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metadata_db_type) -> std::istream& { string db_type_string; in >> db_type_string; - if ("sqlite" == db_type_string) { - metadata_db_type = GlobalMetadataDBConfig::MetadataDBType::SQLite; - } else if ("mysql" == db_type_string) { - metadata_db_type = GlobalMetadataDBConfig::MetadataDBType::MySQL; - } else { - throw invalid_argument(fmt::format("Unknown database type: {}",db_type_string)); + for (size_t i = 0; i < GlobalMetadataDBConfig::cMetadataDBTypeNames.size(); ++i) { + if (GlobalMetadataDBConfig::cMetadataDBTypeNames[i] == db_type_string) { + metadata_db_type = static_cast(i); + return in; + } } - return in; + throw invalid_argument(fmt::format("Unknown database type: {}", db_type_string)); } auto GlobalMetadataDBConfig::add_command_line_options( boost::program_options::options_description& options_description ) -> void { + constexpr std::string_view cMetadataDbTypeMysqlOptDescPrefix{"[db-type=mysql]"}; + options_description.add_options() - ("db-type", - boost::program_options::value(&m_metadata_db_type)->default_value(cDefaultMetadataDbType, cDefaultMetadataDbTypeName.data()), - "Database type [sqlite | mysql]") - ("db-host", - boost::program_options::value(&m_metadata_db_host)->default_value(cDefaultMetadataDbHost.data()), - "[db-type=mysql] Database host") - ("db-port", - boost::program_options::value(&m_metadata_db_port)->default_value(cDefaultMetadataDbPort), - "[db-type=mysql] Database port") - ("db-name", - boost::program_options::value(&m_metadata_db_name)->default_value(cDefaultMetadataDbName.data()), - "[db-type=mysql] Database name") - ("db-table-prefix", - boost::program_options::value(&m_metadata_table_prefix)->default_value(cDefaultMetadataTablePrefix.data()), - "[db-type=mysql] Database table prefix"); + ("db-type", + boost::program_options::value(&m_metadata_db_type) + ->default_value(cDefaultMetadataDbType, + cMetadataDBTypeNames[enum_to_underlying_type(cDefaultMetadataDbType)]. + data()), + fmt::format("Database type [{} | {}]", + cMetadataDBTypeNames + [enum_to_underlying_type(MetadataDBType::SQLite)], + cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MySQL)]).c_str() + ) + ("db-host", + boost::program_options::value(&m_metadata_db_host)->default_value( + cDefaultMetadataDbHost.data()), + fmt::format( + "{} Database host", + cMetadataDbTypeMysqlOptDescPrefix + ).c_str()) + ("db-port", + boost::program_options::value(&m_metadata_db_port)->default_value( + cDefaultMetadataDbPort), + fmt::format( + "{} Database port (default: {})", + cMetadataDbTypeMysqlOptDescPrefix, + cDefaultMetadataDbPort + ).c_str()) + ("db-name", + boost::program_options::value(&m_metadata_db_name)->default_value( + cDefaultMetadataDbName.data()), + fmt::format( + "{} Database name (default: {})", + cMetadataDbTypeMysqlOptDescPrefix, + cDefaultMetadataDbName.data() + ).c_str()) + ("db-table-prefix", + boost::program_options::value(&m_metadata_table_prefix)->default_value( + cDefaultMetadataTablePrefix.data()), + fmt::format( + "{} Database table prefix (default: {})", + cMetadataDbTypeMysqlOptDescPrefix, + cDefaultMetadataTablePrefix.data() + ).c_str()); } auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index c69dddb4a1..b49bacebfb 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -6,6 +6,8 @@ #include +#include "type_utils.hpp" + namespace clp { /** * Class encapsulating the global metadata database's configuration details @@ -16,8 +18,14 @@ class GlobalMetadataDBConfig { enum class MetadataDBType : uint8_t { SQLite = 0, MySQL, + MetadataDBTypeLength }; + // Constants + static constexpr std:: + array + cMetadataDBTypeNames{"sqlite", "mysql"}; + /** * Overloads operator >> to read MetadataDBType from an input stream. * @param in @@ -76,7 +84,6 @@ class GlobalMetadataDBConfig { private: // Variables - static constexpr std::string_view cDefaultMetadataDbTypeName{"sqlite"}; static constexpr MetadataDBType cDefaultMetadataDbType{MetadataDBType::SQLite}; static constexpr std::string_view cDefaultMetadataDbHost{"127.0.0.1"}; static constexpr int cDefaultMetadataDbPort{3306}; From e39e1d7da025674f5344ce9d1c7861f1b5b199d7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 4 Aug 2025 02:13:45 -0400 Subject: [PATCH 029/108] update SQLite validation to check for default values --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 358c735b6d..4f103bff40 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -100,9 +100,13 @@ auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { auto GlobalMetadataDBConfig::validate() const -> void { if (m_metadata_db_type == MetadataDBType::SQLite) { // SQLite does not require extra parameters. - if (false == m_metadata_db_host.empty() || cDefaultMetadataDbPort != m_metadata_db_port - || false == m_metadata_db_name.empty() || false == m_metadata_table_prefix.empty() - || false == m_metadata_db_username.empty() || false == m_metadata_db_password.empty()) + if (cDefaultMetadataDbHost != m_metadata_db_host + || cDefaultMetadataDbPort != m_metadata_db_port + || cDefaultMetadataDbName != m_metadata_db_name + || cDefaultMetadataTablePrefix != m_metadata_table_prefix + || false == m_metadata_db_username.empty() + || false == m_metadata_db_password.empty() + ) { throw invalid_argument( "MySQL-specific parameters were provided when '--db-type' is 'SQLite'. " From 7b397503026048acf62996527e25ab955954d114 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 4 Aug 2025 02:24:16 -0400 Subject: [PATCH 030/108] update tests --- .../tests/test-GlobalMetadataDBConfig.cpp | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index f471b1658c..682845fdb6 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -64,20 +64,6 @@ TEST_CASE( REQUIRE(config.get_metadata_table_prefix() == "test_prefix_"); } -TEST_CASE("Test reading credentials from environment variables", "[GlobalMetadataDBConfig]") { - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", "test-pass"); - - GlobalMetadataDBConfig config; - config.read_credentials_from_env_if_needed(); - - REQUIRE(config.get_metadata_db_username() == "test-user"); - REQUIRE(config.get_metadata_db_password() == "test-pass"); - - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); -} - TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBConfig]") { constexpr std::array argv{ "test", @@ -98,14 +84,17 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With valid credentials") { set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); + config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); + REQUIRE(config.get_metadata_db_username() == "test-user"); + REQUIRE(config.get_metadata_db_password() == "test-pass"); + unset_env_var("CLP_DB_USER"); unset_env_var("CLP_DB_PASS"); } SECTION("With missing credentials") { - // Neither username nor password set SECTION("Neither username nor password") { config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); @@ -130,5 +119,14 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { GlobalMetadataDBConfig config; auto const vm = parse_args(argv, config); + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", "test-pass"); + + config.read_credentials_from_env_if_needed(); + REQUIRE(config.get_metadata_db_username().empty()); + REQUIRE(config.get_metadata_db_password().empty()); REQUIRE_NOTHROW(config.validate()); + + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); } From 2ba751dd83eb6a2a637803f1baf694ae46162fa2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 4 Aug 2025 02:24:20 -0400 Subject: [PATCH 031/108] lint --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 4f103bff40..59cd522ef3 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -104,9 +104,7 @@ auto GlobalMetadataDBConfig::validate() const -> void { || cDefaultMetadataDbPort != m_metadata_db_port || cDefaultMetadataDbName != m_metadata_db_name || cDefaultMetadataTablePrefix != m_metadata_table_prefix - || false == m_metadata_db_username.empty() - || false == m_metadata_db_password.empty() - ) + || false == m_metadata_db_username.empty() || false == m_metadata_db_password.empty()) { throw invalid_argument( "MySQL-specific parameters were provided when '--db-type' is 'SQLite'. " From 1aa27fe744d6c8e3680418e3dabad38d045cce59 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 02:55:33 -0400 Subject: [PATCH 032/108] docs - Update docs/src/user-guide/core-unstructured/clp.md Co-authored-by: haiqi96 <14502009+haiqi96@users.noreply.github.com> --- docs/src/user-guide/core-unstructured/clp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index 5275955273..05da3db395 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -107,7 +107,7 @@ Currently, timestamps must be specified as milliseconds since the UNIX epoch. :::{warning} By default, `clp` uses an embedded SQLite database. Running multiple `clp` instances with SQLite -can fail due to "database is locked" because SQLite does not support concurrent writes. +can fail due to "database is locked" error because SQLite does not support concurrent writes. ::: To enable parallel compression to the same archives directory, `clp`/`clg` can be configured to use a MySQL-type database (e.g., MariaDB) as follows: From dc0c89968a0425aceedf150b9a116622fa1edb0d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 03:29:55 -0400 Subject: [PATCH 033/108] refactor(core): make metadata DB password optional --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 4 +--- .../tests/test-GlobalMetadataDBConfig.cpp | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 59cd522ef3..e262a7abb0 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -141,8 +141,6 @@ auto GlobalMetadataDBConfig::validate() const -> void { throw invalid_argument("Database 'CLP_DB_USER' not specified or empty."); } - if (m_metadata_db_password.empty()) { - throw invalid_argument("Database 'CLP_DB_PASS' not specified or empty."); - } + // NOTE: No need to check `m_metadata_db_password` because `CLP_DB_PASS` is optional. } } // namespace clp diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 682845fdb6..d9af5cbcc3 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -94,17 +94,24 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo unset_env_var("CLP_DB_PASS"); } + SECTION("With empty password") { + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", ""); + + config.read_credentials_from_env_if_needed(); + REQUIRE_NOTHROW(config.validate()); + REQUIRE(config.get_metadata_db_username() == "test-user"); + REQUIRE(config.get_metadata_db_password() == ""); + + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); + } + SECTION("With missing credentials") { SECTION("Neither username nor password") { config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } - SECTION("Username set but password missing") { - set_env_var("CLP_DB_USER", "test-user"); - config.read_credentials_from_env_if_needed(); - REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); - unset_env_var("CLP_DB_USER"); - } SECTION("Password set but username missing") { set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); From a4fd2450fc63486bbd311578413836894971dddb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 03:44:26 -0400 Subject: [PATCH 034/108] test(core): add more unit tests for SQLite database configuration validation --- .../tests/test-GlobalMetadataDBConfig.cpp | 56 +++++++++++++++---- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index d9af5cbcc3..c9b5c09f49 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -122,18 +122,52 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { - constexpr std::array argv{"test", "--db-type", "sqlite"}; - GlobalMetadataDBConfig config; - auto const vm = parse_args(argv, config); + SECTION("SQLite with db-host argument") { + constexpr std::array argv{"test", "--db-type", "sqlite", "--db-host", "test-host"}; + GlobalMetadataDBConfig config; + + REQUIRE_NOTHROW(parse_args(argv, config)); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + } + + SECTION("SQLite with db-port argument") { + constexpr std::array argv{"test", "--db-type", "sqlite", "--db-port", "8888"}; + GlobalMetadataDBConfig config; - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", "test-pass"); + REQUIRE_NOTHROW(parse_args(argv, config)); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + } - config.read_credentials_from_env_if_needed(); - REQUIRE(config.get_metadata_db_username().empty()); - REQUIRE(config.get_metadata_db_password().empty()); - REQUIRE_NOTHROW(config.validate()); + SECTION("SQLite with db-name argument") { + constexpr std::array argv{"test", "--db-type", "sqlite", "--db-name", "test-db"}; + GlobalMetadataDBConfig config; + + REQUIRE_NOTHROW(parse_args(argv, config)); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + } + + SECTION("SQLite with db-table-prefix argument") { + constexpr std::array argv{"test", "--db-type", "sqlite", "--db-table-prefix", "test_prefix_"}; + GlobalMetadataDBConfig config; + + REQUIRE_NOTHROW(parse_args(argv, config)); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + } - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); + SECTION("With username and password") { + constexpr std::array argv{"test", "--db-type", "sqlite"}; + GlobalMetadataDBConfig config; + auto const vm = parse_args(argv, config); + + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", "test-pass"); + + config.read_credentials_from_env_if_needed(); + REQUIRE(config.get_metadata_db_username().empty()); + REQUIRE(config.get_metadata_db_password().empty()); + REQUIRE_NOTHROW(config.validate()); + + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); + } } From ea9765f4e0de210883cbbd599ef1542c5b006f13 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 03:58:00 -0400 Subject: [PATCH 035/108] test(core): add additional validation tests for GlobalMetadataDBConfig --- .../tests/test-GlobalMetadataDBConfig.cpp | 219 ++++++++++++++---- 1 file changed, 173 insertions(+), 46 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index c9b5c09f49..dd7e75ac01 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -65,64 +65,190 @@ TEST_CASE( } TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBConfig]") { - constexpr std::array argv{ - "test", - "--db-type", - "mysql", - "--db-host", - "test-host", - "--db-port", - "8888", - "--db-name", - "test-db", - "--db-table-prefix", - "test_prefix_" - }; - GlobalMetadataDBConfig config; - auto const vm = parse_args(argv, config); + SECTION("With all arguments") { + constexpr std::array argv{ + "test", + "--db-type", + "mysql", + "--db-host", + "test-host", + "--db-port", + "8888", + "--db-name", + "test-db", + "--db-table-prefix", + "test_prefix_" + }; + GlobalMetadataDBConfig config; + auto const vm = parse_args(argv, config); - SECTION("With valid credentials") { - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", "test-pass"); + SECTION("With valid credentials") { + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", "test-pass"); - config.read_credentials_from_env_if_needed(); - REQUIRE_NOTHROW(config.validate()); - REQUIRE(config.get_metadata_db_username() == "test-user"); - REQUIRE(config.get_metadata_db_password() == "test-pass"); + config.read_credentials_from_env_if_needed(); + REQUIRE_NOTHROW(config.validate()); + REQUIRE(config.get_metadata_db_username() == "test-user"); + REQUIRE(config.get_metadata_db_password() == "test-pass"); - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); - } + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); + } - SECTION("With empty password") { - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", ""); + SECTION("With empty password") { + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", ""); - config.read_credentials_from_env_if_needed(); - REQUIRE_NOTHROW(config.validate()); - REQUIRE(config.get_metadata_db_username() == "test-user"); - REQUIRE(config.get_metadata_db_password() == ""); + config.read_credentials_from_env_if_needed(); + REQUIRE_NOTHROW(config.validate()); + REQUIRE(config.get_metadata_db_username() == "test-user"); + REQUIRE(config.get_metadata_db_password() == ""); - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); + } + + SECTION("With missing credentials") { + SECTION("Neither username nor password") { + config.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + } + SECTION("Password set but username missing") { + set_env_var("CLP_DB_PASS", "test-pass"); + config.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + unset_env_var("CLP_DB_PASS"); + } + } } - SECTION("With missing credentials") { - SECTION("Neither username nor password") { - config.read_credentials_from_env_if_needed(); - REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + SECTION("With invalid port values") { + SECTION("Port too low") { + constexpr std::array argv_with_low_port{ + "test", + "--db-type", + "mysql", + "--db-host", + "test-host", + "--db-port", + "0", + "--db-name", + "test-db", + "--db-table-prefix", + "test_prefix_" + }; + GlobalMetadataDBConfig config_low_port; + parse_args(argv_with_low_port, config_low_port); + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", "test-pass"); + config_low_port.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config_low_port.validate(), std::invalid_argument); + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); + } + + SECTION("Port too high") { + constexpr std::array argv_with_high_port{ + "test", + "--db-type", + "mysql", + "--db-host", + "test-host", + "--db-port", + "65536", + "--db-name", + "test-db", + "--db-table-prefix", + "test_prefix_" + }; + GlobalMetadataDBConfig config_high_port; + parse_args(argv_with_high_port, config_high_port); + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", "test-pass"); + config_high_port.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config_high_port.validate(), std::invalid_argument); + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); } - SECTION("Password set but username missing") { + } + + SECTION("With empty required arguments") { + SECTION("Empty db-host") { + constexpr std::array argv_empty_host{ + "test", + "--db-type", + "mysql", + "--db-host", + "", + "--db-port", + "8888", + "--db-name", + "test-db", + "--db-table-prefix", + "test_prefix_" + }; + GlobalMetadataDBConfig config_empty_host; + parse_args(argv_empty_host, config_empty_host); + set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); - config.read_credentials_from_env_if_needed(); - REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + config_empty_host.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config_empty_host.validate(), std::invalid_argument); + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); + } + + SECTION("Empty db-name") { + constexpr std::array argv_empty_name{ + "test", + "--db-type", + "mysql", + "--db-host", + "test-host", + "--db-port", + "8888", + "--db-name", + "", + "--db-table-prefix", + "test_prefix_" + }; + GlobalMetadataDBConfig config_empty_name; + parse_args(argv_empty_name, config_empty_name); + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", "test-pass"); + config_empty_name.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config_empty_name.validate(), std::invalid_argument); + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); + } + + SECTION("Empty db-table-prefix") { + constexpr std::array argv_empty_prefix{ + "test", + "--db-type", + "mysql", + "--db-host", + "test-host", + "--db-port", + "8888", + "--db-name", + "test-db", + "--db-table-prefix", + "" + }; + GlobalMetadataDBConfig config_empty_prefix; + parse_args(argv_empty_prefix, config_empty_prefix); + set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_PASS", "test-pass"); + config_empty_prefix.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config_empty_prefix.validate(), std::invalid_argument); + unset_env_var("CLP_DB_USER"); unset_env_var("CLP_DB_PASS"); } } } TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { - SECTION("SQLite with db-host argument") { + SECTION("With non-default db-host argument") { constexpr std::array argv{"test", "--db-type", "sqlite", "--db-host", "test-host"}; GlobalMetadataDBConfig config; @@ -130,7 +256,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } - SECTION("SQLite with db-port argument") { + SECTION("With non-default db-port argument") { constexpr std::array argv{"test", "--db-type", "sqlite", "--db-port", "8888"}; GlobalMetadataDBConfig config; @@ -138,7 +264,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } - SECTION("SQLite with db-name argument") { + SECTION("With non-default db-name argument") { constexpr std::array argv{"test", "--db-type", "sqlite", "--db-name", "test-db"}; GlobalMetadataDBConfig config; @@ -146,8 +272,9 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } - SECTION("SQLite with db-table-prefix argument") { - constexpr std::array argv{"test", "--db-type", "sqlite", "--db-table-prefix", "test_prefix_"}; + SECTION("With non-default db-table-prefix argument") { + constexpr std::array + argv{"test", "--db-type", "sqlite", "--db-table-prefix", "test_prefix_"}; GlobalMetadataDBConfig config; REQUIRE_NOTHROW(parse_args(argv, config)); From 06bc4823b5bd9c4d89cf9e059a352e3208070826 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 04:03:22 -0400 Subject: [PATCH 036/108] refactor(core): Change the return type of parse_args function from variables_map to void --- components/core/tests/test-GlobalMetadataDBConfig.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index dd7e75ac01..05283d321d 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -16,8 +16,7 @@ void unset_env_var(char const* name) { } template -auto parse_args(std::array const& argv, GlobalMetadataDBConfig& config) - -> boost::program_options::variables_map { +auto parse_args(std::array const& argv, GlobalMetadataDBConfig& config) -> void { boost::program_options::options_description options_desc; config.add_command_line_options(options_desc); @@ -32,8 +31,6 @@ auto parse_args(std::array const& argv, GlobalMetadataDBConfig& vm ); boost::program_options::notify(vm); - - return vm; } } // namespace @@ -55,7 +52,7 @@ TEST_CASE( "test_prefix_" }; GlobalMetadataDBConfig config; - auto const vm = parse_args(argv, config); + parse_args(argv, config); REQUIRE(config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL); REQUIRE(config.get_metadata_db_host() == "test-host"); @@ -80,7 +77,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "test_prefix_" }; GlobalMetadataDBConfig config; - auto const vm = parse_args(argv, config); + parse_args(argv, config); SECTION("With valid credentials") { set_env_var("CLP_DB_USER", "test-user"); @@ -284,7 +281,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With username and password") { constexpr std::array argv{"test", "--db-type", "sqlite"}; GlobalMetadataDBConfig config; - auto const vm = parse_args(argv, config); + parse_args(argv, config); set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); From b06d71aae7bf2203c9b849dc2482f9543f82b416 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 11:50:53 -0400 Subject: [PATCH 037/108] Docs - Update components/core/src/clp/GlobalMetadataDBConfig.cpp Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index e262a7abb0..3feefcbd68 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -82,7 +82,7 @@ auto GlobalMetadataDBConfig::add_command_line_options( auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { if (MetadataDBType::SQLite == m_metadata_db_type) { - // SQLite does not require credentials. + // SQLite doesn't require extra parameters. return; } From e647fcfac25f5014c0f7ec6964c1e87ea854a36b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 11:52:46 -0400 Subject: [PATCH 038/108] Exception message - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 3feefcbd68..025491de03 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -107,8 +107,8 @@ auto GlobalMetadataDBConfig::validate() const -> void { || false == m_metadata_db_username.empty() || false == m_metadata_db_password.empty()) { throw invalid_argument( - "MySQL-specific parameters were provided when '--db-type' is 'SQLite'. " - "Please remove them or set '--db-type=mysql'." + "MySQL-specific parameters cannot be used with --db-type='sqlite'." + " Please remove them or set '--db-type=mysql'." ); } return; From ba0c2c52b82e2fe74f0e7f52d48c8df84d4024a8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 12:56:48 -0400 Subject: [PATCH 039/108] Docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index b49bacebfb..c83cd3fb5a 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -22,6 +22,7 @@ class GlobalMetadataDBConfig { }; // Constants + // NOTE: The strings in the array must match the order of the enums values in `MetadataDBType`. static constexpr std:: array cMetadataDBTypeNames{"sqlite", "mysql"}; From 3745ca2e00a3f81953772077cf61965d0d1e7f8f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 13:04:49 -0400 Subject: [PATCH 040/108] Docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .../core/src/clp/GlobalMetadataDBConfig.hpp | 15 ++++++++++----- .../executor/compress/compression_task.py | 2 +- docs/src/user-guide/core-unstructured/clp.md | 17 +++++++++-------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index c83cd3fb5a..ed3a0b0660 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -29,9 +29,10 @@ class GlobalMetadataDBConfig { /** * Overloads operator >> to read MetadataDBType from an input stream. + * * @param in * @param metadata_db_type - * @return std::istream& reference to the input stream + * @return The input stream. */ friend auto operator>>(std::istream& in, MetadataDBType& metadata_db_type) -> std::istream&; @@ -41,21 +42,25 @@ class GlobalMetadataDBConfig { // Methods /** * Adds database-related command-line options to the given options_description. + * * @param options_description */ auto add_command_line_options(boost::program_options::options_description& options_description) -> void; /** - * Reads database credentials from environment variables if required by the database type. - * - CLP_DB_USER for database username - * - CLP_DB_PASS for database password + * Reads database credentials from the following environment variables, if required by the + * database type: + * + * - CLP_DB_USER for the database username + * - CLP_DB_PASS for the database password */ auto read_credentials_from_env_if_needed() -> void; /** * Validates that all required parameters are available and well-formed. - * @throw std::invalid_argument if validation fails + * + * @throw std::invalid_argument if validation fails. */ auto validate() const -> void; diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index bcc8d6492a..a083bfe6f4 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -462,7 +462,7 @@ def run_clp( archive_path, ] - # Set database credentials in environment variables for indexer + # Set environment variables for database credentials indexer_env = dict(os.environ) indexer_env.update( _get_db_connection_env_vars(clp_metadata_db_connection_config) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index 05da3db395..ffb48b6ae7 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -107,28 +107,29 @@ Currently, timestamps must be specified as milliseconds since the UNIX epoch. :::{warning} By default, `clp` uses an embedded SQLite database. Running multiple `clp` instances with SQLite -can fail due to "database is locked" error because SQLite does not support concurrent writes. +can fail due to the error "database is locked" because SQLite doesn't support concurrent writes. ::: + To enable parallel compression to the same archives directory, `clp`/`clg` can be configured to use a MySQL-type database (e.g., MariaDB) as follows: * Install and configure MariaDB using the instructions for your platform -* Create a user who has privileges to create databases, create tables, insert records, and delete +* Create a user that has privileges to create databases, create tables, insert records, and delete records. * Install the MariaDB Python package `pip3 install mariadb` * This is necessary to run the database initialization script. If you prefer, you can run the SQL statements in `tools/scripts/db/init-db.py` directly. -* Run `tools/scripts/db/init-db.py`. This will initialize the database CLP requires. Use - command-line options to configure MySQL connection parameters and set environment variables +* Run `tools/scripts/db/init-db.py`. This will initialize the database that CLP requires. Use + command-line options to configure database connection parameters and set environment variables for database credentials: * `--db-host ` to specify the database host (default: `"localhost"`) * `--db-port ` to specify the database port (default: `3306`) * `--db-name ` to specify the database name (default: `"clp-db"`) * `--db-table-prefix ` to specify the table prefix (default: `"clp_"`) - * Set `CLP_DB_USER` environment variable for the database username - * Set `CLP_DB_PASS` environment variable for the database password -* Run `clp` or `clg` with the same command options and environment variables, with the addition of - the database type command-line option: + * Set the `CLP_DB_USER` environment variable for the database username + * Set the `CLP_DB_PASS` environment variable for the database password +* Run `clp` or `clg` with the same command-line options and environment variables, with the addition + of the database type command-line option: * `--db-type mysql` to specify MySQL as the database type * To compress in parallel, simply run another instance of `clp` concurrently. From fe3115c3eda1a641d52327e997ea86c1d45a2539 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 13:29:20 -0400 Subject: [PATCH 041/108] style - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/tools/scripts/db/init-db.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/core/tools/scripts/db/init-db.py b/components/core/tools/scripts/db/init-db.py index 8bc7e98092..833d363872 100644 --- a/components/core/tools/scripts/db/init-db.py +++ b/components/core/tools/scripts/db/init-db.py @@ -48,7 +48,9 @@ def main(argv): try: # Create database try: - mysql_cursor.execute("CREATE DATABASE IF NOT EXISTS `{}` DEFAULT CHARACTER SET 'utf8'".format(db_name)) + mysql_cursor.execute( + "CREATE DATABASE IF NOT EXISTS `{}` DEFAULT CHARACTER SET 'utf8'".format(db_name) + ) except mariadb.Error as err: logger.error("Failed to create database - {}".format(err.msg)) return -1 From 015f5837289f86587c55805fac2dbe1c3542fc1e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 13:30:07 -0400 Subject: [PATCH 042/108] style - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/tools/scripts/db/init-db.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/core/tools/scripts/db/init-db.py b/components/core/tools/scripts/db/init-db.py index 833d363872..cecaef57cf 100644 --- a/components/core/tools/scripts/db/init-db.py +++ b/components/core/tools/scripts/db/init-db.py @@ -35,8 +35,7 @@ def main(argv): username = os.environ["CLP_DB_USER"] password = os.environ["CLP_DB_PASS"] except KeyError as e: - key = e.args[0] - raise Exception(f"'{key}' is missing from environment variables.") + raise Exception(f"Environment variable {e} hasn't been set.") try: mysql_conn = mariadb.connect(host=host, port=port, username=username, password=password) From 67f2d495c3f7320ba3f3e32b429fd1810b480c1d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 13:30:34 -0400 Subject: [PATCH 043/108] style - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 025491de03..89ffb9230b 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -6,6 +6,7 @@ #include #include "type_utils.hpp" + using std::invalid_argument; using std::string; From bec5805ef1bb2419ea299acd4601b23b6ee0eb06 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 13:31:07 -0400 Subject: [PATCH 044/108] Use `.at()` to access array - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 89ffb9230b..36889fc852 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -23,7 +23,7 @@ auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metada in >> db_type_string; for (size_t i = 0; i < GlobalMetadataDBConfig::cMetadataDBTypeNames.size(); ++i) { - if (GlobalMetadataDBConfig::cMetadataDBTypeNames[i] == db_type_string) { + if (GlobalMetadataDBConfig::cMetadataDBTypeNames.at(i) == db_type_string) { metadata_db_type = static_cast(i); return in; } From aab68bfd9ec92bebdaee670e020489c80ae6b800 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 11:39:38 -0400 Subject: [PATCH 045/108] build(core): remove CLP_NEED_YAMLCPP from set_clp_tests_dependencies --- components/core/cmake/Options/options.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/cmake/Options/options.cmake b/components/core/cmake/Options/options.cmake index 40b1c610e7..6666064e10 100644 --- a/components/core/cmake/Options/options.cmake +++ b/components/core/cmake/Options/options.cmake @@ -180,7 +180,6 @@ function(set_clp_tests_dependencies) CLP_NEED_SIMDJSON CLP_NEED_SPDLOG CLP_NEED_SQLITE - CLP_NEED_YAMLCPP CLP_NEED_YSTDLIB CLP_NEED_ZSTD ) From 6279484dc9e3775df7fc460a2d524a59dadc38d4 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 11:50:35 -0400 Subject: [PATCH 046/108] style(core): suppress clang-format and fix style --- .../core/src/clp/clg/CommandLineArguments.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/components/core/src/clp/clg/CommandLineArguments.cpp b/components/core/src/clp/clg/CommandLineArguments.cpp index 33f9325c6a..37f59b531e 100644 --- a/components/core/src/clp/clg/CommandLineArguments.cpp +++ b/components/core/src/clp/clg/CommandLineArguments.cpp @@ -41,13 +41,17 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { config_file_path += '/'; } config_file_path += cDefaultConfigFilename; - options_general.add_options()("help,h", "Print help")("version,V", "Print version")( - "config-file", - po::value(&config_file_path) - ->value_name("FILE") - ->default_value(config_file_path), - "Use configuration options from FILE" - ); + // clang-format off + options_general.add_options() + ("help,h", "Print help") + ("version,V", "Print version") + ( + "config-file", + po::value(&config_file_path)->value_name("FILE") + ->default_value(config_file_path), + "Use configuration options from FILE" + ); + // clang-format on m_metadata_db_config.add_command_line_options(options_general); // Define input options From 230629e2c5fc016523154f065aa9e2d5e43b0b25 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 13:28:41 -0400 Subject: [PATCH 047/108] Move table_prefix variable declaration to minimize scope --- .../job_orchestration/executor/compress/compression_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index a083bfe6f4..cf8995f247 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -431,7 +431,6 @@ def run_clp( # which case the process will still return success. proc.terminate() - table_prefix = clp_metadata_db_connection_config["table_prefix"] if s3_error is None: # We've started a new archive so add the previous archive's last reported size to # the total @@ -440,6 +439,7 @@ def run_clp( with closing(sql_adapter.create_connection(True)) as db_conn, closing( db_conn.cursor(dictionary=True) ) as db_cursor: + table_prefix = clp_metadata_db_connection_config["table_prefix"] if StorageEngine.CLP_S == clp_storage_engine: update_archive_metadata( db_cursor, table_prefix, dataset, last_archive_stats From dfde4fa9c044bafec1842b9c232396073abcc20b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 13:35:27 -0400 Subject: [PATCH 048/108] remove default values from metadata DB config descriptions --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 36889fc852..eb00c61cbc 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -59,25 +59,22 @@ auto GlobalMetadataDBConfig::add_command_line_options( boost::program_options::value(&m_metadata_db_port)->default_value( cDefaultMetadataDbPort), fmt::format( - "{} Database port (default: {})", - cMetadataDbTypeMysqlOptDescPrefix, - cDefaultMetadataDbPort + "{} Database port", + cMetadataDbTypeMysqlOptDescPrefix ).c_str()) ("db-name", boost::program_options::value(&m_metadata_db_name)->default_value( cDefaultMetadataDbName.data()), fmt::format( - "{} Database name (default: {})", - cMetadataDbTypeMysqlOptDescPrefix, - cDefaultMetadataDbName.data() + "{} Database name", + cMetadataDbTypeMysqlOptDescPrefix ).c_str()) ("db-table-prefix", boost::program_options::value(&m_metadata_table_prefix)->default_value( cDefaultMetadataTablePrefix.data()), fmt::format( - "{} Database table prefix (default: {})", - cMetadataDbTypeMysqlOptDescPrefix, - cDefaultMetadataTablePrefix.data() + "{} Database table prefix", + cMetadataDbTypeMysqlOptDescPrefix ).c_str()); } From ae6d1e05faa4eb67e7e40ecab2b40624a4ad6ca2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 13:41:50 -0400 Subject: [PATCH 049/108] refactor(core): move global metadata DB config constants --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 7 +++++++ .../core/src/clp/GlobalMetadataDBConfig.hpp | 18 ++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index eb00c61cbc..387b46a129 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -12,6 +12,13 @@ using std::string; namespace { // Constants +constexpr clp::GlobalMetadataDBConfig::MetadataDBType cDefaultMetadataDbType{ + clp::GlobalMetadataDBConfig::MetadataDBType::SQLite +}; +constexpr std::string_view cDefaultMetadataDbHost{"127.0.0.1"}; +constexpr int cDefaultMetadataDbPort{3306}; +constexpr std::string_view cDefaultMetadataDbName{"clp-db"}; +constexpr std::string_view cDefaultMetadataTablePrefix{"clp_"}; constexpr int cMinPort{1}; constexpr int cMaxPort{65'535}; } // namespace diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index ed3a0b0660..f627029ca6 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -90,18 +90,12 @@ class GlobalMetadataDBConfig { private: // Variables - static constexpr MetadataDBType cDefaultMetadataDbType{MetadataDBType::SQLite}; - static constexpr std::string_view cDefaultMetadataDbHost{"127.0.0.1"}; - static constexpr int cDefaultMetadataDbPort{3306}; - static constexpr std::string_view cDefaultMetadataDbName{"clp-db"}; - static constexpr std::string_view cDefaultMetadataTablePrefix{"clp_"}; - - MetadataDBType m_metadata_db_type{cDefaultMetadataDbType}; - - std::string m_metadata_db_host{cDefaultMetadataDbHost}; - int m_metadata_db_port{cDefaultMetadataDbPort}; - std::string m_metadata_db_name{cDefaultMetadataDbName}; - std::string m_metadata_table_prefix{cDefaultMetadataTablePrefix}; + MetadataDBType m_metadata_db_type; + + std::string m_metadata_db_host; + int m_metadata_db_port; + std::string m_metadata_db_name; + std::string m_metadata_table_prefix; std::string m_metadata_db_username; std::string m_metadata_db_password; From 667dce6e470fbe6b70f9585851cd868fea52e7ff Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 14:30:00 -0400 Subject: [PATCH 050/108] Move GlobalMetadataDBConfig initialization to constructor --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 4 +- .../core/src/clp/GlobalMetadataDBConfig.hpp | 9 ++- .../core/src/clp/clg/CommandLineArguments.cpp | 6 +- .../core/src/clp/clg/CommandLineArguments.hpp | 6 +- .../core/src/clp/clp/CommandLineArguments.cpp | 6 +- .../core/src/clp/clp/CommandLineArguments.hpp | 5 +- .../clp_s/indexer/CommandLineArguments.cpp | 3 +- .../tests/test-GlobalMetadataDBConfig.cpp | 62 ++++++++----------- 8 files changed, 45 insertions(+), 56 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 387b46a129..1ea45014b1 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -39,9 +39,9 @@ auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metada throw invalid_argument(fmt::format("Unknown database type: {}", db_type_string)); } -auto GlobalMetadataDBConfig::add_command_line_options( +GlobalMetadataDBConfig::GlobalMetadataDBConfig( boost::program_options::options_description& options_description -) -> void { +) { constexpr std::string_view cMetadataDbTypeMysqlOptDescPrefix{"[db-type=mysql]"}; options_description.add_options() diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index f627029ca6..93669b7e5d 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -37,17 +37,16 @@ class GlobalMetadataDBConfig { friend auto operator>>(std::istream& in, MetadataDBType& metadata_db_type) -> std::istream&; // Constructors - GlobalMetadataDBConfig() = default; - - // Methods /** * Adds database-related command-line options to the given options_description. * * @param options_description */ - auto add_command_line_options(boost::program_options::options_description& options_description) - -> void; + explicit GlobalMetadataDBConfig( + boost::program_options::options_description& options_description + ); + // Methods /** * Reads database credentials from the following environment variables, if required by the * database type: diff --git a/components/core/src/clp/clg/CommandLineArguments.cpp b/components/core/src/clp/clg/CommandLineArguments.cpp index 37f59b531e..26af9d7edf 100644 --- a/components/core/src/clp/clg/CommandLineArguments.cpp +++ b/components/core/src/clp/clg/CommandLineArguments.cpp @@ -52,7 +52,7 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { "Use configuration options from FILE" ); // clang-format on - m_metadata_db_config.add_command_line_options(options_general); + m_metadata_db_config.emplace(options_general); // Define input options po::options_description options_input("Input Options"); @@ -194,8 +194,8 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { // Initialize and validate global metadata DB config try { - m_metadata_db_config.read_credentials_from_env_if_needed(); - m_metadata_db_config.validate(); + m_metadata_db_config->read_credentials_from_env_if_needed(); + m_metadata_db_config->validate(); } catch (std::exception& e) { SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what()); return ParsingResult::Failure; diff --git a/components/core/src/clp/clg/CommandLineArguments.hpp b/components/core/src/clp/clg/CommandLineArguments.hpp index bbbdad19b3..d5c6b150e8 100644 --- a/components/core/src/clp/clg/CommandLineArguments.hpp +++ b/components/core/src/clp/clg/CommandLineArguments.hpp @@ -1,8 +1,8 @@ #ifndef CLP_CLG_COMMANDLINEARGUMENTS_HPP #define CLP_CLG_COMMANDLINEARGUMENTS_HPP +#include #include -#include #include @@ -46,7 +46,7 @@ class CommandLineArguments : public CommandLineArgumentsBase { epochtime_t get_search_end_ts() const { return m_search_end_ts; } - GlobalMetadataDBConfig const& get_metadata_db_config() const { return m_metadata_db_config; } + GlobalMetadataDBConfig const& get_metadata_db_config() const { return *m_metadata_db_config; } private: // Methods @@ -60,7 +60,7 @@ class CommandLineArguments : public CommandLineArgumentsBase { std::string m_file_path; OutputMethod m_output_method; epochtime_t m_search_begin_ts, m_search_end_ts; - GlobalMetadataDBConfig m_metadata_db_config; + std::optional m_metadata_db_config; }; } // namespace clp::clg diff --git a/components/core/src/clp/clp/CommandLineArguments.cpp b/components/core/src/clp/clp/CommandLineArguments.cpp index bf927e1af0..f387124fb0 100644 --- a/components/core/src/clp/clp/CommandLineArguments.cpp +++ b/components/core/src/clp/clp/CommandLineArguments.cpp @@ -48,7 +48,7 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { ->default_value(config_file_path), "Use configuration options from FILE" ); - m_metadata_db_config.add_command_line_options(options_general); + m_metadata_db_config.emplace(options_general); // Define functional options po::options_description options_functional("Input Options"); @@ -113,8 +113,8 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { // Initialize and validate global metadata DB config try { - m_metadata_db_config.read_credentials_from_env_if_needed(); - m_metadata_db_config.validate(); + m_metadata_db_config->read_credentials_from_env_if_needed(); + m_metadata_db_config->validate(); } catch (std::exception& e) { SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what()); return ParsingResult::Failure; diff --git a/components/core/src/clp/clp/CommandLineArguments.hpp b/components/core/src/clp/clp/CommandLineArguments.hpp index 6e14a4b3b3..0ca8ad40d7 100644 --- a/components/core/src/clp/clp/CommandLineArguments.hpp +++ b/components/core/src/clp/clp/CommandLineArguments.hpp @@ -1,6 +1,7 @@ #ifndef CLP_CLP_COMMANDLINEARGUMENTS_HPP #define CLP_CLP_COMMANDLINEARGUMENTS_HPP +#include #include #include @@ -73,7 +74,7 @@ class CommandLineArguments : public CommandLineArgumentsBase { size_t get_ir_target_size() const { return m_ir_target_size; } - GlobalMetadataDBConfig const& get_metadata_db_config() const { return m_metadata_db_config; } + GlobalMetadataDBConfig const& get_metadata_db_config() const { return *m_metadata_db_config; } private: // Methods @@ -100,7 +101,7 @@ class CommandLineArguments : public CommandLineArgumentsBase { Command m_command; std::string m_archives_dir; std::vector m_input_paths; - GlobalMetadataDBConfig m_metadata_db_config; + std::optional m_metadata_db_config; }; } // namespace clp::clp diff --git a/components/core/src/clp_s/indexer/CommandLineArguments.cpp b/components/core/src/clp_s/indexer/CommandLineArguments.cpp index fa1b72d1e9..e4e8d0a6ad 100644 --- a/components/core/src/clp_s/indexer/CommandLineArguments.cpp +++ b/components/core/src/clp_s/indexer/CommandLineArguments.cpp @@ -29,8 +29,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { "Create the column metadata table if it doesn't exist" ); // clang-format on - clp::GlobalMetadataDBConfig metadata_db_config; - metadata_db_config.add_command_line_options(output_options); + clp::GlobalMetadataDBConfig metadata_db_config{output_options}; // Define visible options po::options_description visible_options; diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 05283d321d..8db3775edc 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -16,9 +16,9 @@ void unset_env_var(char const* name) { } template -auto parse_args(std::array const& argv, GlobalMetadataDBConfig& config) -> void { +auto parse_args(std::array const& argv) -> GlobalMetadataDBConfig { boost::program_options::options_description options_desc; - config.add_command_line_options(options_desc); + GlobalMetadataDBConfig config(options_desc); boost::program_options::variables_map vm; constexpr auto argc{static_cast(N)}; @@ -31,6 +31,8 @@ auto parse_args(std::array const& argv, GlobalMetadataDBConfig& vm ); boost::program_options::notify(vm); + + return config; } } // namespace @@ -51,8 +53,7 @@ TEST_CASE( "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config; - parse_args(argv, config); + GlobalMetadataDBConfig config{parse_args(argv)}; REQUIRE(config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL); REQUIRE(config.get_metadata_db_host() == "test-host"); @@ -76,8 +77,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config; - parse_args(argv, config); + GlobalMetadataDBConfig config{parse_args(argv)}; SECTION("With valid credentials") { set_env_var("CLP_DB_USER", "test-user"); @@ -134,12 +134,11 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config_low_port; - parse_args(argv_with_low_port, config_low_port); + GlobalMetadataDBConfig config{parse_args(argv_with_low_port)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); - config_low_port.read_credentials_from_env_if_needed(); - REQUIRE_THROWS_AS(config_low_port.validate(), std::invalid_argument); + config.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); unset_env_var("CLP_DB_USER"); unset_env_var("CLP_DB_PASS"); } @@ -158,12 +157,11 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config_high_port; - parse_args(argv_with_high_port, config_high_port); + GlobalMetadataDBConfig config{parse_args(argv_with_high_port)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); - config_high_port.read_credentials_from_env_if_needed(); - REQUIRE_THROWS_AS(config_high_port.validate(), std::invalid_argument); + config.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); unset_env_var("CLP_DB_USER"); unset_env_var("CLP_DB_PASS"); } @@ -184,12 +182,11 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config_empty_host; - parse_args(argv_empty_host, config_empty_host); + GlobalMetadataDBConfig config{parse_args(argv_empty_host)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); - config_empty_host.read_credentials_from_env_if_needed(); - REQUIRE_THROWS_AS(config_empty_host.validate(), std::invalid_argument); + config.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); unset_env_var("CLP_DB_USER"); unset_env_var("CLP_DB_PASS"); } @@ -208,12 +205,11 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config_empty_name; - parse_args(argv_empty_name, config_empty_name); + GlobalMetadataDBConfig config{parse_args(argv_empty_name)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); - config_empty_name.read_credentials_from_env_if_needed(); - REQUIRE_THROWS_AS(config_empty_name.validate(), std::invalid_argument); + config.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); unset_env_var("CLP_DB_USER"); unset_env_var("CLP_DB_PASS"); } @@ -232,12 +228,11 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "" }; - GlobalMetadataDBConfig config_empty_prefix; - parse_args(argv_empty_prefix, config_empty_prefix); + GlobalMetadataDBConfig config{parse_args(argv_empty_prefix)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); - config_empty_prefix.read_credentials_from_env_if_needed(); - REQUIRE_THROWS_AS(config_empty_prefix.validate(), std::invalid_argument); + config.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); unset_env_var("CLP_DB_USER"); unset_env_var("CLP_DB_PASS"); } @@ -247,41 +242,36 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-host argument") { constexpr std::array argv{"test", "--db-type", "sqlite", "--db-host", "test-host"}; - GlobalMetadataDBConfig config; + GlobalMetadataDBConfig config{parse_args(argv)}; - REQUIRE_NOTHROW(parse_args(argv, config)); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-port argument") { constexpr std::array argv{"test", "--db-type", "sqlite", "--db-port", "8888"}; - GlobalMetadataDBConfig config; + GlobalMetadataDBConfig config{parse_args(argv)}; - REQUIRE_NOTHROW(parse_args(argv, config)); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-name argument") { constexpr std::array argv{"test", "--db-type", "sqlite", "--db-name", "test-db"}; - GlobalMetadataDBConfig config; + GlobalMetadataDBConfig config{parse_args(argv)}; - REQUIRE_NOTHROW(parse_args(argv, config)); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-table-prefix argument") { constexpr std::array argv{"test", "--db-type", "sqlite", "--db-table-prefix", "test_prefix_"}; - GlobalMetadataDBConfig config; + GlobalMetadataDBConfig config{parse_args(argv)}; - REQUIRE_NOTHROW(parse_args(argv, config)); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With username and password") { constexpr std::array argv{"test", "--db-type", "sqlite"}; - GlobalMetadataDBConfig config; - parse_args(argv, config); + GlobalMetadataDBConfig config{parse_args(argv)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); From 57cf478286e3f5bb04145e4e7577c5cf48a257f1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 15:11:26 -0400 Subject: [PATCH 051/108] suppress clang-format; use string() instead of string_view::data() for passing string to boost utils. --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 93 +++++++++++-------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 1ea45014b1..0f44794b0d 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -44,45 +44,62 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( ) { constexpr std::string_view cMetadataDbTypeMysqlOptDescPrefix{"[db-type=mysql]"}; + // clang-format off options_description.add_options() - ("db-type", - boost::program_options::value(&m_metadata_db_type) - ->default_value(cDefaultMetadataDbType, - cMetadataDBTypeNames[enum_to_underlying_type(cDefaultMetadataDbType)]. - data()), - fmt::format("Database type [{} | {}]", - cMetadataDBTypeNames - [enum_to_underlying_type(MetadataDBType::SQLite)], - cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MySQL)]).c_str() - ) - ("db-host", - boost::program_options::value(&m_metadata_db_host)->default_value( - cDefaultMetadataDbHost.data()), - fmt::format( - "{} Database host", - cMetadataDbTypeMysqlOptDescPrefix - ).c_str()) - ("db-port", - boost::program_options::value(&m_metadata_db_port)->default_value( - cDefaultMetadataDbPort), - fmt::format( - "{} Database port", - cMetadataDbTypeMysqlOptDescPrefix - ).c_str()) - ("db-name", - boost::program_options::value(&m_metadata_db_name)->default_value( - cDefaultMetadataDbName.data()), - fmt::format( - "{} Database name", - cMetadataDbTypeMysqlOptDescPrefix - ).c_str()) - ("db-table-prefix", - boost::program_options::value(&m_metadata_table_prefix)->default_value( - cDefaultMetadataTablePrefix.data()), - fmt::format( - "{} Database table prefix", - cMetadataDbTypeMysqlOptDescPrefix - ).c_str()); + ( + "db-type", + boost::program_options::value(&m_metadata_db_type) + ->default_value( + cDefaultMetadataDbType, + string( + cMetadataDBTypeNames[enum_to_underlying_type( + cDefaultMetadataDbType + )] + ) + ), + fmt::format( + "Database type [{} | {}]", + cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::SQLite)], + cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MySQL)] + ).c_str() + ) + ( + "db-host", + boost::program_options::value(&m_metadata_db_host) + ->default_value(string(cDefaultMetadataDbHost)), + fmt::format( + "{} Database host", + cMetadataDbTypeMysqlOptDescPrefix + ).c_str() + ) + ( + "db-port", + boost::program_options::value(&m_metadata_db_port) + ->default_value(cDefaultMetadataDbPort), + fmt::format( + "{} Database port", + cMetadataDbTypeMysqlOptDescPrefix + ).c_str() + ) + ( + "db-name", + boost::program_options::value(&m_metadata_db_name) + ->default_value(string(cDefaultMetadataDbName)), + fmt::format( + "{} Database name", + cMetadataDbTypeMysqlOptDescPrefix + ).c_str() + ) + ( + "db-table-prefix", + boost::program_options::value(&m_metadata_table_prefix) + ->default_value(string(cDefaultMetadataTablePrefix)), + fmt::format( + "{} Database table prefix", + cMetadataDbTypeMysqlOptDescPrefix + ).c_str() + ); + // clang-format on } auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { From 0f6ca8561831c45b2bc2d6ba591fa80e86acebf8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 16:44:28 -0400 Subject: [PATCH 052/108] Update GlobalMetadataDBConfig to use optional for username and password --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 14 ++++++++------ components/core/src/clp/GlobalMetadataDBConfig.hpp | 9 +++++---- components/core/src/clp/clg/clg.cpp | 4 ++-- components/core/src/clp/clp/compression.cpp | 4 ++-- components/core/src/clp/clp/utils.cpp | 4 ++-- .../core/tests/test-GlobalMetadataDBConfig.cpp | 10 ++++++++-- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 0f44794b0d..2b674dd196 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -110,12 +110,12 @@ auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { auto const* db_username{std::getenv("CLP_DB_USER")}; if (nullptr != db_username) { - m_metadata_db_username = db_username; + m_metadata_db_username.emplace(db_username); } auto const* db_password{std::getenv("CLP_DB_PASS")}; if (nullptr != db_password) { - m_metadata_db_password = db_password; + m_metadata_db_password.emplace(db_password); } } @@ -126,7 +126,7 @@ auto GlobalMetadataDBConfig::validate() const -> void { || cDefaultMetadataDbPort != m_metadata_db_port || cDefaultMetadataDbName != m_metadata_db_name || cDefaultMetadataTablePrefix != m_metadata_table_prefix - || false == m_metadata_db_username.empty() || false == m_metadata_db_password.empty()) + || m_metadata_db_username.has_value() || m_metadata_db_password.has_value()) { throw invalid_argument( "MySQL-specific parameters cannot be used with --db-type='sqlite'." @@ -159,10 +159,12 @@ auto GlobalMetadataDBConfig::validate() const -> void { throw invalid_argument("Database '--db-table_prefix' is empty."); } - if (m_metadata_db_username.empty()) { - throw invalid_argument("Database 'CLP_DB_USER' not specified or empty."); + if (false == m_metadata_db_username.has_value()) { + throw invalid_argument("Database 'CLP_DB_USER' not specified."); } - // NOTE: No need to check `m_metadata_db_password` because `CLP_DB_PASS` is optional. + if (false == m_metadata_db_password.has_value()) { + throw invalid_argument("Database 'CLP_DB_PASS' not specified."); + } } } // namespace clp diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index 93669b7e5d..0e343601a3 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -2,6 +2,7 @@ #define CLP_GLOBALMETADATADBCONFIG_HPP #include +#include #include #include @@ -75,11 +76,11 @@ class GlobalMetadataDBConfig { return m_metadata_db_name; } - [[nodiscard]] auto get_metadata_db_username() const -> std::string const& { + [[nodiscard]] auto get_metadata_db_username() const -> std::optional const& { return m_metadata_db_username; } - [[nodiscard]] auto get_metadata_db_password() const -> std::string const& { + [[nodiscard]] auto get_metadata_db_password() const -> std::optional const& { return m_metadata_db_password; } @@ -96,8 +97,8 @@ class GlobalMetadataDBConfig { std::string m_metadata_db_name; std::string m_metadata_table_prefix; - std::string m_metadata_db_username; - std::string m_metadata_db_password; + std::optional m_metadata_db_username; + std::optional m_metadata_db_password; }; } // namespace clp diff --git a/components/core/src/clp/clg/clg.cpp b/components/core/src/clp/clg/clg.cpp index 04d5e0a901..cdd92e1b33 100644 --- a/components/core/src/clp/clg/clg.cpp +++ b/components/core/src/clp/clg/clg.cpp @@ -550,8 +550,8 @@ int main(int argc, char const* argv[]) { global_metadata_db = std::make_unique( global_metadata_db_config.get_metadata_db_host(), global_metadata_db_config.get_metadata_db_port(), - global_metadata_db_config.get_metadata_db_username(), - global_metadata_db_config.get_metadata_db_password(), + global_metadata_db_config.get_metadata_db_username().value_or(""), + global_metadata_db_config.get_metadata_db_password().value_or(""), global_metadata_db_config.get_metadata_db_name(), global_metadata_db_config.get_metadata_table_prefix() ); diff --git a/components/core/src/clp/clp/compression.cpp b/components/core/src/clp/clp/compression.cpp index 14333afcc9..cd90141df5 100644 --- a/components/core/src/clp/clp/compression.cpp +++ b/components/core/src/clp/clp/compression.cpp @@ -85,8 +85,8 @@ bool compress( global_metadata_db = std::make_unique( global_metadata_db_config.get_metadata_db_host(), global_metadata_db_config.get_metadata_db_port(), - global_metadata_db_config.get_metadata_db_username(), - global_metadata_db_config.get_metadata_db_password(), + global_metadata_db_config.get_metadata_db_username().value_or(""), + global_metadata_db_config.get_metadata_db_password().value_or(""), global_metadata_db_config.get_metadata_db_name(), global_metadata_db_config.get_metadata_table_prefix() ); diff --git a/components/core/src/clp/clp/utils.cpp b/components/core/src/clp/clp/utils.cpp index 123f9a8369..9985ada221 100644 --- a/components/core/src/clp/clp/utils.cpp +++ b/components/core/src/clp/clp/utils.cpp @@ -187,8 +187,8 @@ std::unique_ptr get_global_metadata_db( return std::make_unique( global_metadata_db_config.get_metadata_db_host(), global_metadata_db_config.get_metadata_db_port(), - global_metadata_db_config.get_metadata_db_username(), - global_metadata_db_config.get_metadata_db_password(), + global_metadata_db_config.get_metadata_db_username().value_or(""), + global_metadata_db_config.get_metadata_db_password().value_or(""), global_metadata_db_config.get_metadata_db_name(), global_metadata_db_config.get_metadata_table_prefix() ); diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 8db3775edc..4d7de69460 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -116,6 +116,12 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); unset_env_var("CLP_DB_PASS"); } + SECTION("Username set but password missing") { + set_env_var("CLP_DB_USER", "test-user"); + config.read_credentials_from_env_if_needed(); + REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); + unset_env_var("CLP_DB_USER"); + } } } @@ -277,8 +283,8 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); - REQUIRE(config.get_metadata_db_username().empty()); - REQUIRE(config.get_metadata_db_password().empty()); + REQUIRE_FALSE(config.get_metadata_db_username().has_value()); + REQUIRE_FALSE(config.get_metadata_db_password().has_value()); REQUIRE_NOTHROW(config.validate()); unset_env_var("CLP_DB_USER"); From 8df0d2f1771a89760e0ade1f37c238172c04e572 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 17:11:32 -0400 Subject: [PATCH 053/108] update CLP database initialization documentation to remove default values --- docs/src/user-guide/core-unstructured/clp.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index ffb48b6ae7..b2299e5ae8 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -121,11 +121,11 @@ a MySQL-type database (e.g., MariaDB) as follows: statements in `tools/scripts/db/init-db.py` directly. * Run `tools/scripts/db/init-db.py`. This will initialize the database that CLP requires. Use command-line options to configure database connection parameters and set environment variables - for database credentials: - * `--db-host ` to specify the database host (default: `"localhost"`) - * `--db-port ` to specify the database port (default: `3306`) - * `--db-name ` to specify the database name (default: `"clp-db"`) - * `--db-table-prefix ` to specify the table prefix (default: `"clp_"`) + for database credentials. If a command-line option isn't specified, the default value is used: + * `--db-host ` to specify the database host + * `--db-port ` to specify the database port + * `--db-name ` to specify the database name + * `--db-table-prefix ` to specify the table prefix * Set the `CLP_DB_USER` environment variable for the database username * Set the `CLP_DB_PASS` environment variable for the database password * Run `clp` or `clg` with the same command-line options and environment variables, with the addition From f07d036335af61a24eb8bd4f092ed4758bc1a22b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 7 Aug 2025 17:13:45 -0400 Subject: [PATCH 054/108] fix compile issue --- components/core/src/clp_s/indexer/IndexManager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp_s/indexer/IndexManager.cpp b/components/core/src/clp_s/indexer/IndexManager.cpp index c22f7ab8b7..46ba6bb67a 100644 --- a/components/core/src/clp_s/indexer/IndexManager.cpp +++ b/components/core/src/clp_s/indexer/IndexManager.cpp @@ -17,8 +17,8 @@ IndexManager::IndexManager( m_mysql_index_storage = std::make_unique( db_config->get_metadata_db_host(), db_config->get_metadata_db_port(), - db_config->get_metadata_db_username(), - db_config->get_metadata_db_password(), + db_config->get_metadata_db_username().value(), + db_config->get_metadata_db_password().value(), db_config->get_metadata_db_name(), db_config->get_metadata_table_prefix() ); From 7f52c1f552f84c7fc026058ffa514f83caee3c03 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 8 Aug 2025 01:03:04 -0400 Subject: [PATCH 055/108] fit clang-tidy issues in test-GlobalMetadataDBConfig.cpp --- .../tests/test-GlobalMetadataDBConfig.cpp | 84 ++++++++++--------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 4d7de69460..67db608083 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -1,5 +1,11 @@ +#include #include +#include +#include +#include +#include +#include #include #include "../src/clp/GlobalMetadataDBConfig.hpp" @@ -8,23 +14,23 @@ using namespace clp; namespace { void set_env_var(char const* name, char const* value) { - setenv(name, value, 1); + setenv(name, value, 1); // NOLINT(concurrency-mt-unsafe) } void unset_env_var(char const* name) { - unsetenv(name); + unsetenv(name); // NOLINT(concurrency-mt-unsafe) } -template -auto parse_args(std::array const& argv) -> GlobalMetadataDBConfig { +template +auto parse_args(std::array const& argv) -> GlobalMetadataDBConfig { boost::program_options::options_description options_desc; GlobalMetadataDBConfig config(options_desc); boost::program_options::variables_map vm; - constexpr auto argc{static_cast(N)}; + constexpr auto cArgc{static_cast(n)}; boost::program_options::store( boost::program_options::parse_command_line( - argc, + cArgc, const_cast(argv.data()), options_desc ), @@ -40,7 +46,7 @@ TEST_CASE( "Test parsing command line arguments for GlobalMetadataDBConfig", "[GlobalMetadataDBConfig]" ) { - constexpr std::array argv{ + constexpr std::array cArgv{ "test", "--db-type", "mysql", @@ -53,18 +59,18 @@ TEST_CASE( "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config{parse_args(argv)}; + auto const config{parse_args(cArgv)}; - REQUIRE(config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL); - REQUIRE(config.get_metadata_db_host() == "test-host"); - REQUIRE(config.get_metadata_db_port() == 8888); - REQUIRE(config.get_metadata_db_name() == "test-db"); - REQUIRE(config.get_metadata_table_prefix() == "test_prefix_"); + REQUIRE((config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL)); + REQUIRE((config.get_metadata_db_host() == "test-host")); + REQUIRE((config.get_metadata_db_port() == 8888)); + REQUIRE((config.get_metadata_db_name() == "test-db")); + REQUIRE((config.get_metadata_table_prefix() == "test_prefix_")); } TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBConfig]") { SECTION("With all arguments") { - constexpr std::array argv{ + constexpr std::array cArgv{ "test", "--db-type", "mysql", @@ -77,7 +83,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config{parse_args(argv)}; + auto config{parse_args(cArgv)}; SECTION("With valid credentials") { set_env_var("CLP_DB_USER", "test-user"); @@ -85,8 +91,8 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); - REQUIRE(config.get_metadata_db_username() == "test-user"); - REQUIRE(config.get_metadata_db_password() == "test-pass"); + REQUIRE((config.get_metadata_db_username() == "test-user")); + REQUIRE((config.get_metadata_db_password() == "test-pass")); unset_env_var("CLP_DB_USER"); unset_env_var("CLP_DB_PASS"); @@ -98,8 +104,8 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); - REQUIRE(config.get_metadata_db_username() == "test-user"); - REQUIRE(config.get_metadata_db_password() == ""); + REQUIRE((config.get_metadata_db_username() == "test-user")); + REQUIRE((config.get_metadata_db_password() == "")); unset_env_var("CLP_DB_USER"); unset_env_var("CLP_DB_PASS"); @@ -127,7 +133,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With invalid port values") { SECTION("Port too low") { - constexpr std::array argv_with_low_port{ + constexpr std::array cArgvWithLowPort{ "test", "--db-type", "mysql", @@ -140,7 +146,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config{parse_args(argv_with_low_port)}; + auto config{parse_args(cArgvWithLowPort)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); @@ -150,7 +156,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Port too high") { - constexpr std::array argv_with_high_port{ + constexpr std::array cArgvWithHighPort{ "test", "--db-type", "mysql", @@ -163,7 +169,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config{parse_args(argv_with_high_port)}; + auto config{parse_args(cArgvWithHighPort)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); @@ -175,7 +181,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With empty required arguments") { SECTION("Empty db-host") { - constexpr std::array argv_empty_host{ + constexpr std::array cArgvEmptyHost{ "test", "--db-type", "mysql", @@ -188,7 +194,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config{parse_args(argv_empty_host)}; + auto config{parse_args(cArgvEmptyHost)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); @@ -198,7 +204,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Empty db-name") { - constexpr std::array argv_empty_name{ + constexpr std::array cArgvEmptyName{ "test", "--db-type", "mysql", @@ -211,7 +217,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - GlobalMetadataDBConfig config{parse_args(argv_empty_name)}; + auto config{parse_args(cArgvEmptyName)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); @@ -221,7 +227,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Empty db-table-prefix") { - constexpr std::array argv_empty_prefix{ + constexpr std::array cArgvEmptyPrefix{ "test", "--db-type", "mysql", @@ -234,7 +240,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "" }; - GlobalMetadataDBConfig config{parse_args(argv_empty_prefix)}; + auto config{parse_args(cArgvEmptyPrefix)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); @@ -247,37 +253,37 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-host argument") { - constexpr std::array argv{"test", "--db-type", "sqlite", "--db-host", "test-host"}; - GlobalMetadataDBConfig config{parse_args(argv)}; + constexpr std::array cArgv{"test", "--db-type", "sqlite", "--db-host", "test-host"}; + auto const config{parse_args(cArgv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-port argument") { - constexpr std::array argv{"test", "--db-type", "sqlite", "--db-port", "8888"}; - GlobalMetadataDBConfig config{parse_args(argv)}; + constexpr std::array cArgv{"test", "--db-type", "sqlite", "--db-port", "8888"}; + auto const config{parse_args(cArgv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-name argument") { - constexpr std::array argv{"test", "--db-type", "sqlite", "--db-name", "test-db"}; - GlobalMetadataDBConfig config{parse_args(argv)}; + constexpr std::array cArgv{"test", "--db-type", "sqlite", "--db-name", "test-db"}; + auto const config{parse_args(cArgv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-table-prefix argument") { constexpr std::array - argv{"test", "--db-type", "sqlite", "--db-table-prefix", "test_prefix_"}; - GlobalMetadataDBConfig config{parse_args(argv)}; + cArgv{"test", "--db-type", "sqlite", "--db-table-prefix", "test_prefix_"}; + auto const config{parse_args(cArgv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With username and password") { - constexpr std::array argv{"test", "--db-type", "sqlite"}; - GlobalMetadataDBConfig config{parse_args(argv)}; + constexpr std::array cArgv{"test", "--db-type", "sqlite"}; + auto config{parse_args(cArgv)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); From 8f6aaf9537e15b601ed097e3e10b45e75a1122b8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 8 Aug 2025 01:33:05 -0400 Subject: [PATCH 056/108] fit clang-tidy issues in GlobalMetadataDBConfig.cpp --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 2b674dd196..a70f987264 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -1,8 +1,13 @@ #include "GlobalMetadataDBConfig.hpp" #include +#include #include +#include +#include +#include +#include #include #include "type_utils.hpp" @@ -41,7 +46,12 @@ auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metada GlobalMetadataDBConfig::GlobalMetadataDBConfig( boost::program_options::options_description& options_description -) { +) + : m_metadata_db_type{cDefaultMetadataDbType}, + m_metadata_db_host{cDefaultMetadataDbHost}, + m_metadata_db_port{cDefaultMetadataDbPort}, + m_metadata_db_name{cDefaultMetadataDbName}, + m_metadata_table_prefix{cDefaultMetadataTablePrefix} { constexpr std::string_view cMetadataDbTypeMysqlOptDescPrefix{"[db-type=mysql]"}; // clang-format off @@ -108,12 +118,12 @@ auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { return; } - auto const* db_username{std::getenv("CLP_DB_USER")}; + auto const* db_username{std::getenv("CLP_DB_USER")}; // NOLINT(concurrency-mt-unsafe) if (nullptr != db_username) { m_metadata_db_username.emplace(db_username); } - auto const* db_password{std::getenv("CLP_DB_PASS")}; + auto const* db_password{std::getenv("CLP_DB_PASS")}; // NOLINT(concurrency-mt-unsafe) if (nullptr != db_password) { m_metadata_db_password.emplace(db_password); } From e1548995b5295b05b9d68294e58c06d99896dcf2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 09:58:19 -0400 Subject: [PATCH 057/108] Docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- docs/src/user-guide/core-unstructured/clp.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index b2299e5ae8..af6a1702f3 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -107,7 +107,8 @@ Currently, timestamps must be specified as milliseconds since the UNIX epoch. :::{warning} By default, `clp` uses an embedded SQLite database. Running multiple `clp` instances with SQLite -can fail due to the error "database is locked" because SQLite doesn't support concurrent writes. +(rather than a MySQL-type database) can fail due to the error "database is locked" because SQLite +doesn't support concurrent writes. ::: To enable parallel compression to the same archives directory, `clp`/`clg` can be configured to use @@ -116,18 +117,18 @@ a MySQL-type database (e.g., MariaDB) as follows: * Install and configure MariaDB using the instructions for your platform * Create a user that has privileges to create databases, create tables, insert records, and delete records. -* Install the MariaDB Python package `pip3 install mariadb` +* Install the MariaDB Python package: `pip3 install mariadb` * This is necessary to run the database initialization script. If you prefer, you can run the SQL statements in `tools/scripts/db/init-db.py` directly. -* Run `tools/scripts/db/init-db.py`. This will initialize the database that CLP requires. Use - command-line options to configure database connection parameters and set environment variables - for database credentials. If a command-line option isn't specified, the default value is used: +* Run `tools/scripts/db/init-db.py` to initialize the database that CLP requires. Use the following + command-line options to configure database connection parameters and set environment variables for + database credentials. If a command-line option isn't specified, its default value will be used: * `--db-host ` to specify the database host * `--db-port ` to specify the database port * `--db-name ` to specify the database name * `--db-table-prefix ` to specify the table prefix - * Set the `CLP_DB_USER` environment variable for the database username - * Set the `CLP_DB_PASS` environment variable for the database password + * Set the `CLP_DB_USER` environment variable for the database user's username + * Set the `CLP_DB_PASS` environment variable for the database user's password * Run `clp` or `clg` with the same command-line options and environment variables, with the addition of the database type command-line option: * `--db-type mysql` to specify MySQL as the database type From 85e00ce20788f40b8157303dc07cdb66fef9bc09 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 09:59:41 -0400 Subject: [PATCH 058/108] rename to improve clarity of database connection functions --- .../executor/compress/compression_task.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index cf8995f247..7780405c7d 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -182,7 +182,9 @@ def _upload_archive_to_s3( s3_put(s3_config, archive_src_path, dest_path) -def _get_db_connection_args(clp_metadata_db_connection_config: Dict[str, Any]) -> List[str]: +def _get_db_connection_args_for_clp_cmd( + clp_metadata_db_connection_config: Dict[str, Any], +) -> List[str]: """ :param clp_metadata_db_connection_config: :return: List of database connection arguments for the clp command. @@ -201,7 +203,7 @@ def _get_db_connection_args(clp_metadata_db_connection_config: Dict[str, Any]) - ] -def _get_db_connection_env_vars( +def _get_db_connection_env_vars_for_clp_cmd( clp_metadata_db_connection_config: Dict[str, Any], ) -> Dict[str, str]: """ @@ -253,9 +255,11 @@ def _make_clp_command_and_env( compression_cmd.append(str(schema_path)) # Set database connection parameters - compression_cmd.extend(_get_db_connection_args(clp_metadata_db_connection_config)) + compression_cmd.extend(_get_db_connection_args_for_clp_cmd(clp_metadata_db_connection_config)) compression_env_vars = dict(os.environ) - compression_env_vars.update(_get_db_connection_env_vars(clp_metadata_db_connection_config)) + compression_env_vars.update( + _get_db_connection_env_vars_for_clp_cmd(clp_metadata_db_connection_config) + ) return compression_cmd, compression_env_vars @@ -457,7 +461,7 @@ def run_clp( if StorageEngine.CLP_S == clp_storage_engine: indexer_cmd = [ str(clp_home / "bin" / "indexer"), - *_get_db_connection_args(clp_metadata_db_connection_config), + *_get_db_connection_args_for_clp_cmd(clp_metadata_db_connection_config), dataset, archive_path, ] @@ -465,7 +469,7 @@ def run_clp( # Set environment variables for database credentials indexer_env = dict(os.environ) indexer_env.update( - _get_db_connection_env_vars(clp_metadata_db_connection_config) + _get_db_connection_env_vars_for_clp_cmd(clp_metadata_db_connection_config) ) try: From 4e829fff567b920965df6c64f5fc3e52352fac83 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 10:01:09 -0400 Subject: [PATCH 059/108] Change mysql-only options desc prefix - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index a70f987264..578b6bb66e 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -52,7 +52,7 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( m_metadata_db_port{cDefaultMetadataDbPort}, m_metadata_db_name{cDefaultMetadataDbName}, m_metadata_table_prefix{cDefaultMetadataTablePrefix} { - constexpr std::string_view cMetadataDbTypeMysqlOptDescPrefix{"[db-type=mysql]"}; + constexpr std::string_view cMetadataDbTypeMysqlOptDescPrefix{"(--db-type=mysql only)"}; // clang-format off options_description.add_options() From 8c0ff24e379037b82139acbcf2d639c6add60b04 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 10:01:26 -0400 Subject: [PATCH 060/108] Reformat - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 578b6bb66e..21f75a8fa2 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -62,9 +62,9 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( ->default_value( cDefaultMetadataDbType, string( - cMetadataDBTypeNames[enum_to_underlying_type( - cDefaultMetadataDbType - )] + cMetadataDBTypeNames[ + enum_to_underlying_type(cDefaultMetadataDbType) + ] ) ), fmt::format( From ba9a0ac67fd6a7c0e43006707ce05c385eb75853 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 10:02:14 -0400 Subject: [PATCH 061/108] Docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 21f75a8fa2..fc6ec1ba83 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -131,7 +131,7 @@ auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { auto GlobalMetadataDBConfig::validate() const -> void { if (m_metadata_db_type == MetadataDBType::SQLite) { - // SQLite does not require extra parameters. + // SQLite doesn't require extra parameters. if (cDefaultMetadataDbHost != m_metadata_db_host || cDefaultMetadataDbPort != m_metadata_db_port || cDefaultMetadataDbName != m_metadata_db_name From 91fe77143de9d05d24a052cae875315de36d372c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 12:34:06 -0400 Subject: [PATCH 062/108] Use init list for declartion - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/tests/test-GlobalMetadataDBConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 67db608083..aeffb7df4a 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -24,7 +24,7 @@ void unset_env_var(char const* name) { template auto parse_args(std::array const& argv) -> GlobalMetadataDBConfig { boost::program_options::options_description options_desc; - GlobalMetadataDBConfig config(options_desc); + GlobalMetadataDBConfig config{options_desc}; boost::program_options::variables_map vm; constexpr auto cArgc{static_cast(n)}; From 179df885337b80f3ab395683ddb1cad5e4cfcd10 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 12:34:49 -0400 Subject: [PATCH 063/108] Improve exception messages - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index fc6ec1ba83..296d5e5a50 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -139,7 +139,7 @@ auto GlobalMetadataDBConfig::validate() const -> void { || m_metadata_db_username.has_value() || m_metadata_db_password.has_value()) { throw invalid_argument( - "MySQL-specific parameters cannot be used with --db-type='sqlite'." + "MySQL-specific parameters cannot be used with --db-type=sqlite." " Please remove them or set '--db-type=mysql'." ); } @@ -170,11 +170,11 @@ auto GlobalMetadataDBConfig::validate() const -> void { } if (false == m_metadata_db_username.has_value()) { - throw invalid_argument("Database 'CLP_DB_USER' not specified."); + throw invalid_argument("Environment variable 'CLP_DB_USER' not set."); } if (false == m_metadata_db_password.has_value()) { - throw invalid_argument("Database 'CLP_DB_PASS' not specified."); + throw invalid_argument("Environment variable 'CLP_DB_PASS' not set."); } } } // namespace clp From d7e6afb6f77045d9bc8cb560c5006b607eb900e9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 12:35:32 -0400 Subject: [PATCH 064/108] Avoid inline comment & add rationale for lint suppression - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 296d5e5a50..6b45115b92 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -118,7 +118,9 @@ auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { return; } - auto const* db_username{std::getenv("CLP_DB_USER")}; // NOLINT(concurrency-mt-unsafe) + // Silence the check since this class won't be used in a multithreaded context. + // NOLINTNEXTLINE(concurrency-mt-unsafe) + auto const* db_username{std::getenv("CLP_DB_USER")}; if (nullptr != db_username) { m_metadata_db_username.emplace(db_username); } From 04c30ea99b4fd2de756b4b718366fcc61561ee66 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 12:37:47 -0400 Subject: [PATCH 065/108] Use NOLINTNEXTLINE comment to suppress the concurrency-mt-unsafe warning --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 6b45115b92..eb3b3f3e0f 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -125,7 +125,9 @@ auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { m_metadata_db_username.emplace(db_username); } - auto const* db_password{std::getenv("CLP_DB_PASS")}; // NOLINT(concurrency-mt-unsafe) + // Silence the check since this class won't be used in a multithreaded context. + // NOLINTNEXTLINE(concurrency-mt-unsafe) + auto const* db_password{std::getenv("CLP_DB_PASS")}; if (nullptr != db_password) { m_metadata_db_password.emplace(db_password); } From 96484844e2546bc6ecd50f6162467a3f5a9a35d3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 12:49:02 -0400 Subject: [PATCH 066/108] refactor: Move GlobalMetadataDBConfig default values to class scope - Initialize class members with default values directly in the class definition --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 14 +------------- .../core/src/clp/GlobalMetadataDBConfig.hpp | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index eb3b3f3e0f..04666a6328 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -17,13 +17,6 @@ using std::string; namespace { // Constants -constexpr clp::GlobalMetadataDBConfig::MetadataDBType cDefaultMetadataDbType{ - clp::GlobalMetadataDBConfig::MetadataDBType::SQLite -}; -constexpr std::string_view cDefaultMetadataDbHost{"127.0.0.1"}; -constexpr int cDefaultMetadataDbPort{3306}; -constexpr std::string_view cDefaultMetadataDbName{"clp-db"}; -constexpr std::string_view cDefaultMetadataTablePrefix{"clp_"}; constexpr int cMinPort{1}; constexpr int cMaxPort{65'535}; } // namespace @@ -46,12 +39,7 @@ auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metada GlobalMetadataDBConfig::GlobalMetadataDBConfig( boost::program_options::options_description& options_description -) - : m_metadata_db_type{cDefaultMetadataDbType}, - m_metadata_db_host{cDefaultMetadataDbHost}, - m_metadata_db_port{cDefaultMetadataDbPort}, - m_metadata_db_name{cDefaultMetadataDbName}, - m_metadata_table_prefix{cDefaultMetadataTablePrefix} { +) { constexpr std::string_view cMetadataDbTypeMysqlOptDescPrefix{"(--db-type=mysql only)"}; // clang-format off diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index 0e343601a3..d8526da94f 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -89,13 +89,20 @@ class GlobalMetadataDBConfig { } private: + // Constants + static constexpr MetadataDBType cDefaultMetadataDbType{MetadataDBType::SQLite}; + static constexpr std::string_view cDefaultMetadataDbHost{"127.0.0.1"}; + static constexpr int cDefaultMetadataDbPort{3306}; + static constexpr std::string_view cDefaultMetadataDbName{"clp-db"}; + static constexpr std::string_view cDefaultMetadataTablePrefix{"clp_"}; + // Variables - MetadataDBType m_metadata_db_type; + MetadataDBType m_metadata_db_type{cDefaultMetadataDbType}; - std::string m_metadata_db_host; - int m_metadata_db_port; - std::string m_metadata_db_name; - std::string m_metadata_table_prefix; + std::string m_metadata_db_host{cDefaultMetadataDbHost}; + int m_metadata_db_port{cDefaultMetadataDbPort}; + std::string m_metadata_db_name{cDefaultMetadataDbName}; + std::string m_metadata_table_prefix{cDefaultMetadataTablePrefix}; std::optional m_metadata_db_username; std::optional m_metadata_db_password; From 1b9d318c20bcd4949caef1ee3951c93d50582379 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 12:56:48 -0400 Subject: [PATCH 067/108] revert indentation change --- .../core/src/clp/clg/CommandLineArguments.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/core/src/clp/clg/CommandLineArguments.cpp b/components/core/src/clp/clg/CommandLineArguments.cpp index 26af9d7edf..d4567627d9 100644 --- a/components/core/src/clp/clg/CommandLineArguments.cpp +++ b/components/core/src/clp/clg/CommandLineArguments.cpp @@ -43,14 +43,14 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { config_file_path += cDefaultConfigFilename; // clang-format off options_general.add_options() - ("help,h", "Print help") - ("version,V", "Print version") - ( - "config-file", - po::value(&config_file_path)->value_name("FILE") - ->default_value(config_file_path), - "Use configuration options from FILE" - ); + ("help,h", "Print help") + ("version,V", "Print version") + ( + "config-file", + po::value(&config_file_path)->value_name("FILE") + ->default_value(config_file_path), + "Use configuration options from FILE" + ); // clang-format on m_metadata_db_config.emplace(options_general); From c9834f31c1d83065eeebd471767fcd17dd141bde Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 12:59:53 -0400 Subject: [PATCH 068/108] remove redundant exception handling for metadata DB config validation --- components/core/src/clp/clg/CommandLineArguments.cpp | 9 ++------- components/core/src/clp/clp/CommandLineArguments.cpp | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/components/core/src/clp/clg/CommandLineArguments.cpp b/components/core/src/clp/clg/CommandLineArguments.cpp index d4567627d9..437bae77db 100644 --- a/components/core/src/clp/clg/CommandLineArguments.cpp +++ b/components/core/src/clp/clg/CommandLineArguments.cpp @@ -193,13 +193,8 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { } // Initialize and validate global metadata DB config - try { - m_metadata_db_config->read_credentials_from_env_if_needed(); - m_metadata_db_config->validate(); - } catch (std::exception& e) { - SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what()); - return ParsingResult::Failure; - } + m_metadata_db_config->read_credentials_from_env_if_needed(); + m_metadata_db_config->validate(); // Validate archive path was specified if (m_archives_dir.empty()) { diff --git a/components/core/src/clp/clp/CommandLineArguments.cpp b/components/core/src/clp/clp/CommandLineArguments.cpp index f387124fb0..3b172e9d7f 100644 --- a/components/core/src/clp/clp/CommandLineArguments.cpp +++ b/components/core/src/clp/clp/CommandLineArguments.cpp @@ -112,13 +112,8 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { } // Initialize and validate global metadata DB config - try { - m_metadata_db_config->read_credentials_from_env_if_needed(); - m_metadata_db_config->validate(); - } catch (std::exception& e) { - SPDLOG_ERROR("Failed to validate metadata database config - {}", e.what()); - return ParsingResult::Failure; - } + m_metadata_db_config->read_credentials_from_env_if_needed(); + m_metadata_db_config->validate(); // Validate command if (parsed_command_line_options.count("command") == 0) { From 4971191c5a16005db69cc802c7cb008a60551a42 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 13:17:54 -0400 Subject: [PATCH 069/108] move global metadata DB config constants back into cpp --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 7 +++++++ .../core/src/clp/GlobalMetadataDBConfig.hpp | 17 +++++------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 04666a6328..de94877e6f 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -17,6 +17,13 @@ using std::string; namespace { // Constants +constexpr clp::GlobalMetadataDBConfig::MetadataDBType cDefaultMetadataDbType{ + clp::GlobalMetadataDBConfig::MetadataDBType::SQLite +}; +constexpr std::string_view cDefaultMetadataDbHost{"127.0.0.1"}; +constexpr int cDefaultMetadataDbPort{3306}; +constexpr std::string_view cDefaultMetadataDbName{"clp-db"}; +constexpr std::string_view cDefaultMetadataTablePrefix{"clp_"}; constexpr int cMinPort{1}; constexpr int cMaxPort{65'535}; } // namespace diff --git a/components/core/src/clp/GlobalMetadataDBConfig.hpp b/components/core/src/clp/GlobalMetadataDBConfig.hpp index d8526da94f..9dcc4e927d 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.hpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.hpp @@ -89,20 +89,13 @@ class GlobalMetadataDBConfig { } private: - // Constants - static constexpr MetadataDBType cDefaultMetadataDbType{MetadataDBType::SQLite}; - static constexpr std::string_view cDefaultMetadataDbHost{"127.0.0.1"}; - static constexpr int cDefaultMetadataDbPort{3306}; - static constexpr std::string_view cDefaultMetadataDbName{"clp-db"}; - static constexpr std::string_view cDefaultMetadataTablePrefix{"clp_"}; - // Variables - MetadataDBType m_metadata_db_type{cDefaultMetadataDbType}; + MetadataDBType m_metadata_db_type{}; - std::string m_metadata_db_host{cDefaultMetadataDbHost}; - int m_metadata_db_port{cDefaultMetadataDbPort}; - std::string m_metadata_db_name{cDefaultMetadataDbName}; - std::string m_metadata_table_prefix{cDefaultMetadataTablePrefix}; + std::string m_metadata_db_host; + int m_metadata_db_port{}; + std::string m_metadata_db_name; + std::string m_metadata_table_prefix; std::optional m_metadata_db_username; std::optional m_metadata_db_password; From edb6810d3bf85f6dbd7d5010c6afd0523cb98a71 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 13:19:42 -0400 Subject: [PATCH 070/108] revert indent change and suppress clang-format --- .../core/src/clp/clp/CommandLineArguments.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/components/core/src/clp/clp/CommandLineArguments.cpp b/components/core/src/clp/clp/CommandLineArguments.cpp index 3b172e9d7f..af821a9e99 100644 --- a/components/core/src/clp/clp/CommandLineArguments.cpp +++ b/components/core/src/clp/clp/CommandLineArguments.cpp @@ -41,13 +41,18 @@ CommandLineArguments::parse_arguments(int argc, char const* argv[]) { config_file_path += '/'; } config_file_path += cDefaultConfigFilename; - options_general.add_options()("help,h", "Print help")("version,V", "Print version")( - "config-file", - po::value(&config_file_path) - ->value_name("FILE") - ->default_value(config_file_path), - "Use configuration options from FILE" - ); + // clang-format off + options_general.add_options() + ("help,h", "Print help") + ("version,V", "Print version") + ( + "config-file", + po::value(&config_file_path) + ->value_name("FILE") + ->default_value(config_file_path), + "Use configuration options from FILE" + ); + // clang-format on m_metadata_db_config.emplace(options_general); // Define functional options From 20c61a644414ed5388620f225393e71d503f22e1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 13:19:42 -0400 Subject: [PATCH 071/108] add a default case to the get_metadata_db_type switch statement --- components/core/src/clp/clp/compression.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/core/src/clp/clp/compression.cpp b/components/core/src/clp/clp/compression.cpp index cd90141df5..768db84107 100644 --- a/components/core/src/clp/clp/compression.cpp +++ b/components/core/src/clp/clp/compression.cpp @@ -91,6 +91,8 @@ bool compress( global_metadata_db_config.get_metadata_table_prefix() ); break; + default: + throw ClpOperationFailed(ErrorCode_Unsupported, __FILENAME__, __LINE__); } auto uuid_generator = boost::uuids::random_generator(); From 71ed50024f1be11966567b41cd5d1a548f61542e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 13:30:39 -0400 Subject: [PATCH 072/108] remove redudant namespace scope - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/clg/CommandLineArguments.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/clg/CommandLineArguments.hpp b/components/core/src/clp/clg/CommandLineArguments.hpp index d5c6b150e8..9cc3e98c35 100644 --- a/components/core/src/clp/clg/CommandLineArguments.hpp +++ b/components/core/src/clp/clg/CommandLineArguments.hpp @@ -60,7 +60,7 @@ class CommandLineArguments : public CommandLineArgumentsBase { std::string m_file_path; OutputMethod m_output_method; epochtime_t m_search_begin_ts, m_search_end_ts; - std::optional m_metadata_db_config; + std::optional m_metadata_db_config; }; } // namespace clp::clg From e3eecdeb068a941d3e2a56b7e18abf32c1461ff2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:05:38 -0400 Subject: [PATCH 073/108] ensure MySQL credentials are provided for global metadata DB --- components/core/src/clp/clg/clg.cpp | 11 +++++++++-- components/core/src/clp/clp/compression.cpp | 11 +++++++++-- components/core/src/clp/clp/utils.cpp | 11 +++++++++-- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/components/core/src/clp/clg/clg.cpp b/components/core/src/clp/clg/clg.cpp index cdd92e1b33..2a28397407 100644 --- a/components/core/src/clp/clg/clg.cpp +++ b/components/core/src/clp/clg/clg.cpp @@ -547,11 +547,18 @@ int main(int argc, char const* argv[]) { break; } case GlobalMetadataDBConfig::MetadataDBType::MySQL: + auto const& global_metadata_db_username = global_metadata_db_config.get_metadata_db_username(); + auto const& global_metadata_db_password = global_metadata_db_config.get_metadata_db_password(); + if (false == global_metadata_db_username.has_value() || + false == global_metadata_db_password.has_value()) { + SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); + return -1; + } global_metadata_db = std::make_unique( global_metadata_db_config.get_metadata_db_host(), global_metadata_db_config.get_metadata_db_port(), - global_metadata_db_config.get_metadata_db_username().value_or(""), - global_metadata_db_config.get_metadata_db_password().value_or(""), + global_metadata_db_username.value(), + global_metadata_db_password.value(), global_metadata_db_config.get_metadata_db_name(), global_metadata_db_config.get_metadata_table_prefix() ); diff --git a/components/core/src/clp/clp/compression.cpp b/components/core/src/clp/clp/compression.cpp index 768db84107..a9a9336c45 100644 --- a/components/core/src/clp/clp/compression.cpp +++ b/components/core/src/clp/clp/compression.cpp @@ -82,11 +82,18 @@ bool compress( break; } case GlobalMetadataDBConfig::MetadataDBType::MySQL: + auto const& global_metadata_db_username = global_metadata_db_config.get_metadata_db_username(); + auto const& global_metadata_db_password = global_metadata_db_config.get_metadata_db_password(); + if (false == global_metadata_db_username.has_value() || + false == global_metadata_db_password.has_value()) { + SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); + return false; + } global_metadata_db = std::make_unique( global_metadata_db_config.get_metadata_db_host(), global_metadata_db_config.get_metadata_db_port(), - global_metadata_db_config.get_metadata_db_username().value_or(""), - global_metadata_db_config.get_metadata_db_password().value_or(""), + global_metadata_db_username.value(), + global_metadata_db_password.value(), global_metadata_db_config.get_metadata_db_name(), global_metadata_db_config.get_metadata_table_prefix() ); diff --git a/components/core/src/clp/clp/utils.cpp b/components/core/src/clp/clp/utils.cpp index 9985ada221..26b6606f49 100644 --- a/components/core/src/clp/clp/utils.cpp +++ b/components/core/src/clp/clp/utils.cpp @@ -184,11 +184,18 @@ std::unique_ptr get_global_metadata_db( return std::make_unique(global_metadata_db_path.string()); } case GlobalMetadataDBConfig::MetadataDBType::MySQL: + auto const& global_metadata_db_username = global_metadata_db_config.get_metadata_db_username(); + auto const& global_metadata_db_password = global_metadata_db_config.get_metadata_db_password(); + if (false == global_metadata_db_username.has_value() || + false == global_metadata_db_password.has_value()) { + SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); + throw ClpOperationFailed(ErrorCode_BadParam, __FILENAME__, __LINE__); + } return std::make_unique( global_metadata_db_config.get_metadata_db_host(), global_metadata_db_config.get_metadata_db_port(), - global_metadata_db_config.get_metadata_db_username().value_or(""), - global_metadata_db_config.get_metadata_db_password().value_or(""), + global_metadata_db_username.value(), + global_metadata_db_password.value(), global_metadata_db_config.get_metadata_db_name(), global_metadata_db_config.get_metadata_table_prefix() ); From b8cbe0a96c2a352becfdae35227e707323866103 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:09:12 -0400 Subject: [PATCH 074/108] docs: add exception documentation to function comments --- components/core/src/clp/clp/compression.hpp | 1 + components/core/src/clp/clp/utils.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/components/core/src/clp/clp/compression.hpp b/components/core/src/clp/clp/compression.hpp index e8ab7364f0..ce3200dd9d 100644 --- a/components/core/src/clp/clp/compression.hpp +++ b/components/core/src/clp/clp/compression.hpp @@ -22,6 +22,7 @@ namespace clp::clp { * @param reader_parser * @param use_heuristic * @return true if compression was successful, false otherwise + * @throw ClpOperationFailed if the global metadata DB cannot be initialized. */ bool compress( CommandLineArguments& command_line_args, diff --git a/components/core/src/clp/clp/utils.hpp b/components/core/src/clp/clp/utils.hpp index 47adc50f20..15c76c4f46 100644 --- a/components/core/src/clp/clp/utils.hpp +++ b/components/core/src/clp/clp/utils.hpp @@ -77,6 +77,7 @@ bool validate_paths_exist(std::vector const& paths); * @param global_metadata_db_config * @param archives_dir * @return The relevant global metadata DB class. + * @throw ClpOperationFailed if the initialization failed. */ std::unique_ptr get_global_metadata_db( GlobalMetadataDBConfig const& global_metadata_db_config, From 7d1a2730f8ed1db9bb076c3d01ba67552deca190 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:13:06 -0400 Subject: [PATCH 075/108] test(core): rename test argc & argv to PascalCase --- .../tests/test-GlobalMetadataDBConfig.cpp | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index aeffb7df4a..fc4c51a4cb 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -22,16 +22,16 @@ void unset_env_var(char const* name) { } template -auto parse_args(std::array const& argv) -> GlobalMetadataDBConfig { +auto parse_args(std::array const& argV) -> GlobalMetadataDBConfig { boost::program_options::options_description options_desc; GlobalMetadataDBConfig config{options_desc}; boost::program_options::variables_map vm; - constexpr auto cArgc{static_cast(n)}; + constexpr auto cArgC{static_cast(n)}; boost::program_options::store( boost::program_options::parse_command_line( - cArgc, - const_cast(argv.data()), + cArgC, + const_cast(argV.data()), options_desc ), vm @@ -46,7 +46,7 @@ TEST_CASE( "Test parsing command line arguments for GlobalMetadataDBConfig", "[GlobalMetadataDBConfig]" ) { - constexpr std::array cArgv{ + constexpr std::array cArgV{ "test", "--db-type", "mysql", @@ -59,7 +59,7 @@ TEST_CASE( "--db-table-prefix", "test_prefix_" }; - auto const config{parse_args(cArgv)}; + auto const config{parse_args(cArgV)}; REQUIRE((config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL)); REQUIRE((config.get_metadata_db_host() == "test-host")); @@ -70,7 +70,7 @@ TEST_CASE( TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBConfig]") { SECTION("With all arguments") { - constexpr std::array cArgv{ + constexpr std::array cArgV{ "test", "--db-type", "mysql", @@ -83,7 +83,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - auto config{parse_args(cArgv)}; + auto config{parse_args(cArgV)}; SECTION("With valid credentials") { set_env_var("CLP_DB_USER", "test-user"); @@ -133,7 +133,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With invalid port values") { SECTION("Port too low") { - constexpr std::array cArgvWithLowPort{ + constexpr std::array cArgVWithLowPort{ "test", "--db-type", "mysql", @@ -146,7 +146,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - auto config{parse_args(cArgvWithLowPort)}; + auto config{parse_args(cArgVWithLowPort)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); @@ -156,7 +156,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Port too high") { - constexpr std::array cArgvWithHighPort{ + constexpr std::array cArgVWithHighPort{ "test", "--db-type", "mysql", @@ -169,7 +169,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - auto config{parse_args(cArgvWithHighPort)}; + auto config{parse_args(cArgVWithHighPort)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); @@ -181,7 +181,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With empty required arguments") { SECTION("Empty db-host") { - constexpr std::array cArgvEmptyHost{ + constexpr std::array cArgVEmptyHost{ "test", "--db-type", "mysql", @@ -194,7 +194,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - auto config{parse_args(cArgvEmptyHost)}; + auto config{parse_args(cArgVEmptyHost)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); @@ -204,7 +204,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Empty db-name") { - constexpr std::array cArgvEmptyName{ + constexpr std::array cArgVEmptyName{ "test", "--db-type", "mysql", @@ -217,7 +217,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "test_prefix_" }; - auto config{parse_args(cArgvEmptyName)}; + auto config{parse_args(cArgVEmptyName)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); @@ -227,7 +227,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Empty db-table-prefix") { - constexpr std::array cArgvEmptyPrefix{ + constexpr std::array cArgVEmptyPrefix{ "test", "--db-type", "mysql", @@ -240,7 +240,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo "--db-table-prefix", "" }; - auto config{parse_args(cArgvEmptyPrefix)}; + auto config{parse_args(cArgVEmptyPrefix)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); @@ -253,37 +253,37 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-host argument") { - constexpr std::array cArgv{"test", "--db-type", "sqlite", "--db-host", "test-host"}; - auto const config{parse_args(cArgv)}; + constexpr std::array cArgV{"test", "--db-type", "sqlite", "--db-host", "test-host"}; + auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-port argument") { - constexpr std::array cArgv{"test", "--db-type", "sqlite", "--db-port", "8888"}; - auto const config{parse_args(cArgv)}; + constexpr std::array cArgV{"test", "--db-type", "sqlite", "--db-port", "8888"}; + auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-name argument") { - constexpr std::array cArgv{"test", "--db-type", "sqlite", "--db-name", "test-db"}; - auto const config{parse_args(cArgv)}; + constexpr std::array cArgV{"test", "--db-type", "sqlite", "--db-name", "test-db"}; + auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-table-prefix argument") { constexpr std::array - cArgv{"test", "--db-type", "sqlite", "--db-table-prefix", "test_prefix_"}; - auto const config{parse_args(cArgv)}; + cArgV{"test", "--db-type", "sqlite", "--db-table-prefix", "test_prefix_"}; + auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With username and password") { - constexpr std::array cArgv{"test", "--db-type", "sqlite"}; - auto config{parse_args(cArgv)}; + constexpr std::array cArgV{"test", "--db-type", "sqlite"}; + auto config{parse_args(cArgV)}; set_env_var("CLP_DB_USER", "test-user"); set_env_var("CLP_DB_PASS", "test-pass"); From baf993f288649f6029436d431dcd7946b9058ed7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:13:17 -0400 Subject: [PATCH 076/108] lint --- components/core/src/clp/clg/clg.cpp | 11 +++++++---- components/core/src/clp/clp/compression.cpp | 11 +++++++---- components/core/src/clp/clp/utils.cpp | 11 +++++++---- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/components/core/src/clp/clg/clg.cpp b/components/core/src/clp/clg/clg.cpp index 2a28397407..522281fe42 100644 --- a/components/core/src/clp/clg/clg.cpp +++ b/components/core/src/clp/clg/clg.cpp @@ -547,10 +547,13 @@ int main(int argc, char const* argv[]) { break; } case GlobalMetadataDBConfig::MetadataDBType::MySQL: - auto const& global_metadata_db_username = global_metadata_db_config.get_metadata_db_username(); - auto const& global_metadata_db_password = global_metadata_db_config.get_metadata_db_password(); - if (false == global_metadata_db_username.has_value() || - false == global_metadata_db_password.has_value()) { + auto const& global_metadata_db_username + = global_metadata_db_config.get_metadata_db_username(); + auto const& global_metadata_db_password + = global_metadata_db_config.get_metadata_db_password(); + if (false == global_metadata_db_username.has_value() + || false == global_metadata_db_password.has_value()) + { SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); return -1; } diff --git a/components/core/src/clp/clp/compression.cpp b/components/core/src/clp/clp/compression.cpp index a9a9336c45..1ea9aa8731 100644 --- a/components/core/src/clp/clp/compression.cpp +++ b/components/core/src/clp/clp/compression.cpp @@ -82,10 +82,13 @@ bool compress( break; } case GlobalMetadataDBConfig::MetadataDBType::MySQL: - auto const& global_metadata_db_username = global_metadata_db_config.get_metadata_db_username(); - auto const& global_metadata_db_password = global_metadata_db_config.get_metadata_db_password(); - if (false == global_metadata_db_username.has_value() || - false == global_metadata_db_password.has_value()) { + auto const& global_metadata_db_username + = global_metadata_db_config.get_metadata_db_username(); + auto const& global_metadata_db_password + = global_metadata_db_config.get_metadata_db_password(); + if (false == global_metadata_db_username.has_value() + || false == global_metadata_db_password.has_value()) + { SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); return false; } diff --git a/components/core/src/clp/clp/utils.cpp b/components/core/src/clp/clp/utils.cpp index 26b6606f49..67a71b063e 100644 --- a/components/core/src/clp/clp/utils.cpp +++ b/components/core/src/clp/clp/utils.cpp @@ -184,10 +184,13 @@ std::unique_ptr get_global_metadata_db( return std::make_unique(global_metadata_db_path.string()); } case GlobalMetadataDBConfig::MetadataDBType::MySQL: - auto const& global_metadata_db_username = global_metadata_db_config.get_metadata_db_username(); - auto const& global_metadata_db_password = global_metadata_db_config.get_metadata_db_password(); - if (false == global_metadata_db_username.has_value() || - false == global_metadata_db_password.has_value()) { + auto const& global_metadata_db_username + = global_metadata_db_config.get_metadata_db_username(); + auto const& global_metadata_db_password + = global_metadata_db_config.get_metadata_db_password(); + if (false == global_metadata_db_username.has_value() + || false == global_metadata_db_password.has_value()) + { SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); throw ClpOperationFailed(ErrorCode_BadParam, __FILENAME__, __LINE__); } From 399e1bded629ae40bafb375318e751534339d502 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:17:04 -0400 Subject: [PATCH 077/108] remove unnecessary const_cast in test-GlobalMetadataDBConfig --- components/core/tests/test-GlobalMetadataDBConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index fc4c51a4cb..1a367209f7 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -31,7 +31,7 @@ auto parse_args(std::array const& argV) -> GlobalMetadataDBConfi boost::program_options::store( boost::program_options::parse_command_line( cArgC, - const_cast(argV.data()), + argV.data(), options_desc ), vm From a686b5422668c129fb910c6ff904961bfeffcef9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:19:03 -0400 Subject: [PATCH 078/108] add missing braces for MySQL case in switch statements --- components/core/src/clp/clg/clg.cpp | 3 ++- components/core/src/clp/clp/compression.cpp | 3 ++- components/core/src/clp/clp/utils.cpp | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/clg/clg.cpp b/components/core/src/clp/clg/clg.cpp index 522281fe42..5979e71f3d 100644 --- a/components/core/src/clp/clg/clg.cpp +++ b/components/core/src/clp/clg/clg.cpp @@ -546,7 +546,7 @@ int main(int argc, char const* argv[]) { ); break; } - case GlobalMetadataDBConfig::MetadataDBType::MySQL: + case GlobalMetadataDBConfig::MetadataDBType::MySQL: { auto const& global_metadata_db_username = global_metadata_db_config.get_metadata_db_username(); auto const& global_metadata_db_password @@ -566,6 +566,7 @@ int main(int argc, char const* argv[]) { global_metadata_db_config.get_metadata_table_prefix() ); break; + } } global_metadata_db->open(); diff --git a/components/core/src/clp/clp/compression.cpp b/components/core/src/clp/clp/compression.cpp index 1ea9aa8731..5e7c5fd60d 100644 --- a/components/core/src/clp/clp/compression.cpp +++ b/components/core/src/clp/clp/compression.cpp @@ -81,7 +81,7 @@ bool compress( = std::make_unique(global_metadata_db_path.string()); break; } - case GlobalMetadataDBConfig::MetadataDBType::MySQL: + case GlobalMetadataDBConfig::MetadataDBType::MySQL: { auto const& global_metadata_db_username = global_metadata_db_config.get_metadata_db_username(); auto const& global_metadata_db_password @@ -101,6 +101,7 @@ bool compress( global_metadata_db_config.get_metadata_table_prefix() ); break; + } default: throw ClpOperationFailed(ErrorCode_Unsupported, __FILENAME__, __LINE__); } diff --git a/components/core/src/clp/clp/utils.cpp b/components/core/src/clp/clp/utils.cpp index 67a71b063e..efb8b5396b 100644 --- a/components/core/src/clp/clp/utils.cpp +++ b/components/core/src/clp/clp/utils.cpp @@ -183,7 +183,7 @@ std::unique_ptr get_global_metadata_db( / static_cast(streaming_archive::cMetadataDBFileName); return std::make_unique(global_metadata_db_path.string()); } - case GlobalMetadataDBConfig::MetadataDBType::MySQL: + case GlobalMetadataDBConfig::MetadataDBType::MySQL: { auto const& global_metadata_db_username = global_metadata_db_config.get_metadata_db_username(); auto const& global_metadata_db_password @@ -202,6 +202,7 @@ std::unique_ptr get_global_metadata_db( global_metadata_db_config.get_metadata_db_name(), global_metadata_db_config.get_metadata_table_prefix() ); + } default: throw ClpOperationFailed(ErrorCode_Unsupported, __FILENAME__, __LINE__); } From b2ad16683b4e8f7d804e6076b6b6edc8f6f90ae1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:19:10 -0400 Subject: [PATCH 079/108] lint --- components/core/tests/test-GlobalMetadataDBConfig.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 1a367209f7..05cac9fbf6 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -29,11 +29,7 @@ auto parse_args(std::array const& argV) -> GlobalMetadataDBConfi boost::program_options::variables_map vm; constexpr auto cArgC{static_cast(n)}; boost::program_options::store( - boost::program_options::parse_command_line( - cArgC, - argV.data(), - options_desc - ), + boost::program_options::parse_command_line(cArgC, argV.data(), options_desc), vm ); boost::program_options::notify(vm); From e0a6ecc55faa15e95f6953bf46ec2a15b3c285cd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:20:32 -0400 Subject: [PATCH 080/108] Use `.value()` to get optional value instead of dereference - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/clg/CommandLineArguments.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/clg/CommandLineArguments.hpp b/components/core/src/clp/clg/CommandLineArguments.hpp index 9cc3e98c35..7bfca48a2f 100644 --- a/components/core/src/clp/clg/CommandLineArguments.hpp +++ b/components/core/src/clp/clg/CommandLineArguments.hpp @@ -46,7 +46,7 @@ class CommandLineArguments : public CommandLineArgumentsBase { epochtime_t get_search_end_ts() const { return m_search_end_ts; } - GlobalMetadataDBConfig const& get_metadata_db_config() const { return *m_metadata_db_config; } + GlobalMetadataDBConfig const& get_metadata_db_config() const { return m_metadata_db_config.value(); } private: // Methods From ea08810f61a5c3437d6db566f206ad8b7a351f62 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:21:12 -0400 Subject: [PATCH 081/108] Use `.value()` to get optional value instead of dereference - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/src/clp/clp/CommandLineArguments.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/clp/CommandLineArguments.hpp b/components/core/src/clp/clp/CommandLineArguments.hpp index 0ca8ad40d7..e1ec565211 100644 --- a/components/core/src/clp/clp/CommandLineArguments.hpp +++ b/components/core/src/clp/clp/CommandLineArguments.hpp @@ -74,7 +74,7 @@ class CommandLineArguments : public CommandLineArgumentsBase { size_t get_ir_target_size() const { return m_ir_target_size; } - GlobalMetadataDBConfig const& get_metadata_db_config() const { return *m_metadata_db_config; } + GlobalMetadataDBConfig const& get_metadata_db_config() const { return m_metadata_db_config.value(); } private: // Methods From e14eb39f36d8ee2bffb718f100f65000645e7a3a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:25:10 -0400 Subject: [PATCH 082/108] remove exception handling for metadata database config validation --- .../core/src/clp_s/indexer/CommandLineArguments.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/components/core/src/clp_s/indexer/CommandLineArguments.cpp b/components/core/src/clp_s/indexer/CommandLineArguments.cpp index e4e8d0a6ad..7ba23c674e 100644 --- a/components/core/src/clp_s/indexer/CommandLineArguments.cpp +++ b/components/core/src/clp_s/indexer/CommandLineArguments.cpp @@ -102,15 +102,8 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { ); return ParsingResult::Failure; } - - try { - metadata_db_config.read_credentials_from_env_if_needed(); - metadata_db_config.validate(); - } catch (std::exception& e) { - SPDLOG_ERROR("Failed to validate metadata database config - {}.", e.what()); - return ParsingResult::Failure; - } - + metadata_db_config.read_credentials_from_env_if_needed(); + metadata_db_config.validate(); m_metadata_db_config = std::move(metadata_db_config); } catch (std::exception& e) { SPDLOG_ERROR("{}", e.what()); From a03cf807665d186ff95ba5c3d14e1d8eba11fbad Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:41:21 -0400 Subject: [PATCH 083/108] test: DRY unset_env_var --- .../tests/test-GlobalMetadataDBConfig.cpp | 29 +++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 05cac9fbf6..7ea4b0e5c1 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -89,9 +89,6 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo REQUIRE_NOTHROW(config.validate()); REQUIRE((config.get_metadata_db_username() == "test-user")); REQUIRE((config.get_metadata_db_password() == "test-pass")); - - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); } SECTION("With empty password") { @@ -102,9 +99,6 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo REQUIRE_NOTHROW(config.validate()); REQUIRE((config.get_metadata_db_username() == "test-user")); REQUIRE((config.get_metadata_db_password() == "")); - - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); } SECTION("With missing credentials") { @@ -116,13 +110,11 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); - unset_env_var("CLP_DB_PASS"); } SECTION("Username set but password missing") { set_env_var("CLP_DB_USER", "test-user"); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); - unset_env_var("CLP_DB_USER"); } } } @@ -147,8 +139,6 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); } SECTION("Port too high") { @@ -170,8 +160,6 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); } } @@ -195,8 +183,6 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); } SECTION("Empty db-name") { @@ -218,8 +204,6 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); } SECTION("Empty db-table-prefix") { @@ -241,10 +225,12 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo set_env_var("CLP_DB_PASS", "test-pass"); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); } } + + // Tear down. + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); } TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { @@ -288,8 +274,9 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { REQUIRE_FALSE(config.get_metadata_db_username().has_value()); REQUIRE_FALSE(config.get_metadata_db_password().has_value()); REQUIRE_NOTHROW(config.validate()); - - unset_env_var("CLP_DB_USER"); - unset_env_var("CLP_DB_PASS"); } + + // Tear down. + unset_env_var("CLP_DB_USER"); + unset_env_var("CLP_DB_PASS"); } From dbe94d3dfc94fd49145cccf1df258d903e826597 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:42:35 -0400 Subject: [PATCH 084/108] avoid inline comments --- components/core/tests/test-GlobalMetadataDBConfig.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 7ea4b0e5c1..f53f4ff367 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -14,11 +14,15 @@ using namespace clp; namespace { void set_env_var(char const* name, char const* value) { - setenv(name, value, 1); // NOLINT(concurrency-mt-unsafe) + // Silence the check since this class won't be used in a multithreaded context. + // NOLINTNEXTLINE(concurrency-mt-unsafe) + setenv(name, value, 1); } void unset_env_var(char const* name) { - unsetenv(name); // NOLINT(concurrency-mt-unsafe) + // Silence the check since this class won't be used in a multithreaded context. + // NOLINTNEXTLINE(concurrency-mt-unsafe) + unsetenv(name); } template From 9cf11c3a8b24260c356d6d043ac7f23263167974 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 14:42:59 -0400 Subject: [PATCH 085/108] lint --- components/core/src/clp/clg/CommandLineArguments.hpp | 4 +++- components/core/src/clp/clp/CommandLineArguments.hpp | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp/clg/CommandLineArguments.hpp b/components/core/src/clp/clg/CommandLineArguments.hpp index 7bfca48a2f..e128c57021 100644 --- a/components/core/src/clp/clg/CommandLineArguments.hpp +++ b/components/core/src/clp/clg/CommandLineArguments.hpp @@ -46,7 +46,9 @@ class CommandLineArguments : public CommandLineArgumentsBase { epochtime_t get_search_end_ts() const { return m_search_end_ts; } - GlobalMetadataDBConfig const& get_metadata_db_config() const { return m_metadata_db_config.value(); } + GlobalMetadataDBConfig const& get_metadata_db_config() const { + return m_metadata_db_config.value(); + } private: // Methods diff --git a/components/core/src/clp/clp/CommandLineArguments.hpp b/components/core/src/clp/clp/CommandLineArguments.hpp index e1ec565211..272a6ec759 100644 --- a/components/core/src/clp/clp/CommandLineArguments.hpp +++ b/components/core/src/clp/clp/CommandLineArguments.hpp @@ -74,7 +74,9 @@ class CommandLineArguments : public CommandLineArgumentsBase { size_t get_ir_target_size() const { return m_ir_target_size; } - GlobalMetadataDBConfig const& get_metadata_db_config() const { return m_metadata_db_config.value(); } + GlobalMetadataDBConfig const& get_metadata_db_config() const { + return m_metadata_db_config.value(); + } private: // Methods From 26f9a561a46b9452d808305c8188e73358dc5e84 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 15:14:02 -0400 Subject: [PATCH 086/108] fix(core): improve global metadata DB config validation --- .../core/src/clp_s/indexer/IndexManager.cpp | 41 +++++++++++-------- .../core/src/clp_s/indexer/IndexManager.hpp | 3 ++ 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/components/core/src/clp_s/indexer/IndexManager.cpp b/components/core/src/clp_s/indexer/IndexManager.cpp index 46ba6bb67a..fabafd2302 100644 --- a/components/core/src/clp_s/indexer/IndexManager.cpp +++ b/components/core/src/clp_s/indexer/IndexManager.cpp @@ -5,6 +5,8 @@ #include #include +#include + #include "../archive_constants.hpp" #include "../TimestampDictionaryReader.hpp" @@ -13,24 +15,31 @@ IndexManager::IndexManager( std::optional const& db_config, bool should_create_table ) { - if (db_config.has_value()) { - m_mysql_index_storage = std::make_unique( - db_config->get_metadata_db_host(), - db_config->get_metadata_db_port(), - db_config->get_metadata_db_username().value(), - db_config->get_metadata_db_password().value(), - db_config->get_metadata_db_name(), - db_config->get_metadata_table_prefix() - ); - m_mysql_index_storage->open(); - m_field_update_callback = [this](std::string& field_name, NodeType field_type) { - m_mysql_index_storage->add_field(field_name, field_type); - }; - m_should_create_table = should_create_table; - m_output_type = OutputType::Database; - } else { + if (false == db_config.has_value()) { + throw OperationFailed(ErrorCodeBadParam, __FILENAME__, __LINE__); + } + + auto const& db_username = db_config->get_metadata_db_username(); + auto const& db_password = db_config->get_metadata_db_password(); + if (false == db_username.has_value() || false == db_password.has_value()) { + SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); throw OperationFailed(ErrorCodeBadParam, __FILENAME__, __LINE__); } + + m_mysql_index_storage = std::make_unique( + db_config->get_metadata_db_host(), + db_config->get_metadata_db_port(), + db_username.value(), + db_password.value(), + db_config->get_metadata_db_name(), + db_config->get_metadata_table_prefix() + ); + m_mysql_index_storage->open(); + m_field_update_callback = [this](std::string& field_name, NodeType field_type) { + m_mysql_index_storage->add_field(field_name, field_type); + }; + m_should_create_table = should_create_table; + m_output_type = OutputType::Database; } IndexManager::~IndexManager() { diff --git a/components/core/src/clp_s/indexer/IndexManager.hpp b/components/core/src/clp_s/indexer/IndexManager.hpp index 33a7bfbc7b..788ade3e82 100644 --- a/components/core/src/clp_s/indexer/IndexManager.hpp +++ b/components/core/src/clp_s/indexer/IndexManager.hpp @@ -39,6 +39,9 @@ class IndexManager { }; // Constructors + /** + * @throw OperationFailed if the global metadata DB config is invalid. + */ IndexManager( std::optional const& db_config = std::nullopt, bool should_create_table = false From b886c33d3bdd03222ba7850bce55c0edff837da6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 15:21:59 -0400 Subject: [PATCH 087/108] use constant strings for mysql and sqlite --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index de94877e6f..83a9b58194 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -10,6 +10,7 @@ #include #include +#include "GlobalMySQLMetadataDB.hpp" #include "type_utils.hpp" using std::invalid_argument; @@ -47,7 +48,10 @@ auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metada GlobalMetadataDBConfig::GlobalMetadataDBConfig( boost::program_options::options_description& options_description ) { - constexpr std::string_view cMetadataDbTypeMysqlOptDescPrefix{"(--db-type=mysql only)"}; + constexpr std::string_view cMetadataDbTypeMysqlOptDescPrefix{fmt::format( + "(--db-type={} only)", + cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MYSQL)] + )}; // clang-format off options_description.add_options() @@ -138,8 +142,12 @@ auto GlobalMetadataDBConfig::validate() const -> void { || m_metadata_db_username.has_value() || m_metadata_db_password.has_value()) { throw invalid_argument( - "MySQL-specific parameters cannot be used with --db-type=sqlite." - " Please remove them or set '--db-type=mysql'." + fmt::format( + "MySQL-specific parameters cannot be used with --db-type={}." + " Please remove them or set '--db-type={}'.", + cMetadataDBTypeNames[enum_to_underlying_type(m_metadata_db_type)], + cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MYSQL)] + ) ); } return; From 2c10dbba027078b70e770cd0477f822714fcd0a2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 15:23:44 -0400 Subject: [PATCH 088/108] Use if-initializer syntax to simplify null checks for environment variables --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index 83a9b58194..b87d9718c1 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -119,15 +119,13 @@ auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void { // Silence the check since this class won't be used in a multithreaded context. // NOLINTNEXTLINE(concurrency-mt-unsafe) - auto const* db_username{std::getenv("CLP_DB_USER")}; - if (nullptr != db_username) { + if (auto const* db_username{std::getenv("CLP_DB_USER")}; nullptr != db_username) { m_metadata_db_username.emplace(db_username); } // Silence the check since this class won't be used in a multithreaded context. // NOLINTNEXTLINE(concurrency-mt-unsafe) - auto const* db_password{std::getenv("CLP_DB_PASS")}; - if (nullptr != db_password) { + if (auto const* db_password{std::getenv("CLP_DB_PASS")}; nullptr != db_password) { m_metadata_db_password.emplace(db_password); } } From 69b99dae9ebc086bbac19360d5dc768487691ded Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 15:36:47 -0400 Subject: [PATCH 089/108] fix(core): update MySQL db type reference in global metadata DB config --- components/core/src/clp/GlobalMetadataDBConfig.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index b87d9718c1..de9fade282 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -48,9 +48,9 @@ auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metada GlobalMetadataDBConfig::GlobalMetadataDBConfig( boost::program_options::options_description& options_description ) { - constexpr std::string_view cMetadataDbTypeMysqlOptDescPrefix{fmt::format( + std::string_view const cMetadataDbTypeMysqlOptDescPrefix{fmt::format( "(--db-type={} only)", - cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MYSQL)] + cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MySQL)] )}; // clang-format off @@ -144,7 +144,7 @@ auto GlobalMetadataDBConfig::validate() const -> void { "MySQL-specific parameters cannot be used with --db-type={}." " Please remove them or set '--db-type={}'.", cMetadataDBTypeNames[enum_to_underlying_type(m_metadata_db_type)], - cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MYSQL)] + cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MySQL)] ) ); } From 9d5c7ab0ab2d9c5da285dabcffc02a42a678d887 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 16:23:49 -0400 Subject: [PATCH 090/108] test(core): refactor GlobalMetadataDBConfig tests --- .../tests/test-GlobalMetadataDBConfig.cpp | 223 ++++++++---------- 1 file changed, 94 insertions(+), 129 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index f53f4ff367..2fbae8c553 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -13,6 +14,45 @@ using namespace clp; namespace { +// Constants +constexpr std::string_view cMysqlDbType{"mysql"}; +constexpr std::string_view cMysqlDbHost{"test-host"}; +constexpr std::string_view cMysqlDbPort{"8888"}; +constexpr std::string_view cMysqlDbName{"test-db"}; +constexpr std::string_view cMysqlDbTablePrefix{"test_prefix_"}; +constexpr std::string_view cMysqlDbUser{"test-user"}; +constexpr std::string_view cMysqlDbPass{"test-pass"}; +constexpr std::array cCommonMysqlArgs{ + "test", + "--db-type", + cMysqlDbType.data(), + "--db-host", + cMysqlDbHost.data(), + "--db-port", + cMysqlDbPort.data(), + "--db-name", + cMysqlDbName.data(), + "--db-table-prefix", + cMysqlDbTablePrefix.data() +}; +constexpr size_t cArgIdxDbHost{4}; +constexpr size_t cArgIdxDbPort{6}; +constexpr size_t cArgIdxDbName{8}; +constexpr size_t cArgIdxDbTablePrefix{10}; + +constexpr std::string_view cSqliteDbType{"sqlite"}; +constexpr std::array cCommonSqliteArgs{ + "test", + "--db-type", + cSqliteDbType.data(), + + // Placeholders for overwriting with MySQL-specific arguments like `--db-host`. + "", + "", +}; +constexpr size_t cArgIdxOverwriteArgName{3}; +constexpr size_t cArgIdxOverwriteArgValue{4}; + void set_env_var(char const* name, char const* value) { // Silence the check since this class won't be used in a multithreaded context. // NOLINTNEXTLINE(concurrency-mt-unsafe) @@ -46,62 +86,36 @@ TEST_CASE( "Test parsing command line arguments for GlobalMetadataDBConfig", "[GlobalMetadataDBConfig]" ) { - constexpr std::array cArgV{ - "test", - "--db-type", - "mysql", - "--db-host", - "test-host", - "--db-port", - "8888", - "--db-name", - "test-db", - "--db-table-prefix", - "test_prefix_" - }; - auto const config{parse_args(cArgV)}; + auto const config{parse_args(cCommonMysqlArgs)}; REQUIRE((config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL)); - REQUIRE((config.get_metadata_db_host() == "test-host")); - REQUIRE((config.get_metadata_db_port() == 8888)); - REQUIRE((config.get_metadata_db_name() == "test-db")); - REQUIRE((config.get_metadata_table_prefix() == "test_prefix_")); + REQUIRE((config.get_metadata_db_host() == cMysqlDbHost)); + REQUIRE((config.get_metadata_db_port() == std::stoi(std::string(cMysqlDbPort)))); + REQUIRE((config.get_metadata_db_name() == cMysqlDbName)); + REQUIRE((config.get_metadata_table_prefix() == cMysqlDbTablePrefix)); } TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBConfig]") { SECTION("With all arguments") { - constexpr std::array cArgV{ - "test", - "--db-type", - "mysql", - "--db-host", - "test-host", - "--db-port", - "8888", - "--db-name", - "test-db", - "--db-table-prefix", - "test_prefix_" - }; - auto config{parse_args(cArgV)}; + auto config{parse_args(cCommonMysqlArgs)}; SECTION("With valid credentials") { - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", "test-pass"); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); - REQUIRE((config.get_metadata_db_username() == "test-user")); - REQUIRE((config.get_metadata_db_password() == "test-pass")); + REQUIRE((config.get_metadata_db_username() == cMysqlDbUser)); + REQUIRE((config.get_metadata_db_password() == cMysqlDbPass)); } SECTION("With empty password") { - set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); set_env_var("CLP_DB_PASS", ""); config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); - REQUIRE((config.get_metadata_db_username() == "test-user")); + REQUIRE((config.get_metadata_db_username() == cMysqlDbUser)); REQUIRE((config.get_metadata_db_password() == "")); } @@ -111,12 +125,12 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Password set but username missing") { - set_env_var("CLP_DB_PASS", "test-pass"); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Username set but password missing") { - set_env_var("CLP_DB_USER", "test-user"); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -125,43 +139,21 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With invalid port values") { SECTION("Port too low") { - constexpr std::array cArgVWithLowPort{ - "test", - "--db-type", - "mysql", - "--db-host", - "test-host", - "--db-port", - "0", - "--db-name", - "test-db", - "--db-table-prefix", - "test_prefix_" - }; - auto config{parse_args(cArgVWithLowPort)}; - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", "test-pass"); + auto cArgV{cCommonMysqlArgs}; + cArgV[cArgIdxDbPort] = "0"; + auto config{parse_args(cArgV)}; + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Port too high") { - constexpr std::array cArgVWithHighPort{ - "test", - "--db-type", - "mysql", - "--db-host", - "test-host", - "--db-port", - "65536", - "--db-name", - "test-db", - "--db-table-prefix", - "test_prefix_" - }; - auto config{parse_args(cArgVWithHighPort)}; - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", "test-pass"); + auto cArgV{cCommonMysqlArgs}; + cArgV[cArgIdxDbPort] = "65536"; + auto config{parse_args(cArgV)}; + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -169,64 +161,31 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With empty required arguments") { SECTION("Empty db-host") { - constexpr std::array cArgVEmptyHost{ - "test", - "--db-type", - "mysql", - "--db-host", - "", - "--db-port", - "8888", - "--db-name", - "test-db", - "--db-table-prefix", - "test_prefix_" - }; - auto config{parse_args(cArgVEmptyHost)}; - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", "test-pass"); + auto cArgV{cCommonMysqlArgs}; + cArgV[cArgIdxDbHost] = ""; + auto config{parse_args(cArgV)}; + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Empty db-name") { - constexpr std::array cArgVEmptyName{ - "test", - "--db-type", - "mysql", - "--db-host", - "test-host", - "--db-port", - "8888", - "--db-name", - "", - "--db-table-prefix", - "test_prefix_" - }; - auto config{parse_args(cArgVEmptyName)}; - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", "test-pass"); + auto cArgV{cCommonMysqlArgs}; + cArgV[cArgIdxDbName] = ""; + auto config{parse_args(cArgV)}; + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Empty db-table-prefix") { - constexpr std::array cArgVEmptyPrefix{ - "test", - "--db-type", - "mysql", - "--db-host", - "test-host", - "--db-port", - "8888", - "--db-name", - "test-db", - "--db-table-prefix", - "" - }; - auto config{parse_args(cArgVEmptyPrefix)}; - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", "test-pass"); + auto cArgV{cCommonMysqlArgs}; + cArgV[cArgIdxDbTablePrefix] = ""; + auto config{parse_args(cArgV)}; + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -239,40 +198,46 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-host argument") { - constexpr std::array cArgV{"test", "--db-type", "sqlite", "--db-host", "test-host"}; + auto cArgV{cCommonSqliteArgs}; + cArgV[cArgIdxOverwriteArgName] = "--db-host"; + cArgV[cArgIdxOverwriteArgValue] = cMysqlDbHost.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-port argument") { - constexpr std::array cArgV{"test", "--db-type", "sqlite", "--db-port", "8888"}; + auto cArgV{cCommonSqliteArgs}; + cArgV[cArgIdxOverwriteArgName] = "--db-port"; + cArgV[cArgIdxOverwriteArgValue] = cMysqlDbPort.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-name argument") { - constexpr std::array cArgV{"test", "--db-type", "sqlite", "--db-name", "test-db"}; + auto cArgV{cCommonSqliteArgs}; + cArgV[cArgIdxOverwriteArgName] = "--db-name"; + cArgV[cArgIdxOverwriteArgValue] = cMysqlDbName.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-table-prefix argument") { - constexpr std::array - cArgV{"test", "--db-type", "sqlite", "--db-table-prefix", "test_prefix_"}; + auto cArgV{cCommonSqliteArgs}; + cArgV[cArgIdxOverwriteArgName] = "--db-table-prefix"; + cArgV[cArgIdxOverwriteArgValue] = cMysqlDbTablePrefix.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With username and password") { - constexpr std::array cArgV{"test", "--db-type", "sqlite"}; - auto config{parse_args(cArgV)}; + auto config{parse_args(cCommonSqliteArgs)}; - set_env_var("CLP_DB_USER", "test-user"); - set_env_var("CLP_DB_PASS", "test-pass"); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_FALSE(config.get_metadata_db_username().has_value()); From 5156dc9266650a6761694bf735e9aa56ec899ff5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 16:28:42 -0400 Subject: [PATCH 091/108] refactor: rename MySQL-related variables and options --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 10 +- .../tests/test-GlobalMetadataDBConfig.cpp | 96 +++++++++---------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index de9fade282..f995ceceda 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -48,7 +48,7 @@ auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metada GlobalMetadataDBConfig::GlobalMetadataDBConfig( boost::program_options::options_description& options_description ) { - std::string_view const cMetadataDbTypeMysqlOptDescPrefix{fmt::format( + std::string_view const cMetadataDbTypeMySqlOptDescPrefix{fmt::format( "(--db-type={} only)", cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MySQL)] )}; @@ -78,7 +78,7 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( ->default_value(string(cDefaultMetadataDbHost)), fmt::format( "{} Database host", - cMetadataDbTypeMysqlOptDescPrefix + cMetadataDbTypeMySqlOptDescPrefix ).c_str() ) ( @@ -87,7 +87,7 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( ->default_value(cDefaultMetadataDbPort), fmt::format( "{} Database port", - cMetadataDbTypeMysqlOptDescPrefix + cMetadataDbTypeMySqlOptDescPrefix ).c_str() ) ( @@ -96,7 +96,7 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( ->default_value(string(cDefaultMetadataDbName)), fmt::format( "{} Database name", - cMetadataDbTypeMysqlOptDescPrefix + cMetadataDbTypeMySqlOptDescPrefix ).c_str() ) ( @@ -105,7 +105,7 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( ->default_value(string(cDefaultMetadataTablePrefix)), fmt::format( "{} Database table prefix", - cMetadataDbTypeMysqlOptDescPrefix + cMetadataDbTypeMySqlOptDescPrefix ).c_str() ); // clang-format on diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 2fbae8c553..8db15e5225 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -15,25 +15,25 @@ using namespace clp; namespace { // Constants -constexpr std::string_view cMysqlDbType{"mysql"}; -constexpr std::string_view cMysqlDbHost{"test-host"}; -constexpr std::string_view cMysqlDbPort{"8888"}; -constexpr std::string_view cMysqlDbName{"test-db"}; -constexpr std::string_view cMysqlDbTablePrefix{"test_prefix_"}; -constexpr std::string_view cMysqlDbUser{"test-user"}; -constexpr std::string_view cMysqlDbPass{"test-pass"}; -constexpr std::array cCommonMysqlArgs{ +constexpr std::string_view cMySqlDbType{"mysql"}; +constexpr std::string_view cMySqlDbHost{"test-host"}; +constexpr std::string_view cMySqlDbPort{"8888"}; +constexpr std::string_view cMySqlDbName{"test-db"}; +constexpr std::string_view cMySqlDbTablePrefix{"test_prefix_"}; +constexpr std::string_view cMySqlDbUser{"test-user"}; +constexpr std::string_view cMySqlDbPass{"test-pass"}; +constexpr std::array cCommonMySqlArgs{ "test", "--db-type", - cMysqlDbType.data(), + cMySqlDbType.data(), "--db-host", - cMysqlDbHost.data(), + cMySqlDbHost.data(), "--db-port", - cMysqlDbPort.data(), + cMySqlDbPort.data(), "--db-name", - cMysqlDbName.data(), + cMySqlDbName.data(), "--db-table-prefix", - cMysqlDbTablePrefix.data() + cMySqlDbTablePrefix.data() }; constexpr size_t cArgIdxDbHost{4}; constexpr size_t cArgIdxDbPort{6}; @@ -86,36 +86,36 @@ TEST_CASE( "Test parsing command line arguments for GlobalMetadataDBConfig", "[GlobalMetadataDBConfig]" ) { - auto const config{parse_args(cCommonMysqlArgs)}; + auto const config{parse_args(cCommonMySqlArgs)}; REQUIRE((config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL)); - REQUIRE((config.get_metadata_db_host() == cMysqlDbHost)); - REQUIRE((config.get_metadata_db_port() == std::stoi(std::string(cMysqlDbPort)))); - REQUIRE((config.get_metadata_db_name() == cMysqlDbName)); - REQUIRE((config.get_metadata_table_prefix() == cMysqlDbTablePrefix)); + REQUIRE((config.get_metadata_db_host() == cMySqlDbHost)); + REQUIRE((config.get_metadata_db_port() == std::stoi(std::string(cMySqlDbPort)))); + REQUIRE((config.get_metadata_db_name() == cMySqlDbName)); + REQUIRE((config.get_metadata_table_prefix() == cMySqlDbTablePrefix)); } TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBConfig]") { SECTION("With all arguments") { - auto config{parse_args(cCommonMysqlArgs)}; + auto config{parse_args(cCommonMySqlArgs)}; SECTION("With valid credentials") { - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_USER", cMySqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); - REQUIRE((config.get_metadata_db_username() == cMysqlDbUser)); - REQUIRE((config.get_metadata_db_password() == cMysqlDbPass)); + REQUIRE((config.get_metadata_db_username() == cMySqlDbUser)); + REQUIRE((config.get_metadata_db_password() == cMySqlDbPass)); } SECTION("With empty password") { - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_USER", cMySqlDbUser.data()); set_env_var("CLP_DB_PASS", ""); config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); - REQUIRE((config.get_metadata_db_username() == cMysqlDbUser)); + REQUIRE((config.get_metadata_db_username() == cMySqlDbUser)); REQUIRE((config.get_metadata_db_password() == "")); } @@ -125,12 +125,12 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Password set but username missing") { - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Username set but password missing") { - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_USER", cMySqlDbUser.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -139,21 +139,21 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With invalid port values") { SECTION("Port too low") { - auto cArgV{cCommonMysqlArgs}; + auto cArgV{cCommonMySqlArgs}; cArgV[cArgIdxDbPort] = "0"; auto config{parse_args(cArgV)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_USER", cMySqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Port too high") { - auto cArgV{cCommonMysqlArgs}; + auto cArgV{cCommonMySqlArgs}; cArgV[cArgIdxDbPort] = "65536"; auto config{parse_args(cArgV)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_USER", cMySqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -161,31 +161,31 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With empty required arguments") { SECTION("Empty db-host") { - auto cArgV{cCommonMysqlArgs}; + auto cArgV{cCommonMySqlArgs}; cArgV[cArgIdxDbHost] = ""; auto config{parse_args(cArgV)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_USER", cMySqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Empty db-name") { - auto cArgV{cCommonMysqlArgs}; + auto cArgV{cCommonMySqlArgs}; cArgV[cArgIdxDbName] = ""; auto config{parse_args(cArgV)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_USER", cMySqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Empty db-table-prefix") { - auto cArgV{cCommonMysqlArgs}; + auto cArgV{cCommonMySqlArgs}; cArgV[cArgIdxDbTablePrefix] = ""; auto config{parse_args(cArgV)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_USER", cMySqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -200,7 +200,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-host argument") { auto cArgV{cCommonSqliteArgs}; cArgV[cArgIdxOverwriteArgName] = "--db-host"; - cArgV[cArgIdxOverwriteArgValue] = cMysqlDbHost.data(); + cArgV[cArgIdxOverwriteArgValue] = cMySqlDbHost.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); @@ -209,7 +209,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-port argument") { auto cArgV{cCommonSqliteArgs}; cArgV[cArgIdxOverwriteArgName] = "--db-port"; - cArgV[cArgIdxOverwriteArgValue] = cMysqlDbPort.data(); + cArgV[cArgIdxOverwriteArgValue] = cMySqlDbPort.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); @@ -218,7 +218,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-name argument") { auto cArgV{cCommonSqliteArgs}; cArgV[cArgIdxOverwriteArgName] = "--db-name"; - cArgV[cArgIdxOverwriteArgValue] = cMysqlDbName.data(); + cArgV[cArgIdxOverwriteArgValue] = cMySqlDbName.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); @@ -227,7 +227,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-table-prefix argument") { auto cArgV{cCommonSqliteArgs}; cArgV[cArgIdxOverwriteArgName] = "--db-table-prefix"; - cArgV[cArgIdxOverwriteArgValue] = cMysqlDbTablePrefix.data(); + cArgV[cArgIdxOverwriteArgValue] = cMySqlDbTablePrefix.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); @@ -236,8 +236,8 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With username and password") { auto config{parse_args(cCommonSqliteArgs)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_USER", cMySqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_FALSE(config.get_metadata_db_username().has_value()); From 960094485eacaa8628edcbc0855be7a7c80463dc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 16:42:24 -0400 Subject: [PATCH 092/108] refactor: rename MySQL-related variables as `mysql` rather than `mySql` --- .../core/src/clp/GlobalMetadataDBConfig.cpp | 10 +- .../tests/test-GlobalMetadataDBConfig.cpp | 96 +++++++++---------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/components/core/src/clp/GlobalMetadataDBConfig.cpp b/components/core/src/clp/GlobalMetadataDBConfig.cpp index f995ceceda..de9fade282 100644 --- a/components/core/src/clp/GlobalMetadataDBConfig.cpp +++ b/components/core/src/clp/GlobalMetadataDBConfig.cpp @@ -48,7 +48,7 @@ auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metada GlobalMetadataDBConfig::GlobalMetadataDBConfig( boost::program_options::options_description& options_description ) { - std::string_view const cMetadataDbTypeMySqlOptDescPrefix{fmt::format( + std::string_view const cMetadataDbTypeMysqlOptDescPrefix{fmt::format( "(--db-type={} only)", cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MySQL)] )}; @@ -78,7 +78,7 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( ->default_value(string(cDefaultMetadataDbHost)), fmt::format( "{} Database host", - cMetadataDbTypeMySqlOptDescPrefix + cMetadataDbTypeMysqlOptDescPrefix ).c_str() ) ( @@ -87,7 +87,7 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( ->default_value(cDefaultMetadataDbPort), fmt::format( "{} Database port", - cMetadataDbTypeMySqlOptDescPrefix + cMetadataDbTypeMysqlOptDescPrefix ).c_str() ) ( @@ -96,7 +96,7 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( ->default_value(string(cDefaultMetadataDbName)), fmt::format( "{} Database name", - cMetadataDbTypeMySqlOptDescPrefix + cMetadataDbTypeMysqlOptDescPrefix ).c_str() ) ( @@ -105,7 +105,7 @@ GlobalMetadataDBConfig::GlobalMetadataDBConfig( ->default_value(string(cDefaultMetadataTablePrefix)), fmt::format( "{} Database table prefix", - cMetadataDbTypeMySqlOptDescPrefix + cMetadataDbTypeMysqlOptDescPrefix ).c_str() ); // clang-format on diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 8db15e5225..2fbae8c553 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -15,25 +15,25 @@ using namespace clp; namespace { // Constants -constexpr std::string_view cMySqlDbType{"mysql"}; -constexpr std::string_view cMySqlDbHost{"test-host"}; -constexpr std::string_view cMySqlDbPort{"8888"}; -constexpr std::string_view cMySqlDbName{"test-db"}; -constexpr std::string_view cMySqlDbTablePrefix{"test_prefix_"}; -constexpr std::string_view cMySqlDbUser{"test-user"}; -constexpr std::string_view cMySqlDbPass{"test-pass"}; -constexpr std::array cCommonMySqlArgs{ +constexpr std::string_view cMysqlDbType{"mysql"}; +constexpr std::string_view cMysqlDbHost{"test-host"}; +constexpr std::string_view cMysqlDbPort{"8888"}; +constexpr std::string_view cMysqlDbName{"test-db"}; +constexpr std::string_view cMysqlDbTablePrefix{"test_prefix_"}; +constexpr std::string_view cMysqlDbUser{"test-user"}; +constexpr std::string_view cMysqlDbPass{"test-pass"}; +constexpr std::array cCommonMysqlArgs{ "test", "--db-type", - cMySqlDbType.data(), + cMysqlDbType.data(), "--db-host", - cMySqlDbHost.data(), + cMysqlDbHost.data(), "--db-port", - cMySqlDbPort.data(), + cMysqlDbPort.data(), "--db-name", - cMySqlDbName.data(), + cMysqlDbName.data(), "--db-table-prefix", - cMySqlDbTablePrefix.data() + cMysqlDbTablePrefix.data() }; constexpr size_t cArgIdxDbHost{4}; constexpr size_t cArgIdxDbPort{6}; @@ -86,36 +86,36 @@ TEST_CASE( "Test parsing command line arguments for GlobalMetadataDBConfig", "[GlobalMetadataDBConfig]" ) { - auto const config{parse_args(cCommonMySqlArgs)}; + auto const config{parse_args(cCommonMysqlArgs)}; REQUIRE((config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL)); - REQUIRE((config.get_metadata_db_host() == cMySqlDbHost)); - REQUIRE((config.get_metadata_db_port() == std::stoi(std::string(cMySqlDbPort)))); - REQUIRE((config.get_metadata_db_name() == cMySqlDbName)); - REQUIRE((config.get_metadata_table_prefix() == cMySqlDbTablePrefix)); + REQUIRE((config.get_metadata_db_host() == cMysqlDbHost)); + REQUIRE((config.get_metadata_db_port() == std::stoi(std::string(cMysqlDbPort)))); + REQUIRE((config.get_metadata_db_name() == cMysqlDbName)); + REQUIRE((config.get_metadata_table_prefix() == cMysqlDbTablePrefix)); } TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBConfig]") { SECTION("With all arguments") { - auto config{parse_args(cCommonMySqlArgs)}; + auto config{parse_args(cCommonMysqlArgs)}; SECTION("With valid credentials") { - set_env_var("CLP_DB_USER", cMySqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); - REQUIRE((config.get_metadata_db_username() == cMySqlDbUser)); - REQUIRE((config.get_metadata_db_password() == cMySqlDbPass)); + REQUIRE((config.get_metadata_db_username() == cMysqlDbUser)); + REQUIRE((config.get_metadata_db_password() == cMysqlDbPass)); } SECTION("With empty password") { - set_env_var("CLP_DB_USER", cMySqlDbUser.data()); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); set_env_var("CLP_DB_PASS", ""); config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); - REQUIRE((config.get_metadata_db_username() == cMySqlDbUser)); + REQUIRE((config.get_metadata_db_username() == cMysqlDbUser)); REQUIRE((config.get_metadata_db_password() == "")); } @@ -125,12 +125,12 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Password set but username missing") { - set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Username set but password missing") { - set_env_var("CLP_DB_USER", cMySqlDbUser.data()); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -139,21 +139,21 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With invalid port values") { SECTION("Port too low") { - auto cArgV{cCommonMySqlArgs}; + auto cArgV{cCommonMysqlArgs}; cArgV[cArgIdxDbPort] = "0"; auto config{parse_args(cArgV)}; - set_env_var("CLP_DB_USER", cMySqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Port too high") { - auto cArgV{cCommonMySqlArgs}; + auto cArgV{cCommonMysqlArgs}; cArgV[cArgIdxDbPort] = "65536"; auto config{parse_args(cArgV)}; - set_env_var("CLP_DB_USER", cMySqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -161,31 +161,31 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With empty required arguments") { SECTION("Empty db-host") { - auto cArgV{cCommonMySqlArgs}; + auto cArgV{cCommonMysqlArgs}; cArgV[cArgIdxDbHost] = ""; auto config{parse_args(cArgV)}; - set_env_var("CLP_DB_USER", cMySqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Empty db-name") { - auto cArgV{cCommonMySqlArgs}; + auto cArgV{cCommonMysqlArgs}; cArgV[cArgIdxDbName] = ""; auto config{parse_args(cArgV)}; - set_env_var("CLP_DB_USER", cMySqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Empty db-table-prefix") { - auto cArgV{cCommonMySqlArgs}; + auto cArgV{cCommonMysqlArgs}; cArgV[cArgIdxDbTablePrefix] = ""; auto config{parse_args(cArgV)}; - set_env_var("CLP_DB_USER", cMySqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -200,7 +200,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-host argument") { auto cArgV{cCommonSqliteArgs}; cArgV[cArgIdxOverwriteArgName] = "--db-host"; - cArgV[cArgIdxOverwriteArgValue] = cMySqlDbHost.data(); + cArgV[cArgIdxOverwriteArgValue] = cMysqlDbHost.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); @@ -209,7 +209,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-port argument") { auto cArgV{cCommonSqliteArgs}; cArgV[cArgIdxOverwriteArgName] = "--db-port"; - cArgV[cArgIdxOverwriteArgValue] = cMySqlDbPort.data(); + cArgV[cArgIdxOverwriteArgValue] = cMysqlDbPort.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); @@ -218,7 +218,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-name argument") { auto cArgV{cCommonSqliteArgs}; cArgV[cArgIdxOverwriteArgName] = "--db-name"; - cArgV[cArgIdxOverwriteArgValue] = cMySqlDbName.data(); + cArgV[cArgIdxOverwriteArgValue] = cMysqlDbName.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); @@ -227,7 +227,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-table-prefix argument") { auto cArgV{cCommonSqliteArgs}; cArgV[cArgIdxOverwriteArgName] = "--db-table-prefix"; - cArgV[cArgIdxOverwriteArgValue] = cMySqlDbTablePrefix.data(); + cArgV[cArgIdxOverwriteArgValue] = cMysqlDbTablePrefix.data(); auto const config{parse_args(cArgV)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); @@ -236,8 +236,8 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With username and password") { auto config{parse_args(cCommonSqliteArgs)}; - set_env_var("CLP_DB_USER", cMySqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMySqlDbPass.data()); + set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); REQUIRE_FALSE(config.get_metadata_db_username().has_value()); From 00b22b828eeac7cadca72ca8ce841b48ede1a880 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 16:43:16 -0400 Subject: [PATCH 093/108] refactor: rename command line argument arrays with `argv` --- .../tests/test-GlobalMetadataDBConfig.cpp | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 2fbae8c553..bc446b2c97 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -22,7 +22,7 @@ constexpr std::string_view cMysqlDbName{"test-db"}; constexpr std::string_view cMysqlDbTablePrefix{"test_prefix_"}; constexpr std::string_view cMysqlDbUser{"test-user"}; constexpr std::string_view cMysqlDbPass{"test-pass"}; -constexpr std::array cCommonMysqlArgs{ +constexpr std::array cCommonMysqlArgv{ "test", "--db-type", cMysqlDbType.data(), @@ -41,7 +41,7 @@ constexpr size_t cArgIdxDbName{8}; constexpr size_t cArgIdxDbTablePrefix{10}; constexpr std::string_view cSqliteDbType{"sqlite"}; -constexpr std::array cCommonSqliteArgs{ +constexpr std::array cCommonSqliteArgv{ "test", "--db-type", cSqliteDbType.data(), @@ -86,7 +86,7 @@ TEST_CASE( "Test parsing command line arguments for GlobalMetadataDBConfig", "[GlobalMetadataDBConfig]" ) { - auto const config{parse_args(cCommonMysqlArgs)}; + auto const config{parse_args(cCommonMysqlArgv)}; REQUIRE((config.get_metadata_db_type() == GlobalMetadataDBConfig::MetadataDBType::MySQL)); REQUIRE((config.get_metadata_db_host() == cMysqlDbHost)); @@ -97,7 +97,7 @@ TEST_CASE( TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBConfig]") { SECTION("With all arguments") { - auto config{parse_args(cCommonMysqlArgs)}; + auto config{parse_args(cCommonMysqlArgv)}; SECTION("With valid credentials") { set_env_var("CLP_DB_USER", cMysqlDbUser.data()); @@ -139,7 +139,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With invalid port values") { SECTION("Port too low") { - auto cArgV{cCommonMysqlArgs}; + auto cArgV{cCommonMysqlArgv}; cArgV[cArgIdxDbPort] = "0"; auto config{parse_args(cArgV)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); @@ -149,7 +149,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Port too high") { - auto cArgV{cCommonMysqlArgs}; + auto cArgV{cCommonMysqlArgv}; cArgV[cArgIdxDbPort] = "65536"; auto config{parse_args(cArgV)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); @@ -161,7 +161,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With empty required arguments") { SECTION("Empty db-host") { - auto cArgV{cCommonMysqlArgs}; + auto cArgV{cCommonMysqlArgv}; cArgV[cArgIdxDbHost] = ""; auto config{parse_args(cArgV)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); @@ -171,7 +171,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Empty db-name") { - auto cArgV{cCommonMysqlArgs}; + auto cArgV{cCommonMysqlArgv}; cArgV[cArgIdxDbName] = ""; auto config{parse_args(cArgV)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); @@ -181,7 +181,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Empty db-table-prefix") { - auto cArgV{cCommonMysqlArgs}; + auto cArgV{cCommonMysqlArgv}; cArgV[cArgIdxDbTablePrefix] = ""; auto config{parse_args(cArgV)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); @@ -198,7 +198,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-host argument") { - auto cArgV{cCommonSqliteArgs}; + auto cArgV{cCommonSqliteArgv}; cArgV[cArgIdxOverwriteArgName] = "--db-host"; cArgV[cArgIdxOverwriteArgValue] = cMysqlDbHost.data(); auto const config{parse_args(cArgV)}; @@ -207,7 +207,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { } SECTION("With non-default db-port argument") { - auto cArgV{cCommonSqliteArgs}; + auto cArgV{cCommonSqliteArgv}; cArgV[cArgIdxOverwriteArgName] = "--db-port"; cArgV[cArgIdxOverwriteArgValue] = cMysqlDbPort.data(); auto const config{parse_args(cArgV)}; @@ -216,7 +216,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { } SECTION("With non-default db-name argument") { - auto cArgV{cCommonSqliteArgs}; + auto cArgV{cCommonSqliteArgv}; cArgV[cArgIdxOverwriteArgName] = "--db-name"; cArgV[cArgIdxOverwriteArgValue] = cMysqlDbName.data(); auto const config{parse_args(cArgV)}; @@ -225,7 +225,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { } SECTION("With non-default db-table-prefix argument") { - auto cArgV{cCommonSqliteArgs}; + auto cArgV{cCommonSqliteArgv}; cArgV[cArgIdxOverwriteArgName] = "--db-table-prefix"; cArgV[cArgIdxOverwriteArgValue] = cMysqlDbTablePrefix.data(); auto const config{parse_args(cArgV)}; @@ -234,7 +234,7 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { } SECTION("With username and password") { - auto config{parse_args(cCommonSqliteArgs)}; + auto config{parse_args(cCommonSqliteArgv)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); From b36880230c15b675e583722012ddb24193ee4c65 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 11 Aug 2025 16:46:14 -0400 Subject: [PATCH 094/108] rename `argV`/`argC` back to `argv` / `argc` --- .../tests/test-GlobalMetadataDBConfig.cpp | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index bc446b2c97..3dae104278 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -66,14 +66,14 @@ void unset_env_var(char const* name) { } template -auto parse_args(std::array const& argV) -> GlobalMetadataDBConfig { +auto parse_args(std::array const& argv) -> GlobalMetadataDBConfig { boost::program_options::options_description options_desc; GlobalMetadataDBConfig config{options_desc}; boost::program_options::variables_map vm; - constexpr auto cArgC{static_cast(n)}; + constexpr auto cArgc{static_cast(n)}; boost::program_options::store( - boost::program_options::parse_command_line(cArgC, argV.data(), options_desc), + boost::program_options::parse_command_line(cArgc, argv.data(), options_desc), vm ); boost::program_options::notify(vm); @@ -139,9 +139,9 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With invalid port values") { SECTION("Port too low") { - auto cArgV{cCommonMysqlArgv}; - cArgV[cArgIdxDbPort] = "0"; - auto config{parse_args(cArgV)}; + auto cArgv{cCommonMysqlArgv}; + cArgv[cArgIdxDbPort] = "0"; + auto config{parse_args(cArgv)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); @@ -149,9 +149,9 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Port too high") { - auto cArgV{cCommonMysqlArgv}; - cArgV[cArgIdxDbPort] = "65536"; - auto config{parse_args(cArgV)}; + auto cArgv{cCommonMysqlArgv}; + cArgv[cArgIdxDbPort] = "65536"; + auto config{parse_args(cArgv)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); @@ -161,9 +161,9 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With empty required arguments") { SECTION("Empty db-host") { - auto cArgV{cCommonMysqlArgv}; - cArgV[cArgIdxDbHost] = ""; - auto config{parse_args(cArgV)}; + auto cArgv{cCommonMysqlArgv}; + cArgv[cArgIdxDbHost] = ""; + auto config{parse_args(cArgv)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); @@ -171,9 +171,9 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Empty db-name") { - auto cArgV{cCommonMysqlArgv}; - cArgV[cArgIdxDbName] = ""; - auto config{parse_args(cArgV)}; + auto cArgv{cCommonMysqlArgv}; + cArgv[cArgIdxDbName] = ""; + auto config{parse_args(cArgv)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); @@ -181,9 +181,9 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("Empty db-table-prefix") { - auto cArgV{cCommonMysqlArgv}; - cArgV[cArgIdxDbTablePrefix] = ""; - auto config{parse_args(cArgV)}; + auto cArgv{cCommonMysqlArgv}; + cArgv[cArgIdxDbTablePrefix] = ""; + auto config{parse_args(cArgv)}; set_env_var("CLP_DB_USER", cMysqlDbUser.data()); set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); config.read_credentials_from_env_if_needed(); @@ -198,37 +198,37 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-host argument") { - auto cArgV{cCommonSqliteArgv}; - cArgV[cArgIdxOverwriteArgName] = "--db-host"; - cArgV[cArgIdxOverwriteArgValue] = cMysqlDbHost.data(); - auto const config{parse_args(cArgV)}; + auto cArgv{cCommonSqliteArgv}; + cArgv[cArgIdxOverwriteArgName] = "--db-host"; + cArgv[cArgIdxOverwriteArgValue] = cMysqlDbHost.data(); + auto const config{parse_args(cArgv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-port argument") { - auto cArgV{cCommonSqliteArgv}; - cArgV[cArgIdxOverwriteArgName] = "--db-port"; - cArgV[cArgIdxOverwriteArgValue] = cMysqlDbPort.data(); - auto const config{parse_args(cArgV)}; + auto cArgv{cCommonSqliteArgv}; + cArgv[cArgIdxOverwriteArgName] = "--db-port"; + cArgv[cArgIdxOverwriteArgValue] = cMysqlDbPort.data(); + auto const config{parse_args(cArgv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-name argument") { - auto cArgV{cCommonSqliteArgv}; - cArgV[cArgIdxOverwriteArgName] = "--db-name"; - cArgV[cArgIdxOverwriteArgValue] = cMysqlDbName.data(); - auto const config{parse_args(cArgV)}; + auto cArgv{cCommonSqliteArgv}; + cArgv[cArgIdxOverwriteArgName] = "--db-name"; + cArgv[cArgIdxOverwriteArgValue] = cMysqlDbName.data(); + auto const config{parse_args(cArgv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-table-prefix argument") { - auto cArgV{cCommonSqliteArgv}; - cArgV[cArgIdxOverwriteArgName] = "--db-table-prefix"; - cArgV[cArgIdxOverwriteArgValue] = cMysqlDbTablePrefix.data(); - auto const config{parse_args(cArgV)}; + auto cArgv{cCommonSqliteArgv}; + cArgv[cArgIdxOverwriteArgName] = "--db-table-prefix"; + cArgv[cArgIdxOverwriteArgValue] = cMysqlDbTablePrefix.data(); + auto const config{parse_args(cArgv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } From e524ed7b38683508cb37d28ac7594f4921c37c33 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 07:44:08 -0400 Subject: [PATCH 095/108] Extract config -> global metadata DB creation into utility function; Delete obsolete utility function. --- components/core/CMakeLists.txt | 2 + components/core/src/clp/clg/CMakeLists.txt | 2 + .../core/src/clp/clg/CommandLineArguments.hpp | 4 +- components/core/src/clp/clg/clg.cpp | 37 ++-------- components/core/src/clp/clp/CMakeLists.txt | 2 + .../core/src/clp/clp/CommandLineArguments.hpp | 4 +- components/core/src/clp/clp/compression.cpp | 39 ++--------- components/core/src/clp/clp/compression.hpp | 1 - components/core/src/clp/clp/decompression.cpp | 19 ++++-- components/core/src/clp/clp/utils.cpp | 36 ---------- components/core/src/clp/clp/utils.hpp | 12 ---- .../core/src/clp/global_metadata_db_utils.cpp | 68 +++++++++++++++++++ .../core/src/clp/global_metadata_db_utils.hpp | 21 ++++++ 13 files changed, 125 insertions(+), 122 deletions(-) create mode 100644 components/core/src/clp/global_metadata_db_utils.cpp create mode 100644 components/core/src/clp/global_metadata_db_utils.hpp diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index b10b2bb5bb..4f598c981b 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -558,6 +558,8 @@ set(SOURCE_FILES_unitTest src/clp/FileReader.hpp src/clp/FileWriter.cpp src/clp/FileWriter.hpp + src/clp/global_metadata_db_utils.hpp + src/clp/global_metadata_db_utils.cpp src/clp/GlobalMetadataDB.hpp src/clp/GlobalMetadataDBConfig.cpp src/clp/GlobalMetadataDBConfig.hpp diff --git a/components/core/src/clp/clg/CMakeLists.txt b/components/core/src/clp/clg/CMakeLists.txt index d3ab654d98..659ad5c737 100644 --- a/components/core/src/clp/clg/CMakeLists.txt +++ b/components/core/src/clp/clg/CMakeLists.txt @@ -24,6 +24,8 @@ set( ../FileReader.hpp ../FileWriter.cpp ../FileWriter.hpp + ../global_metadata_db_utils.cpp + ../global_metadata_db_utils.hpp ../GlobalMetadataDB.hpp ../GlobalMetadataDBConfig.cpp ../GlobalMetadataDBConfig.hpp diff --git a/components/core/src/clp/clg/CommandLineArguments.hpp b/components/core/src/clp/clg/CommandLineArguments.hpp index e128c57021..3835c4e662 100644 --- a/components/core/src/clp/clg/CommandLineArguments.hpp +++ b/components/core/src/clp/clg/CommandLineArguments.hpp @@ -46,8 +46,8 @@ class CommandLineArguments : public CommandLineArgumentsBase { epochtime_t get_search_end_ts() const { return m_search_end_ts; } - GlobalMetadataDBConfig const& get_metadata_db_config() const { - return m_metadata_db_config.value(); + std::optional const& get_metadata_db_config() const { + return m_metadata_db_config; } private: diff --git a/components/core/src/clp/clg/clg.cpp b/components/core/src/clp/clg/clg.cpp index 5979e71f3d..b754c61215 100644 --- a/components/core/src/clp/clg/clg.cpp +++ b/components/core/src/clp/clg/clg.cpp @@ -9,6 +9,7 @@ #include #include "../Defs.h" +#include "../global_metadata_db_utils.hpp" #include "../GlobalMySQLMetadataDB.hpp" #include "../GlobalSQLiteMetadataDB.hpp" #include "../Grep.hpp" @@ -535,38 +536,10 @@ int main(int argc, char const* argv[]) { return -1; } - auto const& global_metadata_db_config = command_line_args.get_metadata_db_config(); - std::unique_ptr global_metadata_db; - switch (global_metadata_db_config.get_metadata_db_type()) { - case GlobalMetadataDBConfig::MetadataDBType::SQLite: { - auto global_metadata_db_path - = archives_dir / clp::streaming_archive::cMetadataDBFileName; - global_metadata_db = std::make_unique( - global_metadata_db_path.string() - ); - break; - } - case GlobalMetadataDBConfig::MetadataDBType::MySQL: { - auto const& global_metadata_db_username - = global_metadata_db_config.get_metadata_db_username(); - auto const& global_metadata_db_password - = global_metadata_db_config.get_metadata_db_password(); - if (false == global_metadata_db_username.has_value() - || false == global_metadata_db_password.has_value()) - { - SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); - return -1; - } - global_metadata_db = std::make_unique( - global_metadata_db_config.get_metadata_db_host(), - global_metadata_db_config.get_metadata_db_port(), - global_metadata_db_username.value(), - global_metadata_db_password.value(), - global_metadata_db_config.get_metadata_db_name(), - global_metadata_db_config.get_metadata_table_prefix() - ); - break; - } + auto global_metadata_db + = create_global_metadata_db(command_line_args.get_metadata_db_config(), archives_dir); + if (nullptr == global_metadata_db) { + return -1; } global_metadata_db->open(); diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index 50aa6a61fb..b4a1c4108e 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -33,6 +33,8 @@ set( ../FileReader.hpp ../FileWriter.cpp ../FileWriter.hpp + ../global_metadata_db_utils.cpp + ../global_metadata_db_utils.hpp ../GlobalMetadataDB.hpp ../GlobalMetadataDBConfig.cpp ../GlobalMetadataDBConfig.hpp diff --git a/components/core/src/clp/clp/CommandLineArguments.hpp b/components/core/src/clp/clp/CommandLineArguments.hpp index 272a6ec759..c6d7fe0cfd 100644 --- a/components/core/src/clp/clp/CommandLineArguments.hpp +++ b/components/core/src/clp/clp/CommandLineArguments.hpp @@ -74,8 +74,8 @@ class CommandLineArguments : public CommandLineArgumentsBase { size_t get_ir_target_size() const { return m_ir_target_size; } - GlobalMetadataDBConfig const& get_metadata_db_config() const { - return m_metadata_db_config.value(); + std::optional const& get_metadata_db_config() const { + return m_metadata_db_config; } private: diff --git a/components/core/src/clp/clp/compression.cpp b/components/core/src/clp/clp/compression.cpp index 5e7c5fd60d..964c0fc6f7 100644 --- a/components/core/src/clp/clp/compression.cpp +++ b/components/core/src/clp/clp/compression.cpp @@ -7,6 +7,7 @@ #include #include +#include "../global_metadata_db_utils.hpp" #include "../GlobalMySQLMetadataDB.hpp" #include "../GlobalSQLiteMetadataDB.hpp" #include "../spdlog_with_specializations.hpp" @@ -63,7 +64,7 @@ bool compress( std::unique_ptr reader_parser, bool use_heuristic ) { - auto output_dir = boost::filesystem::path(command_line_args.get_output_dir()); + auto output_dir = std::filesystem::path(command_line_args.get_archives_dir()); // Create output directory in case it doesn't exist auto error_code = create_directory(output_dir.parent_path().string(), 0700, true); @@ -72,38 +73,10 @@ bool compress( return false; } - auto const& global_metadata_db_config = command_line_args.get_metadata_db_config(); - std::unique_ptr global_metadata_db; - switch (global_metadata_db_config.get_metadata_db_type()) { - case GlobalMetadataDBConfig::MetadataDBType::SQLite: { - auto global_metadata_db_path = output_dir / streaming_archive::cMetadataDBFileName; - global_metadata_db - = std::make_unique(global_metadata_db_path.string()); - break; - } - case GlobalMetadataDBConfig::MetadataDBType::MySQL: { - auto const& global_metadata_db_username - = global_metadata_db_config.get_metadata_db_username(); - auto const& global_metadata_db_password - = global_metadata_db_config.get_metadata_db_password(); - if (false == global_metadata_db_username.has_value() - || false == global_metadata_db_password.has_value()) - { - SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); - return false; - } - global_metadata_db = std::make_unique( - global_metadata_db_config.get_metadata_db_host(), - global_metadata_db_config.get_metadata_db_port(), - global_metadata_db_username.value(), - global_metadata_db_password.value(), - global_metadata_db_config.get_metadata_db_name(), - global_metadata_db_config.get_metadata_table_prefix() - ); - break; - } - default: - throw ClpOperationFailed(ErrorCode_Unsupported, __FILENAME__, __LINE__); + auto global_metadata_db + = create_global_metadata_db(command_line_args.get_metadata_db_config(), output_dir); + if (nullptr == global_metadata_db) { + return false; } auto uuid_generator = boost::uuids::random_generator(); diff --git a/components/core/src/clp/clp/compression.hpp b/components/core/src/clp/clp/compression.hpp index ce3200dd9d..e8ab7364f0 100644 --- a/components/core/src/clp/clp/compression.hpp +++ b/components/core/src/clp/clp/compression.hpp @@ -22,7 +22,6 @@ namespace clp::clp { * @param reader_parser * @param use_heuristic * @return true if compression was successful, false otherwise - * @throw ClpOperationFailed if the global metadata DB cannot be initialized. */ bool compress( CommandLineArguments& command_line_args, diff --git a/components/core/src/clp/clp/decompression.cpp b/components/core/src/clp/clp/decompression.cpp index ce7cbd5c7a..497631f692 100644 --- a/components/core/src/clp/clp/decompression.cpp +++ b/components/core/src/clp/clp/decompression.cpp @@ -5,6 +5,7 @@ #include "../ErrorCode.hpp" #include "../FileWriter.hpp" +#include "../global_metadata_db_utils.hpp" #include "../GlobalMySQLMetadataDB.hpp" #include "../GlobalSQLiteMetadataDB.hpp" #include "../ir/constants.hpp" @@ -40,8 +41,13 @@ bool decompress( try { auto archives_dir = std::filesystem::path(command_line_args.get_archives_dir()); - auto const& global_metadata_db_config = command_line_args.get_metadata_db_config(); - auto global_metadata_db = get_global_metadata_db(global_metadata_db_config, archives_dir); + auto global_metadata_db = create_global_metadata_db( + command_line_args.get_metadata_db_config(), + archives_dir + ); + if (nullptr == global_metadata_db) { + return false; + } streaming_archive::reader::Archive archive_reader; @@ -245,8 +251,13 @@ bool decompress_to_ir(CommandLineArguments& command_line_args) { try { std::filesystem::path archives_dir{command_line_args.get_archives_dir()}; - auto const& global_metadata_db_config = command_line_args.get_metadata_db_config(); - auto global_metadata_db = get_global_metadata_db(global_metadata_db_config, archives_dir); + auto global_metadata_db = create_global_metadata_db( + command_line_args.get_metadata_db_config(), + archives_dir + ); + if (nullptr == global_metadata_db) { + return false; + } global_metadata_db->open(); string archive_id; diff --git a/components/core/src/clp/clp/utils.cpp b/components/core/src/clp/clp/utils.cpp index efb8b5396b..75504138fa 100644 --- a/components/core/src/clp/clp/utils.cpp +++ b/components/core/src/clp/clp/utils.cpp @@ -171,40 +171,4 @@ bool validate_paths_exist(vector const& paths) { return all_paths_exist; } - -std::unique_ptr get_global_metadata_db( - GlobalMetadataDBConfig const& global_metadata_db_config, - std::filesystem::path const& archives_dir -) { - switch (global_metadata_db_config.get_metadata_db_type()) { - case GlobalMetadataDBConfig::MetadataDBType::SQLite: { - auto global_metadata_db_path - = archives_dir - / static_cast(streaming_archive::cMetadataDBFileName); - return std::make_unique(global_metadata_db_path.string()); - } - case GlobalMetadataDBConfig::MetadataDBType::MySQL: { - auto const& global_metadata_db_username - = global_metadata_db_config.get_metadata_db_username(); - auto const& global_metadata_db_password - = global_metadata_db_config.get_metadata_db_password(); - if (false == global_metadata_db_username.has_value() - || false == global_metadata_db_password.has_value()) - { - SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); - throw ClpOperationFailed(ErrorCode_BadParam, __FILENAME__, __LINE__); - } - return std::make_unique( - global_metadata_db_config.get_metadata_db_host(), - global_metadata_db_config.get_metadata_db_port(), - global_metadata_db_username.value(), - global_metadata_db_password.value(), - global_metadata_db_config.get_metadata_db_name(), - global_metadata_db_config.get_metadata_table_prefix() - ); - } - default: - throw ClpOperationFailed(ErrorCode_Unsupported, __FILENAME__, __LINE__); - } -} } // namespace clp::clp diff --git a/components/core/src/clp/clp/utils.hpp b/components/core/src/clp/clp/utils.hpp index 15c76c4f46..faf4e97103 100644 --- a/components/core/src/clp/clp/utils.hpp +++ b/components/core/src/clp/clp/utils.hpp @@ -71,18 +71,6 @@ bool remove_prefix_and_clean_up_path( * @return true if they all exist, false otherwise */ bool validate_paths_exist(std::vector const& paths); - -/** - * Chooses and initializes the relevant global metadata DB class based on the given config. - * @param global_metadata_db_config - * @param archives_dir - * @return The relevant global metadata DB class. - * @throw ClpOperationFailed if the initialization failed. - */ -std::unique_ptr get_global_metadata_db( - GlobalMetadataDBConfig const& global_metadata_db_config, - std::filesystem::path const& archives_dir -); } // namespace clp::clp #endif // CLP_CLP_UTILS_HPP diff --git a/components/core/src/clp/global_metadata_db_utils.cpp b/components/core/src/clp/global_metadata_db_utils.cpp new file mode 100644 index 0000000000..921f9123df --- /dev/null +++ b/components/core/src/clp/global_metadata_db_utils.cpp @@ -0,0 +1,68 @@ +#include "global_metadata_db_utils.hpp" + +#include + +#include "ErrorCode.hpp" +#include "GlobalMetadataDBConfig.hpp" +#include "GlobalMySQLMetadataDB.hpp" +#include "GlobalSQLiteMetadataDB.hpp" +#include "TraceableException.hpp" + +namespace clp { +auto create_global_metadata_db( + std::optional const& optional_metadata_db_config, + std::filesystem::path const& archives_dir +) -> std::unique_ptr { + constexpr std::string_view cErrorMsgPrefix = "Failed to load global metadata DB config: "; + try { + auto const& metadata_db_config = optional_metadata_db_config.value(); + switch (metadata_db_config.get_metadata_db_type()) { + case GlobalMetadataDBConfig::MetadataDBType::SQLite: { + auto global_metadata_db_path + = archives_dir / streaming_archive::cMetadataDBFileName; + return std::make_unique(global_metadata_db_path.string()); + } + case GlobalMetadataDBConfig::MetadataDBType::MySQL: { + return std::make_unique( + metadata_db_config.get_metadata_db_host(), + metadata_db_config.get_metadata_db_port(), + metadata_db_config.get_metadata_db_username().value(), + metadata_db_config.get_metadata_db_password().value(), + metadata_db_config.get_metadata_db_name(), + metadata_db_config.get_metadata_table_prefix() + ); + } + default: + SPDLOG_ERROR( + "Unhandled metadata DB type: {}", + static_cast(metadata_db_config.get_metadata_db_type()) + ); + return nullptr; + } + } catch (TraceableException const& e) { + auto error_code = e.get_error_code(); + if (ErrorCode_errno == error_code) { + SPDLOG_ERROR( + "{}: {}:{} {}, errno={}", + cErrorMsgPrefix, + e.get_filename(), + e.get_line_number(), + e.what(), + errno + ); + } else { + SPDLOG_ERROR( + "{}: {}:{} {}, error_code={}", + cErrorMsgPrefix, + e.get_filename(), + e.get_line_number(), + e.what(), + error_code + ); + } + } catch (std::exception const& e) { + SPDLOG_ERROR("{}: {}", cErrorMsgPrefix, e.what()); + } + return nullptr; +} +} // namespace clp diff --git a/components/core/src/clp/global_metadata_db_utils.hpp b/components/core/src/clp/global_metadata_db_utils.hpp new file mode 100644 index 0000000000..92399b9c7f --- /dev/null +++ b/components/core/src/clp/global_metadata_db_utils.hpp @@ -0,0 +1,21 @@ +#ifndef CLP_GLOBAL_METADATA_DB_UTILS_HPP +#define CLP_GLOBAL_METADATA_DB_UTILS_HPP + +#include "GlobalMetadataDB.hpp" +#include "GlobalMetadataDBConfig.hpp" + +namespace clp { +/** + * Creates a GlobalMetadataDB instance based on the given configuration. + * + * @param optional_metadata_db_config + * @param archives_dir + * @return The created GlobalMetadataDB instance. + */ +auto create_global_metadata_db( + std::optional const& optional_metadata_db_config, + std::filesystem::path const& archives_dir +) -> std::unique_ptr; +} // namespace clp + +#endif // CLP_GLOBAL_METADATA_DB_UTILS_HPP From 9807fb2de46710a18ad2212ae0291c5ed6f94774 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 07:44:39 -0400 Subject: [PATCH 096/108] IndexManager: Refactor error handling and add missing parameters to docstring. --- .../core/src/clp_s/indexer/IndexManager.cpp | 30 ++++++++----------- .../core/src/clp_s/indexer/IndexManager.hpp | 6 ++-- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/components/core/src/clp_s/indexer/IndexManager.cpp b/components/core/src/clp_s/indexer/IndexManager.cpp index fabafd2302..4b2882e29e 100644 --- a/components/core/src/clp_s/indexer/IndexManager.cpp +++ b/components/core/src/clp_s/indexer/IndexManager.cpp @@ -12,28 +12,22 @@ namespace clp_s::indexer { IndexManager::IndexManager( - std::optional const& db_config, + std::optional const& optional_db_config, bool should_create_table ) { - if (false == db_config.has_value()) { + try { + auto const& db_config = optional_db_config.value(); + m_mysql_index_storage = std::make_unique( + db_config.get_metadata_db_host(), + db_config.get_metadata_db_port(), + db_config.get_metadata_db_username().value(), + db_config.get_metadata_db_password().value(), + db_config.get_metadata_db_name(), + db_config.get_metadata_table_prefix() + ); + } catch (std::bad_optional_access const& e) { throw OperationFailed(ErrorCodeBadParam, __FILENAME__, __LINE__); } - - auto const& db_username = db_config->get_metadata_db_username(); - auto const& db_password = db_config->get_metadata_db_password(); - if (false == db_username.has_value() || false == db_password.has_value()) { - SPDLOG_ERROR("Unexpected missing MySQL credentials for global metadata DB."); - throw OperationFailed(ErrorCodeBadParam, __FILENAME__, __LINE__); - } - - m_mysql_index_storage = std::make_unique( - db_config->get_metadata_db_host(), - db_config->get_metadata_db_port(), - db_username.value(), - db_password.value(), - db_config->get_metadata_db_name(), - db_config->get_metadata_table_prefix() - ); m_mysql_index_storage->open(); m_field_update_callback = [this](std::string& field_name, NodeType field_type) { m_mysql_index_storage->add_field(field_name, field_type); diff --git a/components/core/src/clp_s/indexer/IndexManager.hpp b/components/core/src/clp_s/indexer/IndexManager.hpp index 788ade3e82..92f7fd6fa8 100644 --- a/components/core/src/clp_s/indexer/IndexManager.hpp +++ b/components/core/src/clp_s/indexer/IndexManager.hpp @@ -40,10 +40,12 @@ class IndexManager { // Constructors /** - * @throw OperationFailed if the global metadata DB config is invalid. + * @param optional_db_config + * @param should_create_table + * @throw OperationFailed if `optional_db_config` is invalid. */ IndexManager( - std::optional const& db_config = std::nullopt, + std::optional const& optional_db_config = std::nullopt, bool should_create_table = false ); From 4a2790e7b9614db72d5d14b039090ca3a079f131 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 07:45:18 -0400 Subject: [PATCH 097/108] test-GlobalMetadataDBConfig: Use existing constants rather than duplicating them. --- components/core/tests/test-GlobalMetadataDBConfig.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 3dae104278..128643eb00 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -15,7 +15,6 @@ using namespace clp; namespace { // Constants -constexpr std::string_view cMysqlDbType{"mysql"}; constexpr std::string_view cMysqlDbHost{"test-host"}; constexpr std::string_view cMysqlDbPort{"8888"}; constexpr std::string_view cMysqlDbName{"test-db"}; @@ -25,7 +24,9 @@ constexpr std::string_view cMysqlDbPass{"test-pass"}; constexpr std::array cCommonMysqlArgv{ "test", "--db-type", - cMysqlDbType.data(), + GlobalMetadataDBConfig::cMetadataDBTypeNames + [enum_to_underlying_type(GlobalMetadataDBConfig::MetadataDBType::MySQL)] + .data(), "--db-host", cMysqlDbHost.data(), "--db-port", @@ -40,11 +41,12 @@ constexpr size_t cArgIdxDbPort{6}; constexpr size_t cArgIdxDbName{8}; constexpr size_t cArgIdxDbTablePrefix{10}; -constexpr std::string_view cSqliteDbType{"sqlite"}; constexpr std::array cCommonSqliteArgv{ "test", "--db-type", - cSqliteDbType.data(), + GlobalMetadataDBConfig::cMetadataDBTypeNames + [enum_to_underlying_type(GlobalMetadataDBConfig::MetadataDBType::SQLite)] + .data(), // Placeholders for overwriting with MySQL-specific arguments like `--db-host`. "", From ab36b4dc12829719e1d1f40634b79ffddbeaf121 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 07:53:44 -0400 Subject: [PATCH 098/108] Refactor docs. --- docs/src/user-guide/core-unstructured/clp.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index af6a1702f3..c215998d7d 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -105,14 +105,15 @@ Currently, timestamps must be specified as milliseconds since the UNIX epoch. # Parallel Compression +To enable parallel compression to the same archives directory, `clp`/`clg` need to be configured to +use a MySQL-type database (e.g., MariaDB) rather than the default---an embedded SQLite database +(which doesn't support concurrent writes). + :::{warning} -By default, `clp` uses an embedded SQLite database. Running multiple `clp` instances with SQLite -(rather than a MySQL-type database) can fail due to the error "database is locked" because SQLite -doesn't support concurrent writes. +Running multiple `clp` instances with SQLite can fail due to the error "database is locked". ::: -To enable parallel compression to the same archives directory, `clp`/`clg` can be configured to use -a MySQL-type database (e.g., MariaDB) as follows: +You can configure `clp` and `clg` to use a MySQL-type database as follows: * Install and configure MariaDB using the instructions for your platform * Create a user that has privileges to create databases, create tables, insert records, and delete @@ -129,10 +130,13 @@ a MySQL-type database (e.g., MariaDB) as follows: * `--db-table-prefix ` to specify the table prefix * Set the `CLP_DB_USER` environment variable for the database user's username * Set the `CLP_DB_PASS` environment variable for the database user's password + +To compress logs in parallel using the MySQL database: + * Run `clp` or `clg` with the same command-line options and environment variables, with the addition of the database type command-line option: * `--db-type mysql` to specify MySQL as the database type -* To compress in parallel, simply run another instance of `clp` concurrently. +* Run as many parallel `clp` or `clg` instances as desired. Note that currently, decompression (`clp x`) and search (`clg`) can only be run with a single instance. We are in the process of open-sourcing parallelized versions of these as well. From 2cb47511d4aa59816be761f3ef9c364acfaaf4df Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 08:10:50 -0400 Subject: [PATCH 099/108] Fix docs error. --- docs/src/user-guide/core-unstructured/clp.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index c215998d7d..775b812f50 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -130,13 +130,11 @@ You can configure `clp` and `clg` to use a MySQL-type database as follows: * `--db-table-prefix ` to specify the table prefix * Set the `CLP_DB_USER` environment variable for the database user's username * Set the `CLP_DB_PASS` environment variable for the database user's password - -To compress logs in parallel using the MySQL database: - * Run `clp` or `clg` with the same command-line options and environment variables, with the addition of the database type command-line option: * `--db-type mysql` to specify MySQL as the database type -* Run as many parallel `clp` or `clg` instances as desired. + +To compress logs in parallel, you can now run as many parallel instances of `clp` as desired. Note that currently, decompression (`clp x`) and search (`clg`) can only be run with a single instance. We are in the process of open-sourcing parallelized versions of these as well. From 602436bf6d9e7c0a81552c07d411091eed7cb5d3 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:03:51 -0400 Subject: [PATCH 100/108] Fix clang-tidy errors I introduced. --- components/core/src/clp/clg/clg.cpp | 2 - components/core/src/clp/clp/compression.cpp | 2 - .../core/src/clp/global_metadata_db_utils.cpp | 90 +++++++++---------- .../core/src/clp/global_metadata_db_utils.hpp | 4 + .../tests/test-GlobalMetadataDBConfig.cpp | 10 +-- 5 files changed, 51 insertions(+), 57 deletions(-) diff --git a/components/core/src/clp/clg/clg.cpp b/components/core/src/clp/clg/clg.cpp index b754c61215..7122a948a4 100644 --- a/components/core/src/clp/clg/clg.cpp +++ b/components/core/src/clp/clg/clg.cpp @@ -10,8 +10,6 @@ #include "../Defs.h" #include "../global_metadata_db_utils.hpp" -#include "../GlobalMySQLMetadataDB.hpp" -#include "../GlobalSQLiteMetadataDB.hpp" #include "../Grep.hpp" #include "../GrepCore.hpp" #include "../Profiler.hpp" diff --git a/components/core/src/clp/clp/compression.cpp b/components/core/src/clp/clp/compression.cpp index 964c0fc6f7..0625053618 100644 --- a/components/core/src/clp/clp/compression.cpp +++ b/components/core/src/clp/clp/compression.cpp @@ -8,8 +8,6 @@ #include #include "../global_metadata_db_utils.hpp" -#include "../GlobalMySQLMetadataDB.hpp" -#include "../GlobalSQLiteMetadataDB.hpp" #include "../spdlog_with_specializations.hpp" #include "../streaming_archive/writer/Archive.hpp" #include "../streaming_archive/writer/utils.hpp" diff --git a/components/core/src/clp/global_metadata_db_utils.cpp b/components/core/src/clp/global_metadata_db_utils.cpp index 921f9123df..8b6a58091b 100644 --- a/components/core/src/clp/global_metadata_db_utils.cpp +++ b/components/core/src/clp/global_metadata_db_utils.cpp @@ -1,68 +1,64 @@ #include "global_metadata_db_utils.hpp" +#include +#include +#include +#include + #include -#include "ErrorCode.hpp" +#include "GlobalMetadataDB.hpp" #include "GlobalMetadataDBConfig.hpp" #include "GlobalMySQLMetadataDB.hpp" #include "GlobalSQLiteMetadataDB.hpp" -#include "TraceableException.hpp" +#include "streaming_archive/Constants.hpp" namespace clp { auto create_global_metadata_db( std::optional const& optional_metadata_db_config, std::filesystem::path const& archives_dir ) -> std::unique_ptr { - constexpr std::string_view cErrorMsgPrefix = "Failed to load global metadata DB config: "; - try { - auto const& metadata_db_config = optional_metadata_db_config.value(); - switch (metadata_db_config.get_metadata_db_type()) { - case GlobalMetadataDBConfig::MetadataDBType::SQLite: { - auto global_metadata_db_path - = archives_dir / streaming_archive::cMetadataDBFileName; - return std::make_unique(global_metadata_db_path.string()); - } - case GlobalMetadataDBConfig::MetadataDBType::MySQL: { - return std::make_unique( - metadata_db_config.get_metadata_db_host(), - metadata_db_config.get_metadata_db_port(), - metadata_db_config.get_metadata_db_username().value(), - metadata_db_config.get_metadata_db_password().value(), - metadata_db_config.get_metadata_db_name(), - metadata_db_config.get_metadata_table_prefix() - ); - } - default: + constexpr std::string_view cErrorMsgPrefix = "Failed to load global metadata DB config"; + + if (false == optional_metadata_db_config.has_value()) { + SPDLOG_ERROR("{}: Global metadata DB config not set.", cErrorMsgPrefix); + return {}; + } + auto const& metadata_db_config = optional_metadata_db_config.value(); + + switch (metadata_db_config.get_metadata_db_type()) { + case GlobalMetadataDBConfig::MetadataDBType::SQLite: { + auto global_metadata_db_path = archives_dir / streaming_archive::cMetadataDBFileName; + return std::make_unique(global_metadata_db_path.string()); + } + case GlobalMetadataDBConfig::MetadataDBType::MySQL: { + auto const& metadata_db_username = metadata_db_config.get_metadata_db_username(); + auto const& metadata_db_password = metadata_db_config.get_metadata_db_password(); + if (false == metadata_db_username.has_value() + || false == metadata_db_password.has_value()) + { SPDLOG_ERROR( - "Unhandled metadata DB type: {}", - static_cast(metadata_db_config.get_metadata_db_type()) + "{}: Global MySQL metadata DB credentials unexpectedly not set.", + cErrorMsgPrefix ); - return nullptr; - } - } catch (TraceableException const& e) { - auto error_code = e.get_error_code(); - if (ErrorCode_errno == error_code) { - SPDLOG_ERROR( - "{}: {}:{} {}, errno={}", - cErrorMsgPrefix, - e.get_filename(), - e.get_line_number(), - e.what(), - errno + return {}; + } + return std::make_unique( + metadata_db_config.get_metadata_db_host(), + metadata_db_config.get_metadata_db_port(), + metadata_db_username.value(), + metadata_db_password.value(), + metadata_db_config.get_metadata_db_name(), + metadata_db_config.get_metadata_table_prefix() ); - } else { + } + default: SPDLOG_ERROR( - "{}: {}:{} {}, error_code={}", - cErrorMsgPrefix, - e.get_filename(), - e.get_line_number(), - e.what(), - error_code + "Unhandled metadata DB type: {}", + static_cast(metadata_db_config.get_metadata_db_type()) ); - } - } catch (std::exception const& e) { - SPDLOG_ERROR("{}: {}", cErrorMsgPrefix, e.what()); + return {}; } - return nullptr; + return {}; } } // namespace clp diff --git a/components/core/src/clp/global_metadata_db_utils.hpp b/components/core/src/clp/global_metadata_db_utils.hpp index 92399b9c7f..671eac9eea 100644 --- a/components/core/src/clp/global_metadata_db_utils.hpp +++ b/components/core/src/clp/global_metadata_db_utils.hpp @@ -1,6 +1,10 @@ #ifndef CLP_GLOBAL_METADATA_DB_UTILS_HPP #define CLP_GLOBAL_METADATA_DB_UTILS_HPP +#include +#include +#include + #include "GlobalMetadataDB.hpp" #include "GlobalMetadataDBConfig.hpp" diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 128643eb00..3dae104278 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -15,6 +15,7 @@ using namespace clp; namespace { // Constants +constexpr std::string_view cMysqlDbType{"mysql"}; constexpr std::string_view cMysqlDbHost{"test-host"}; constexpr std::string_view cMysqlDbPort{"8888"}; constexpr std::string_view cMysqlDbName{"test-db"}; @@ -24,9 +25,7 @@ constexpr std::string_view cMysqlDbPass{"test-pass"}; constexpr std::array cCommonMysqlArgv{ "test", "--db-type", - GlobalMetadataDBConfig::cMetadataDBTypeNames - [enum_to_underlying_type(GlobalMetadataDBConfig::MetadataDBType::MySQL)] - .data(), + cMysqlDbType.data(), "--db-host", cMysqlDbHost.data(), "--db-port", @@ -41,12 +40,11 @@ constexpr size_t cArgIdxDbPort{6}; constexpr size_t cArgIdxDbName{8}; constexpr size_t cArgIdxDbTablePrefix{10}; +constexpr std::string_view cSqliteDbType{"sqlite"}; constexpr std::array cCommonSqliteArgv{ "test", "--db-type", - GlobalMetadataDBConfig::cMetadataDBTypeNames - [enum_to_underlying_type(GlobalMetadataDBConfig::MetadataDBType::SQLite)] - .data(), + cSqliteDbType.data(), // Placeholders for overwriting with MySQL-specific arguments like `--db-host`. "", From e93131b91a2ae78a97d017010a67ad594905e2a4 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:04:06 -0400 Subject: [PATCH 101/108] Remove unnecessary default value. --- components/core/src/clp_s/indexer/IndexManager.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp_s/indexer/IndexManager.hpp b/components/core/src/clp_s/indexer/IndexManager.hpp index 92f7fd6fa8..52ace23d3c 100644 --- a/components/core/src/clp_s/indexer/IndexManager.hpp +++ b/components/core/src/clp_s/indexer/IndexManager.hpp @@ -45,7 +45,7 @@ class IndexManager { * @throw OperationFailed if `optional_db_config` is invalid. */ IndexManager( - std::optional const& optional_db_config = std::nullopt, + std::optional const& optional_db_config, bool should_create_table = false ); From 87c352459cf385d23d9366a0635f2db958ad2322 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 10:35:07 -0400 Subject: [PATCH 102/108] Fix pre-existing clang-tidy issues. --- .../tests/test-GlobalMetadataDBConfig.cpp | 140 +++++++++--------- 1 file changed, 74 insertions(+), 66 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 3dae104278..08aa83b1af 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -23,17 +24,17 @@ constexpr std::string_view cMysqlDbTablePrefix{"test_prefix_"}; constexpr std::string_view cMysqlDbUser{"test-user"}; constexpr std::string_view cMysqlDbPass{"test-pass"}; constexpr std::array cCommonMysqlArgv{ - "test", - "--db-type", - cMysqlDbType.data(), - "--db-host", - cMysqlDbHost.data(), - "--db-port", - cMysqlDbPort.data(), - "--db-name", - cMysqlDbName.data(), - "--db-table-prefix", - cMysqlDbTablePrefix.data() + std::string_view{"test"}, + std::string_view{"--db-type"}, + cMysqlDbType, + std::string_view{"--db-host"}, + cMysqlDbHost, + std::string_view{"--db-port"}, + cMysqlDbPort, + std::string_view{"--db-name"}, + cMysqlDbName, + std::string_view{"--db-table-prefix"}, + cMysqlDbTablePrefix }; constexpr size_t cArgIdxDbHost{4}; constexpr size_t cArgIdxDbPort{6}; @@ -42,13 +43,13 @@ constexpr size_t cArgIdxDbTablePrefix{10}; constexpr std::string_view cSqliteDbType{"sqlite"}; constexpr std::array cCommonSqliteArgv{ - "test", - "--db-type", - cSqliteDbType.data(), + std::string_view{"test"}, + std::string_view{"--db-type"}, + cSqliteDbType, // Placeholders for overwriting with MySQL-specific arguments like `--db-host`. - "", - "", + std::string_view{""}, + std::string_view{""}, }; constexpr size_t cArgIdxOverwriteArgName{3}; constexpr size_t cArgIdxOverwriteArgValue{4}; @@ -66,14 +67,21 @@ void unset_env_var(char const* name) { } template -auto parse_args(std::array const& argv) -> GlobalMetadataDBConfig { +auto parse_args(std::array const& argv) -> GlobalMetadataDBConfig { boost::program_options::options_description options_desc; GlobalMetadataDBConfig config{options_desc}; boost::program_options::variables_map vm; constexpr auto cArgc{static_cast(n)}; + + std::vector argv_strings; + std::vector argv_c_strs; + for (auto const& arg : argv) { + argv_strings.emplace_back(arg); + argv_c_strs.emplace_back(argv_strings.back().c_str()); + } boost::program_options::store( - boost::program_options::parse_command_line(cArgc, argv.data(), options_desc), + boost::program_options::parse_command_line(cArgc, argv_c_strs.data(), options_desc), vm ); boost::program_options::notify(vm); @@ -100,8 +108,8 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo auto config{parse_args(cCommonMysqlArgv)}; SECTION("With valid credentials") { - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_USER", std::string{cMysqlDbUser}.c_str()); + set_env_var("CLP_DB_PASS", std::string{cMysqlDbPass}.c_str()); config.read_credentials_from_env_if_needed(); REQUIRE_NOTHROW(config.validate()); @@ -110,7 +118,7 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo } SECTION("With empty password") { - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_USER", std::string{cMysqlDbUser}.c_str()); set_env_var("CLP_DB_PASS", ""); config.read_credentials_from_env_if_needed(); @@ -125,12 +133,12 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Password set but username missing") { - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_PASS", std::string{cMysqlDbPass}.c_str()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Username set but password missing") { - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); + set_env_var("CLP_DB_USER", std::string{cMysqlDbUser}.c_str()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -139,21 +147,21 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With invalid port values") { SECTION("Port too low") { - auto cArgv{cCommonMysqlArgv}; - cArgv[cArgIdxDbPort] = "0"; - auto config{parse_args(cArgv)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + auto argv{cCommonMysqlArgv}; + argv[cArgIdxDbPort] = "0"; + auto config{parse_args(argv)}; + set_env_var("CLP_DB_USER", std::string{cMysqlDbUser}.c_str()); + set_env_var("CLP_DB_PASS", std::string{cMysqlDbPass}.c_str()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Port too high") { - auto cArgv{cCommonMysqlArgv}; - cArgv[cArgIdxDbPort] = "65536"; - auto config{parse_args(cArgv)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + auto argv{cCommonMysqlArgv}; + argv[cArgIdxDbPort] = "65536"; + auto config{parse_args(argv)}; + set_env_var("CLP_DB_USER", std::string{cMysqlDbUser}.c_str()); + set_env_var("CLP_DB_PASS", std::string{cMysqlDbPass}.c_str()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -161,31 +169,31 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo SECTION("With empty required arguments") { SECTION("Empty db-host") { - auto cArgv{cCommonMysqlArgv}; - cArgv[cArgIdxDbHost] = ""; - auto config{parse_args(cArgv)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + auto argv{cCommonMysqlArgv}; + argv[cArgIdxDbHost] = ""; + auto config{parse_args(argv)}; + set_env_var("CLP_DB_USER", std::string{cMysqlDbUser}.c_str()); + set_env_var("CLP_DB_PASS", std::string{cMysqlDbPass}.c_str()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Empty db-name") { - auto cArgv{cCommonMysqlArgv}; - cArgv[cArgIdxDbName] = ""; - auto config{parse_args(cArgv)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + auto argv{cCommonMysqlArgv}; + argv[cArgIdxDbName] = ""; + auto config{parse_args(argv)}; + set_env_var("CLP_DB_USER", std::string{cMysqlDbUser}.c_str()); + set_env_var("CLP_DB_PASS", std::string{cMysqlDbPass}.c_str()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("Empty db-table-prefix") { - auto cArgv{cCommonMysqlArgv}; - cArgv[cArgIdxDbTablePrefix] = ""; - auto config{parse_args(cArgv)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + auto argv{cCommonMysqlArgv}; + argv[cArgIdxDbTablePrefix] = ""; + auto config{parse_args(argv)}; + set_env_var("CLP_DB_USER", std::string{cMysqlDbUser}.c_str()); + set_env_var("CLP_DB_PASS", std::string{cMysqlDbPass}.c_str()); config.read_credentials_from_env_if_needed(); REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -198,37 +206,37 @@ TEST_CASE("Test MySQL arguments and credential validation", "[GlobalMetadataDBCo TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With non-default db-host argument") { - auto cArgv{cCommonSqliteArgv}; - cArgv[cArgIdxOverwriteArgName] = "--db-host"; - cArgv[cArgIdxOverwriteArgValue] = cMysqlDbHost.data(); - auto const config{parse_args(cArgv)}; + auto argv{cCommonSqliteArgv}; + argv[cArgIdxOverwriteArgName] = "--db-host"; + argv[cArgIdxOverwriteArgValue] = cMysqlDbHost; + auto const config{parse_args(argv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-port argument") { - auto cArgv{cCommonSqliteArgv}; - cArgv[cArgIdxOverwriteArgName] = "--db-port"; - cArgv[cArgIdxOverwriteArgValue] = cMysqlDbPort.data(); - auto const config{parse_args(cArgv)}; + auto argv{cCommonSqliteArgv}; + argv[cArgIdxOverwriteArgName] = "--db-port"; + argv[cArgIdxOverwriteArgValue] = cMysqlDbPort; + auto const config{parse_args(argv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-name argument") { - auto cArgv{cCommonSqliteArgv}; - cArgv[cArgIdxOverwriteArgName] = "--db-name"; - cArgv[cArgIdxOverwriteArgValue] = cMysqlDbName.data(); - auto const config{parse_args(cArgv)}; + auto argv{cCommonSqliteArgv}; + argv[cArgIdxOverwriteArgName] = "--db-name"; + argv[cArgIdxOverwriteArgValue] = cMysqlDbName; + auto const config{parse_args(argv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } SECTION("With non-default db-table-prefix argument") { - auto cArgv{cCommonSqliteArgv}; - cArgv[cArgIdxOverwriteArgName] = "--db-table-prefix"; - cArgv[cArgIdxOverwriteArgValue] = cMysqlDbTablePrefix.data(); - auto const config{parse_args(cArgv)}; + auto argv{cCommonSqliteArgv}; + argv[cArgIdxOverwriteArgName] = "--db-table-prefix"; + argv[cArgIdxOverwriteArgValue] = cMysqlDbTablePrefix; + auto const config{parse_args(argv)}; REQUIRE_THROWS_AS(config.validate(), std::invalid_argument); } @@ -236,8 +244,8 @@ TEST_CASE("Test SQLite arguments", "[GlobalMetadataDBConfig]") { SECTION("With username and password") { auto config{parse_args(cCommonSqliteArgv)}; - set_env_var("CLP_DB_USER", cMysqlDbUser.data()); - set_env_var("CLP_DB_PASS", cMysqlDbPass.data()); + set_env_var("CLP_DB_USER", std::string{cMysqlDbUser}.c_str()); + set_env_var("CLP_DB_PASS", std::string{cMysqlDbPass}.c_str()); config.read_credentials_from_env_if_needed(); REQUIRE_FALSE(config.get_metadata_db_username().has_value()); From 7ccf42e8741fc09a7247021f5ef30eb33546e4e8 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 12:51:51 -0400 Subject: [PATCH 103/108] Fix unit tests failure. --- components/core/tests/test-GlobalMetadataDBConfig.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/components/core/tests/test-GlobalMetadataDBConfig.cpp b/components/core/tests/test-GlobalMetadataDBConfig.cpp index 08aa83b1af..2bff1a62ab 100644 --- a/components/core/tests/test-GlobalMetadataDBConfig.cpp +++ b/components/core/tests/test-GlobalMetadataDBConfig.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -74,11 +73,11 @@ auto parse_args(std::array const& argv) -> GlobalMetadataDB boost::program_options::variables_map vm; constexpr auto cArgc{static_cast(n)}; - std::vector argv_strings; - std::vector argv_c_strs; - for (auto const& arg : argv) { - argv_strings.emplace_back(arg); - argv_c_strs.emplace_back(argv_strings.back().c_str()); + std::array argv_strings; + std::array argv_c_strs{}; + for (size_t i = 0; i < n; ++i) { + argv_strings.at(i) = argv.at(i); + argv_c_strs.at(i) = argv_strings.at(i).c_str(); } boost::program_options::store( boost::program_options::parse_command_line(cArgc, argv_c_strs.data(), options_desc), From f6056116521a268da789b475312fd187864e9a7a Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 12:52:15 -0400 Subject: [PATCH 104/108] Document return correctly. --- components/core/src/clp/global_metadata_db_utils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/global_metadata_db_utils.hpp b/components/core/src/clp/global_metadata_db_utils.hpp index 671eac9eea..c99752bfd4 100644 --- a/components/core/src/clp/global_metadata_db_utils.hpp +++ b/components/core/src/clp/global_metadata_db_utils.hpp @@ -14,7 +14,7 @@ namespace clp { * * @param optional_metadata_db_config * @param archives_dir - * @return The created GlobalMetadataDB instance. + * @return The created GlobalMetadataDB instance, or nullptr on failure. */ auto create_global_metadata_db( std::optional const& optional_metadata_db_config, From 32b06fece6b7c2102d41795f92e60e9a9d7b826d Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 12:53:33 -0400 Subject: [PATCH 105/108] Doc tweaks. --- docs/src/user-guide/core-unstructured/clp.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index 775b812f50..2def1165af 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -105,9 +105,9 @@ Currently, timestamps must be specified as milliseconds since the UNIX epoch. # Parallel Compression -To enable parallel compression to the same archives directory, `clp`/`clg` need to be configured to -use a MySQL-type database (e.g., MariaDB) rather than the default---an embedded SQLite database -(which doesn't support concurrent writes). +To enable parallel compression to the same archives directory, `clp` (and by extension, `clg`) needs +to be configured to use a MySQL-type database (e.g., MariaDB) rather than the default---an embedded +SQLite database (which doesn't support concurrent writes). :::{warning} Running multiple `clp` instances with SQLite can fail due to the error "database is locked". @@ -134,7 +134,7 @@ You can configure `clp` and `clg` to use a MySQL-type database as follows: of the database type command-line option: * `--db-type mysql` to specify MySQL as the database type -To compress logs in parallel, you can now run as many parallel instances of `clp` as desired. +To compress logs in parallel, run as many parallel instances of `clp` as desired. Note that currently, decompression (`clp x`) and search (`clg`) can only be run with a single instance. We are in the process of open-sourcing parallelized versions of these as well. From e2ca0b853dd00f0dd15e4279dac807806ebd2d7d Mon Sep 17 00:00:00 2001 From: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 15:24:02 -0400 Subject: [PATCH 106/108] Apply suggestions from code review Co-authored-by: Junhao Liao --- docs/src/user-guide/core-unstructured/clp.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index 2def1165af..796ce4efd7 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -106,14 +106,14 @@ Currently, timestamps must be specified as milliseconds since the UNIX epoch. # Parallel Compression To enable parallel compression to the same archives directory, `clp` (and by extension, `clg`) needs -to be configured to use a MySQL-type database (e.g., MariaDB) rather than the default---an embedded +to be configured to use a MySQL-compatible database (e.g., MariaDB) rather than the default---an embedded SQLite database (which doesn't support concurrent writes). :::{warning} Running multiple `clp` instances with SQLite can fail due to the error "database is locked". ::: -You can configure `clp` and `clg` to use a MySQL-type database as follows: +You can configure `clp` and `clg` to use a MySQL-compatible database as follows: * Install and configure MariaDB using the instructions for your platform * Create a user that has privileges to create databases, create tables, insert records, and delete From e7fd7234cbdf2daf3d9f0863fa8f945bb9276249 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 15:26:16 -0400 Subject: [PATCH 107/108] Wrap to 100 chars. --- docs/src/user-guide/core-unstructured/clp.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md index 796ce4efd7..32b1a51731 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -106,8 +106,8 @@ Currently, timestamps must be specified as milliseconds since the UNIX epoch. # Parallel Compression To enable parallel compression to the same archives directory, `clp` (and by extension, `clg`) needs -to be configured to use a MySQL-compatible database (e.g., MariaDB) rather than the default---an embedded -SQLite database (which doesn't support concurrent writes). +to be configured to use a MySQL-compatible database (e.g., MariaDB) rather than the default---an +embedded SQLite database (which doesn't support concurrent writes). :::{warning} Running multiple `clp` instances with SQLite can fail due to the error "database is locked". From 5267b9f5ede822be4c79e60863b1ad11e964f9a4 Mon Sep 17 00:00:00 2001 From: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 13 Aug 2025 17:32:22 -0400 Subject: [PATCH 108/108] Update components/core/src/clp/clp/compression.cpp Co-authored-by: Junhao Liao --- components/core/src/clp/clp/compression.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp/clp/compression.cpp b/components/core/src/clp/clp/compression.cpp index 0625053618..4fbda717a0 100644 --- a/components/core/src/clp/clp/compression.cpp +++ b/components/core/src/clp/clp/compression.cpp @@ -62,7 +62,7 @@ bool compress( std::unique_ptr reader_parser, bool use_heuristic ) { - auto output_dir = std::filesystem::path(command_line_args.get_archives_dir()); + auto output_dir = std::filesystem::path(command_line_args.get_output_dir()); // Create output directory in case it doesn't exist auto error_code = create_directory(output_dir.parent_path().string(), 0700, true);