diff --git a/components/clp-py-utils/clp_py_utils/clp_metadata_db_utils.py b/components/clp-py-utils/clp_py_utils/clp_metadata_db_utils.py index f5420aaf16..6a8c927b6c 100644 --- a/components/clp-py-utils/clp_py_utils/clp_metadata_db_utils.py +++ b/components/clp-py-utils/clp_py_utils/clp_metadata_db_utils.py @@ -81,10 +81,10 @@ def _create_files_table(db_cursor, table_prefix: str) -> None: ) -def _create_column_metadata_table(db_cursor, table_name: str) -> None: +def _create_column_metadata_table(db_cursor, table_prefix: str) -> None: db_cursor.execute( f""" - CREATE TABLE IF NOT EXISTS `{table_name}` ( + CREATE TABLE IF NOT EXISTS `{table_prefix}{COLUMN_METADATA_TABLE_SUFFIX}` ( `name` VARCHAR(512) NOT NULL, `type` TINYINT NOT NULL, PRIMARY KEY (`name`, `type`) @@ -126,17 +126,13 @@ def create_metadata_db_tables(db_cursor, table_prefix: str, dataset: str | None tags_table_name = f"{table_prefix}{TAGS_TABLE_SUFFIX}" archive_tags_table_name = f"{table_prefix}{ARCHIVE_TAGS_TABLE_SUFFIX}" - # TODO: Update this to - # {table_prefix}{CLP_DEFAULT_DATASET_NAME}_{COLUMN_METADATA_TABLE_SUFFIX} when we can also - # change the indexer to match. - column_metadata_table_name = ( - f"{table_prefix}{COLUMN_METADATA_TABLE_SUFFIX}_{CLP_DEFAULT_DATASET_NAME}" - ) - _create_archives_table(db_cursor, archives_table_name) _create_tags_table(db_cursor, tags_table_name) _create_archive_tags_table( db_cursor, archive_tags_table_name, archives_table_name, tags_table_name ) _create_files_table(db_cursor, table_prefix) - _create_column_metadata_table(db_cursor, column_metadata_table_name) + + # TODO: Create this table only for the `CLP_S` storage-engine after the dataset feature is + # fully implemented. + _create_column_metadata_table(db_cursor, f"{table_prefix}{CLP_DEFAULT_DATASET_NAME}_") diff --git a/components/core/src/clp_s/indexer/CommandLineArguments.cpp b/components/core/src/clp_s/indexer/CommandLineArguments.cpp index 488650a54b..da94ef0c30 100644 --- a/components/core/src/clp_s/indexer/CommandLineArguments.cpp +++ b/components/core/src/clp_s/indexer/CommandLineArguments.cpp @@ -32,7 +32,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { )( "create-table", po::bool_switch(&m_should_create_table), - "Create the table if it doesn't exist" + "Create the column metadata table if it doesn't exist" ); // clang-format on @@ -45,9 +45,9 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { po::options_description positional_options; // clang-format off positional_options.add_options()( - "table-name", - po::value(&m_table_name), - "Name of the table where fields from the archive will be indexed and stored" + "dataset-name", + po::value(&m_dataset_name), + "Name of the dataset for which the column metadata table should be populated" )( "archive-path", po::value(&archive_path), @@ -55,7 +55,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { ); // clang-format on po::positional_options_description positional_options_description; - positional_options_description.add("table-name", 1); + positional_options_description.add("dataset-name", 1); positional_options_description.add("archive-path", 1); // Aggregate all options @@ -89,8 +89,8 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { } // Validate required parameters - if (m_table_name.empty()) { - throw std::invalid_argument("Table name not specified or empty."); + if (m_dataset_name.empty()) { + throw std::invalid_argument("Dataset name not specified or empty."); } if (archive_path.empty()) { throw std::invalid_argument("Archive path not specified or empty."); @@ -127,7 +127,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { } void CommandLineArguments::print_basic_usage() const { - std::cerr << "Usage: " << get_program_name() << " [OPTIONS] TABLE_NAME ARCHIVE_PATH" + std::cerr << "Usage: " << get_program_name() << " [OPTIONS] DATASET_NAME ARCHIVE_PATH" << std::endl; } } // namespace clp_s::indexer diff --git a/components/core/src/clp_s/indexer/CommandLineArguments.hpp b/components/core/src/clp_s/indexer/CommandLineArguments.hpp index 890157ebe9..43a528a570 100644 --- a/components/core/src/clp_s/indexer/CommandLineArguments.hpp +++ b/components/core/src/clp_s/indexer/CommandLineArguments.hpp @@ -28,7 +28,7 @@ class CommandLineArguments { std::string const& get_program_name() const { return m_program_name; } - std::string const& get_table_name() const { return m_table_name; } + std::string const& get_dataset_name() const { return m_dataset_name; } Path const& get_archive_path() const { return m_archive_path; } @@ -44,7 +44,7 @@ class CommandLineArguments { // Variables std::string m_program_name; - std::string m_table_name; + std::string m_dataset_name; Path m_archive_path; std::optional m_metadata_db_config; diff --git a/components/core/src/clp_s/indexer/IndexManager.cpp b/components/core/src/clp_s/indexer/IndexManager.cpp index 98fea48d33..b023f3611b 100644 --- a/components/core/src/clp_s/indexer/IndexManager.cpp +++ b/components/core/src/clp_s/indexer/IndexManager.cpp @@ -37,8 +37,8 @@ IndexManager::~IndexManager() { } } -void IndexManager::update_metadata(std::string const& table_name, Path const& archive_path) { - m_mysql_index_storage->init(table_name, m_should_create_table); +void IndexManager::update_metadata(std::string const& dataset_name, Path const& archive_path) { + m_mysql_index_storage->init(dataset_name, m_should_create_table); ArchiveReader archive_reader; archive_reader.open(archive_path, NetworkAuthOption{}); diff --git a/components/core/src/clp_s/indexer/IndexManager.hpp b/components/core/src/clp_s/indexer/IndexManager.hpp index b9f412c0c5..618d897710 100644 --- a/components/core/src/clp_s/indexer/IndexManager.hpp +++ b/components/core/src/clp_s/indexer/IndexManager.hpp @@ -48,10 +48,10 @@ class IndexManager { // Methods /** * Updates the metadata for a given archive - * @param table_name + * @param dataset_name * @param archive_path */ - void update_metadata(std::string const& table_name, Path const& archive_path); + void update_metadata(std::string const& dataset_name, Path const& archive_path); private: /** diff --git a/components/core/src/clp_s/indexer/MySQLIndexStorage.cpp b/components/core/src/clp_s/indexer/MySQLIndexStorage.cpp index ba4dba01e2..9b28b982cf 100644 --- a/components/core/src/clp_s/indexer/MySQLIndexStorage.cpp +++ b/components/core/src/clp_s/indexer/MySQLIndexStorage.cpp @@ -22,20 +22,22 @@ void MySQLIndexStorage::open() { m_is_open = true; } -void MySQLIndexStorage::init(std::string const& table_name, bool should_create_table) { +void MySQLIndexStorage::init(std::string const& dataset_name, bool should_create_table) { if (false == m_is_open) { throw OperationFailed(ErrorCodeNotReady, __FILENAME__, __LINE__); } + auto const table_name{ + fmt::format("{}{}_{}", m_table_prefix, dataset_name, cColumnMetadataTableSuffix) + }; if (should_create_table) { m_db.execute_query( fmt::format( - "CREATE TABLE IF NOT EXISTS {}{} (" + "CREATE TABLE IF NOT EXISTS {} (" "name VARCHAR(512) NOT NULL, " "type TINYINT NOT NULL," "PRIMARY KEY (name, type)" ")", - m_table_prefix, table_name ) ); @@ -55,14 +57,13 @@ void MySQLIndexStorage::init(std::string const& table_name, bool should_create_t fmt::format_to( statement_buffer_ix, - "INSERT IGNORE INTO {}{} ({}) VALUES ({})", - m_table_prefix, + "INSERT IGNORE INTO {} ({}) VALUES ({})", table_name, clp::get_field_names_sql(table_metadata_field_names), clp::get_placeholders_sql(table_metadata_field_names.size()) ); SPDLOG_DEBUG("{:.{}}", statement_buffer.data(), statement_buffer.size()); - m_insert_field_statement = std::make_unique( + m_insert_field_statement = std::make_unique( m_db.prepare_statement(statement_buffer.data(), statement_buffer.size()) ); diff --git a/components/core/src/clp_s/indexer/MySQLIndexStorage.hpp b/components/core/src/clp_s/indexer/MySQLIndexStorage.hpp index 7374c1aa53..db93690cb0 100644 --- a/components/core/src/clp_s/indexer/MySQLIndexStorage.hpp +++ b/components/core/src/clp_s/indexer/MySQLIndexStorage.hpp @@ -6,16 +6,13 @@ #include "../SchemaTree.hpp" #include "../TraceableException.hpp" -using clp::MySQLDB; -using clp::MySQLPreparedStatement; - namespace clp_s::indexer { /** * Class representing a MySQL storage for column metadata (column names and types) */ class MySQLIndexStorage { public: - static constexpr char cColumnMetadataPrefix[] = "column_metadata_"; + static constexpr char cColumnMetadataTableSuffix[] = "column_metadata"; // Types class OperationFailed : public TraceableException { @@ -41,7 +38,7 @@ class MySQLIndexStorage { m_username(username), m_password(password), m_database_name(database_name), - m_table_prefix(table_prefix + cColumnMetadataPrefix) {} + m_table_prefix(table_prefix) {} // Methods /** @@ -51,10 +48,10 @@ class MySQLIndexStorage { /** * Creates the table if it is required and prepares the insert statement - * @param table_name + * @param dataset_name * @param should_create_table */ - void init(std::string const& table_name, bool should_create_table); + void init(std::string const& dataset_name, bool should_create_table); /** * Closes the database connection @@ -79,9 +76,9 @@ class MySQLIndexStorage { std::string m_database_name; std::string m_table_prefix; - MySQLDB m_db; + clp::MySQLDB m_db; - std::unique_ptr m_insert_field_statement; + std::unique_ptr m_insert_field_statement; }; } // namespace clp_s::indexer diff --git a/components/core/src/clp_s/indexer/indexer.cpp b/components/core/src/clp_s/indexer/indexer.cpp index fdd2aa5bf7..76797d507d 100644 --- a/components/core/src/clp_s/indexer/indexer.cpp +++ b/components/core/src/clp_s/indexer/indexer.cpp @@ -34,7 +34,7 @@ int main(int argc, char const* argv[]) { try { clp_s::indexer::IndexManager index_manager(command_line_arguments.get_db_config()); index_manager.update_metadata( - command_line_arguments.get_table_name(), + command_line_arguments.get_dataset_name(), command_line_arguments.get_archive_path() ); } catch (std::exception& e) { diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index 474f333ec1..4ac42d708c 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -11,6 +11,7 @@ from celery.utils.log import get_task_logger from clp_py_utils.clp_config import ( ARCHIVES_TABLE_SUFFIX, + CLP_DEFAULT_DATASET_NAME, COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_TASKS_TABLE_NAME, Database, @@ -374,14 +375,11 @@ def run_clp( db_conn.commit() if StorageEngine.CLP_S == clp_storage_engine: - # TODO: Since CLP doesn't currently support datasets but users of the index - # require a dataset name, we hardcode a name for now. - dataset_name = "default" indexer_cmd = [ str(clp_home / "bin" / "indexer"), "--db-config-file", str(db_config_file_path), - dataset_name, + CLP_DEFAULT_DATASET_NAME, archive_path, ] try: