feat(indexer): Rename CLI arg table-name to dataset-name for consistency; Update the name of the column metadata table name to put the dataset name first.

components/clp-py-utils/clp_py_utils/clp_metadata_db_utils.py

@@ -81,10 +81,10 @@ def _create_files_table(db_cursor, table_prefix: str) -> None:
     )
 
 
-def _create_column_metadata_table(db_cursor, table_name: str) -> None:
+def _create_column_metadata_table(db_cursor, table_prefix: str) -> None:
     db_cursor.execute(
         f"""
-        CREATE TABLE IF NOT EXISTS `{table_name}` (
+        CREATE TABLE IF NOT EXISTS `{table_prefix}{COLUMN_METADATA_TABLE_SUFFIX}` (
             `name` VARCHAR(512) NOT NULL,
             `type` TINYINT NOT NULL,
             PRIMARY KEY (`name`, `type`)
@@ -126,17 +126,13 @@ def create_metadata_db_tables(db_cursor, table_prefix: str, dataset: str | None
     tags_table_name = f"{table_prefix}{TAGS_TABLE_SUFFIX}"
     archive_tags_table_name = f"{table_prefix}{ARCHIVE_TAGS_TABLE_SUFFIX}"
 
-    # TODO: Update this to
-    # {table_prefix}{CLP_DEFAULT_DATASET_NAME}_{COLUMN_METADATA_TABLE_SUFFIX} when we can also
-    # change the indexer to match.
-    column_metadata_table_name = (
-        f"{table_prefix}{COLUMN_METADATA_TABLE_SUFFIX}_{CLP_DEFAULT_DATASET_NAME}"
-    )
-
     _create_archives_table(db_cursor, archives_table_name)
     _create_tags_table(db_cursor, tags_table_name)
     _create_archive_tags_table(
         db_cursor, archive_tags_table_name, archives_table_name, tags_table_name
     )
     _create_files_table(db_cursor, table_prefix)
-    _create_column_metadata_table(db_cursor, column_metadata_table_name)
+
+    # TODO: Create this table only for the `CLP_S` storage-engine after the dataset feature is
+    # fully implemented.
+    _create_column_metadata_table(db_cursor, f"{table_prefix}{CLP_DEFAULT_DATASET_NAME}_")

components/core/src/clp_s/indexer/CommandLineArguments.cpp

@@ -32,7 +32,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) {
     )(
             "create-table",
             po::bool_switch(&m_should_create_table),
-            "Create the table if it doesn't exist"
+            "Create the column metadata table if it doesn't exist"
     );
     // clang-format on
 
@@ -45,17 +45,17 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) {
     po::options_description positional_options;
     // clang-format off
     positional_options.add_options()(
-            "table-name",
-            po::value<std::string>(&m_table_name),
-            "Name of the table where fields from the archive will be indexed and stored"
+            "dataset-name",
+            po::value<std::string>(&m_dataset_name),
+            "Name of the dataset for which the column metadata table should be populated"
     )(
             "archive-path",
             po::value<std::string>(&archive_path),
             "Path to an archive"
     );
     // clang-format on
     po::positional_options_description positional_options_description;
-    positional_options_description.add("table-name", 1);
+    positional_options_description.add("dataset-name", 1);
     positional_options_description.add("archive-path", 1);
 
     // Aggregate all options
@@ -89,8 +89,8 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) {
         }
 
         // Validate required parameters
-        if (m_table_name.empty()) {
-            throw std::invalid_argument("Table name not specified or empty.");
+        if (m_dataset_name.empty()) {
+            throw std::invalid_argument("Dataset name not specified or empty.");
         }
         if (archive_path.empty()) {
             throw std::invalid_argument("Archive path not specified or empty.");
@@ -127,7 +127,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) {
 }
 
 void CommandLineArguments::print_basic_usage() const {
-    std::cerr << "Usage: " << get_program_name() << " [OPTIONS] TABLE_NAME ARCHIVE_PATH"
+    std::cerr << "Usage: " << get_program_name() << " [OPTIONS] DATASET_NAME ARCHIVE_PATH"
               << std::endl;
 }
 }  // namespace clp_s::indexer

components/core/src/clp_s/indexer/CommandLineArguments.hpp

@@ -28,7 +28,7 @@ class CommandLineArguments {
 
     std::string const& get_program_name() const { return m_program_name; }
 
-    std::string const& get_table_name() const { return m_table_name; }
+    std::string const& get_dataset_name() const { return m_dataset_name; }
 
     Path const& get_archive_path() const { return m_archive_path; }
 
@@ -44,7 +44,7 @@ class CommandLineArguments {
 
     // Variables
     std::string m_program_name;
-    std::string m_table_name;
+    std::string m_dataset_name;
     Path m_archive_path;
 
     std::optional<clp::GlobalMetadataDBConfig> m_metadata_db_config;

components/core/src/clp_s/indexer/IndexManager.cpp

@@ -37,8 +37,8 @@ IndexManager::~IndexManager() {
     }
 }
 
-void IndexManager::update_metadata(std::string const& table_name, Path const& archive_path) {
-    m_mysql_index_storage->init(table_name, m_should_create_table);
+void IndexManager::update_metadata(std::string const& dataset_name, Path const& archive_path) {
+    m_mysql_index_storage->init(dataset_name, m_should_create_table);
 
     ArchiveReader archive_reader;
     archive_reader.open(archive_path, NetworkAuthOption{});

components/core/src/clp_s/indexer/IndexManager.hpp

@@ -48,10 +48,10 @@ class IndexManager {
     // Methods
     /**
      * Updates the metadata for a given archive
-     * @param table_name
+     * @param dataset_name
      * @param archive_path
      */
-    void update_metadata(std::string const& table_name, Path const& archive_path);
+    void update_metadata(std::string const& dataset_name, Path const& archive_path);
 
 private:
     /**

components/core/src/clp_s/indexer/MySQLIndexStorage.cpp

@@ -22,20 +22,22 @@ void MySQLIndexStorage::open() {
     m_is_open = true;
 }
 
-void MySQLIndexStorage::init(std::string const& table_name, bool should_create_table) {
+void MySQLIndexStorage::init(std::string const& dataset_name, bool should_create_table) {
     if (false == m_is_open) {
         throw OperationFailed(ErrorCodeNotReady, __FILENAME__, __LINE__);
     }
 
+    auto const table_name{
+            fmt::format("{}{}_{}", m_table_prefix, dataset_name, cColumnMetadataTableSuffix)
+    };
     if (should_create_table) {
         m_db.execute_query(
                 fmt::format(
-                        "CREATE TABLE IF NOT EXISTS {}{} ("
+                        "CREATE TABLE IF NOT EXISTS {} ("
                         "name VARCHAR(512) NOT NULL, "
                         "type TINYINT NOT NULL,"
                         "PRIMARY KEY (name, type)"
                         ")",
-                        m_table_prefix,
                         table_name
                 )
         );
@@ -55,14 +57,13 @@ void MySQLIndexStorage::init(std::string const& table_name, bool should_create_t
 
     fmt::format_to(
             statement_buffer_ix,
-            "INSERT IGNORE INTO {}{} ({}) VALUES ({})",
-            m_table_prefix,
+            "INSERT IGNORE INTO {} ({}) VALUES ({})",
             table_name,
             clp::get_field_names_sql(table_metadata_field_names),
             clp::get_placeholders_sql(table_metadata_field_names.size())
     );
     SPDLOG_DEBUG("{:.{}}", statement_buffer.data(), statement_buffer.size());
-    m_insert_field_statement = std::make_unique<MySQLPreparedStatement>(
+    m_insert_field_statement = std::make_unique<clp::MySQLPreparedStatement>(
             m_db.prepare_statement(statement_buffer.data(), statement_buffer.size())
     );
 

components/core/src/clp_s/indexer/MySQLIndexStorage.hpp

@@ -6,16 +6,13 @@
 #include "../SchemaTree.hpp"
 #include "../TraceableException.hpp"
 
-using clp::MySQLDB;
-using clp::MySQLPreparedStatement;
-
 namespace clp_s::indexer {
 /**
  * Class representing a MySQL storage for column metadata (column names and types)
  */
 class MySQLIndexStorage {
 public:
-    static constexpr char cColumnMetadataPrefix[] = "column_metadata_";
+    static constexpr char cColumnMetadataTableSuffix[] = "column_metadata";
 
     // Types
     class OperationFailed : public TraceableException {
@@ -41,7 +38,7 @@ class MySQLIndexStorage {
               m_username(username),
               m_password(password),
               m_database_name(database_name),
-              m_table_prefix(table_prefix + cColumnMetadataPrefix) {}
+              m_table_prefix(table_prefix) {}
 
     // Methods
     /**
@@ -51,10 +48,10 @@ class MySQLIndexStorage {
 
     /**
      * Creates the table if it is required and prepares the insert statement
-     * @param table_name
+     * @param dataset_name
      * @param should_create_table
      */
-    void init(std::string const& table_name, bool should_create_table);
+    void init(std::string const& dataset_name, bool should_create_table);
 
     /**
      * Closes the database connection
@@ -79,9 +76,9 @@ class MySQLIndexStorage {
     std::string m_database_name;
     std::string m_table_prefix;
 
-    MySQLDB m_db;
+    clp::MySQLDB m_db;
 
-    std::unique_ptr<MySQLPreparedStatement> m_insert_field_statement;
+    std::unique_ptr<clp::MySQLPreparedStatement> m_insert_field_statement;
 };
 }  // namespace clp_s::indexer
 

components/core/src/clp_s/indexer/indexer.cpp

@@ -34,7 +34,7 @@ int main(int argc, char const* argv[]) {
     try {
         clp_s::indexer::IndexManager index_manager(command_line_arguments.get_db_config());
         index_manager.update_metadata(
-                command_line_arguments.get_table_name(),
+                command_line_arguments.get_dataset_name(),
                 command_line_arguments.get_archive_path()
         );
     } catch (std::exception& e) {

components/job-orchestration/job_orchestration/executor/compress/compression_task.py

@@ -11,6 +11,7 @@
 from celery.utils.log import get_task_logger
 from clp_py_utils.clp_config import (
     ARCHIVES_TABLE_SUFFIX,
+    CLP_DEFAULT_DATASET_NAME,
     COMPRESSION_JOBS_TABLE_NAME,
     COMPRESSION_TASKS_TABLE_NAME,
     Database,
@@ -374,14 +375,11 @@ def run_clp(
                     db_conn.commit()
 
                 if StorageEngine.CLP_S == clp_storage_engine:
-                    # TODO: Since CLP doesn't currently support datasets but users of the index
-                    # require a dataset name, we hardcode a name for now.
-                    dataset_name = "default"
                     indexer_cmd = [
                         str(clp_home / "bin" / "indexer"),
                         "--db-config-file",
                         str(db_config_file_path),
-                        dataset_name,
+                        CLP_DEFAULT_DATASET_NAME,
                         archive_path,
                     ]
                     try: