Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Comment thread
Bill-hbrhbr marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ def handle_job_update(db, db_cursor, job_id, no_progress_reporting):
)

job_last_uncompressed_size = 0
last_current_time = datetime.datetime.now()

while True:
db_cursor.execute(polling_query)
Expand All @@ -79,7 +80,11 @@ def handle_job_update(db, db_cursor, job_id, no_progress_reporting):
# All tasks in the job is done
if not no_progress_reporting:
logger.info("Compression finished.")
print_compression_job_status(job_row, current_time)
if job_last_uncompressed_size < job_uncompressed_size:
print_compression_job_status(job_row, current_time)
else:
# No progress in the final iteration; repeat the last status update verbatim
print_compression_job_status(job_row, last_current_time)
break # Done
if CompressionJobStatus.FAILED == job_status:
# One or more tasks in the job has failed
Expand All @@ -98,6 +103,7 @@ def handle_job_update(db, db_cursor, job_id, no_progress_reporting):
error_msg = f"Unhandled CompressionJobStatus: {job_status}"
raise NotImplementedError(error_msg)

last_current_time = current_time
time.sleep(0.5)


Expand Down
1 change: 1 addition & 0 deletions components/clp-py-utils/clp_py_utils/clp_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
QUERY_TASKS_TABLE_NAME = "query_tasks"
COMPRESSION_JOBS_TABLE_NAME = "compression_jobs"
COMPRESSION_TASKS_TABLE_NAME = "compression_tasks"
ARCHIVES_TABLE_NAME = "clp_archives"

OS_RELEASE_FILE_PATH = pathlib.Path("etc") / "os-release"

Expand Down
24 changes: 2 additions & 22 deletions components/core/src/clp_s/ArchiveWriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,6 @@ void ArchiveWriter::close() {
header_and_metadata_writer.close();
}

if (m_metadata_db) {
update_metadata_db();
}

if (m_print_archive_stats) {
print_archive_stats();
}
Expand Down Expand Up @@ -430,27 +426,11 @@ std::pair<size_t, size_t> ArchiveWriter::store_tables() {
return {table_metadata_compressed_size, table_compressed_size};
}

void ArchiveWriter::update_metadata_db() {
m_metadata_db->open();
clp::streaming_archive::ArchiveMetadata metadata(
cArchiveFormatDevelopmentVersionFlag,
"",
0ULL
);
metadata.increment_static_compressed_size(m_compressed_size);
metadata.increment_static_uncompressed_size(m_uncompressed_size);
metadata.expand_time_range(
m_timestamp_dict.get_begin_timestamp(),
m_timestamp_dict.get_end_timestamp()
);

m_metadata_db->add_archive(m_id, metadata);
m_metadata_db->close();
}

void ArchiveWriter::print_archive_stats() {
nlohmann::json json_msg;
json_msg["id"] = m_id;
json_msg["begin_timestamp"] = m_timestamp_dict.get_begin_timestamp();
json_msg["end_timestamp"] = m_timestamp_dict.get_end_timestamp();
json_msg["uncompressed_size"] = m_uncompressed_size;
json_msg["size"] = m_compressed_size;
std::cout << json_msg.dump(-1, ' ', true, nlohmann::json::error_handler_t::ignore) << std::endl;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A few minor things while we're touching this method.

  1. Could we mark print_archive_stats as const?
  2. Could we replace these string literals with some constexpr std::string_view that live in ArchiveWriter.hpp?

Expand Down
10 changes: 1 addition & 9 deletions components/core/src/clp_s/ArchiveWriter.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_io.hpp>

#include "../clp/GlobalMySQLMetadataDB.hpp"
#include "archive_constants.hpp"
#include "DictionaryWriter.hpp"
#include "Schema.hpp"
Expand Down Expand Up @@ -66,8 +65,7 @@ class ArchiveWriter {
};

// Constructor
explicit ArchiveWriter(std::shared_ptr<clp::GlobalMySQLMetadataDB> metadata_db)
: m_metadata_db(std::move(metadata_db)) {}
ArchiveWriter() = default;

// Destructor
~ArchiveWriter() = default;
Expand Down Expand Up @@ -211,11 +209,6 @@ class ArchiveWriter {
*/
void write_archive_header(FileWriter& archive_writer, size_t metadata_section_size);

/**
* Updates the metadata db with the archive's metadata (id, size, timestamp ranges, etc.)
*/
void update_metadata_db();

/**
* Prints the archive's statistics (id, uncompressed size, compressed size, etc.)
*/
Expand All @@ -238,7 +231,6 @@ class ArchiveWriter {
std::shared_ptr<LogTypeDictionaryWriter> m_log_dict;
std::shared_ptr<LogTypeDictionaryWriter> m_array_dict; // log type dictionary for arrays
TimestampDictionaryWriter m_timestamp_dict;
std::shared_ptr<clp::GlobalMySQLMetadataDB> m_metadata_db;
int m_compression_level{};
bool m_print_archive_stats{};
bool m_single_file_archive{};
Expand Down
2 changes: 0 additions & 2 deletions components/core/src/clp_s/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,6 @@ set(
../clp/GlobalMetadataDB.hpp
../clp/GlobalMetadataDBConfig.cpp
../clp/GlobalMetadataDBConfig.hpp
../clp/GlobalMySQLMetadataDB.cpp
Comment thread
Bill-hbrhbr marked this conversation as resolved.
Resolved
../clp/GlobalMySQLMetadataDB.hpp
../clp/hash_utils.cpp
../clp/hash_utils.hpp
../clp/ir/EncodedTextAst.cpp
Expand Down
29 changes: 0 additions & 29 deletions components/core/src/clp_s/CommandLineArguments.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,6 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) {
// clang-format on

po::options_description compression_options("Compression options");
std::string metadata_db_config_file_path;
std::string input_path_list_file_path;
constexpr std::string_view cJsonFileType{"json"};
constexpr std::string_view cKeyValueIrFileType{"kv-ir"};
Expand Down Expand Up @@ -238,11 +237,6 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) {
po::value<std::string>(&m_timestamp_key)->value_name("TIMESTAMP_COLUMN_KEY")->
default_value(m_timestamp_key),
"Path (e.g. x.y) for the field containing the log event's timestamp."
)(
"db-config-file",
po::value<std::string>(&metadata_db_config_file_path)->value_name("FILE")->
default_value(metadata_db_config_file_path),
"Global metadata DB YAML config"
)(
"files-from,f",
po::value<std::string>(&input_path_list_file_path)
Expand Down Expand Up @@ -353,29 +347,6 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) {
}

validate_network_auth(auth, m_network_auth);

// Parse and validate global metadata DB config
if (false == metadata_db_config_file_path.empty()) {
clp::GlobalMetadataDBConfig metadata_db_config;
try {
metadata_db_config.parse_config_file(metadata_db_config_file_path);
} catch (std::exception& e) {
SPDLOG_ERROR("Failed to validate metadata database config - {}.", e.what());
return ParsingResult::Failure;
}

if (clp::GlobalMetadataDBConfig::MetadataDBType::MySQL
!= metadata_db_config.get_metadata_db_type())
{
SPDLOG_ERROR(
"Invalid metadata database type for {}; only supported type is MySQL.",
m_program_name
);
return ParsingResult::Failure;
}

m_metadata_db_config = std::move(metadata_db_config);
}
} else if ((char)Command::Extract == command_input) {
po::options_description extraction_options;
std::string archive_path;
Expand Down
8 changes: 0 additions & 8 deletions components/core/src/clp_s/CommandLineArguments.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/variables_map.hpp>

#include "../clp/GlobalMetadataDBConfig.hpp"
#include "../reducer/types.hpp"
#include "Defs.hpp"
#include "InputConfig.hpp"
Expand Down Expand Up @@ -94,10 +93,6 @@ class CommandLineArguments {

bool get_ignore_case() const { return m_ignore_case; }

std::optional<clp::GlobalMetadataDBConfig> const& get_metadata_db_config() const {
return m_metadata_db_config;
}

std::string const& get_reducer_host() const { return m_reducer_host; }

int get_reducer_port() const { return m_reducer_port; }
Expand Down Expand Up @@ -200,9 +195,6 @@ class CommandLineArguments {
bool m_disable_log_order{false};
FileType m_file_type{FileType::Json};

// Metadata db variables
std::optional<clp::GlobalMetadataDBConfig> m_metadata_db_config;

// MongoDB configuration variables
std::string m_mongodb_uri;
std::string m_mongodb_collection;
Expand Down
2 changes: 1 addition & 1 deletion components/core/src/clp_s/JsonParser.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ JsonParser::JsonParser(JsonParserOption const& option)
m_archive_options.authoritative_timestamp = m_timestamp_column;
m_archive_options.authoritative_timestamp_namespace = m_timestamp_namespace;

m_archive_writer = std::make_unique<ArchiveWriter>(option.metadata_db);
m_archive_writer = std::make_unique<ArchiveWriter>();
m_archive_writer->open(m_archive_options);
}

Expand Down
2 changes: 0 additions & 2 deletions components/core/src/clp_s/JsonParser.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
#include "../clp/ffi/KeyValuePairLogEvent.hpp"
#include "../clp/ffi/SchemaTree.hpp"
#include "../clp/ffi/Value.hpp"
#include "../clp/GlobalMySQLMetadataDB.hpp"
#include "../clp/ReaderInterface.hpp"
#include "ArchiveWriter.hpp"
#include "CommandLineArguments.hpp"
Expand Down Expand Up @@ -51,7 +50,6 @@ struct JsonParserOption {
bool structurize_arrays{};
bool record_log_order{true};
bool single_file_archive{false};
std::shared_ptr<clp::GlobalMySQLMetadataDB> metadata_db;
NetworkAuthOption network_auth{};
};

Expand Down
14 changes: 0 additions & 14 deletions components/core/src/clp_s/clp-s.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
#include <spdlog/spdlog.h>

#include "../clp/CurlGlobalInstance.hpp"
#include "../clp/GlobalMySQLMetadataDB.hpp"
#include "../clp/streaming_archive/ArchiveMetadata.hpp"
#include "../reducer/network_utils.hpp"
#include "CommandLineArguments.hpp"
Expand Down Expand Up @@ -102,19 +101,6 @@ bool compress(CommandLineArguments const& command_line_arguments) {
option.structurize_arrays = command_line_arguments.get_structurize_arrays();
option.record_log_order = command_line_arguments.get_record_log_order();

auto const& db_config_container = command_line_arguments.get_metadata_db_config();
if (db_config_container.has_value()) {
auto const& db_config = db_config_container.value();
option.metadata_db = std::make_shared<clp::GlobalMySQLMetadataDB>(
db_config.get_metadata_db_host(),
db_config.get_metadata_db_port(),
db_config.get_metadata_db_username(),
db_config.get_metadata_db_password(),
db_config.get_metadata_db_name(),
db_config.get_metadata_table_prefix()
);
}

clp_s::JsonParser parser(option);
if (CommandLineArguments::FileType::KeyValueIr == option.input_file_type) {
if (false == parser.parse_from_ir()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from celery.app.task import Task
from celery.utils.log import get_task_logger
from clp_py_utils.clp_config import (
ARCHIVES_TABLE_NAME,
COMPRESSION_JOBS_TABLE_NAME,
COMPRESSION_TASKS_TABLE_NAME,
Database,
Expand Down Expand Up @@ -82,6 +83,21 @@ def update_job_metadata_and_tags(db_cursor, job_id, table_prefix, tag_ids, archi
)


def update_archive_metadata(db_cursor, archive_stats):
archive_stats_defaults = {
"begin_timestamp": 0,
"end_timestamp": 0,
"creator_id": "",
"creation_ix": 0,
}
for k, v in archive_stats_defaults.items():
archive_stats.setdefault(k, v)
keys = ", ".join(archive_stats.keys())
value_placeholders = ", ".join(["%s"] * len(archive_stats))
query = f"INSERT INTO {ARCHIVES_TABLE_NAME} ({keys}) VALUES ({value_placeholders})"
Comment thread
Bill-hbrhbr marked this conversation as resolved.
Resolved
db_cursor.execute(query, list(archive_stats.values()))


def _generate_fs_logs_list(
output_file_path: pathlib.Path,
paths_to_compress: PathsToCompress,
Expand Down Expand Up @@ -182,7 +198,6 @@ def make_clp_s_command_and_env(
"--target-encoded-size",
str(clp_config.output.target_segment_size + clp_config.output.target_dictionaries_size),
"--compression-level", str(clp_config.output.compression_level),
"--db-config-file", str(db_config_file_path),
Comment thread
Bill-hbrhbr marked this conversation as resolved.
Resolved
]
# fmt: on

Expand Down Expand Up @@ -347,6 +362,8 @@ def run_clp(
with closing(sql_adapter.create_connection(True)) as db_conn, closing(
db_conn.cursor(dictionary=True)
) as db_cursor:
if StorageEngine.CLP_S == clp_storage_engine:
update_archive_metadata(db_cursor, last_archive_stats)
update_job_metadata_and_tags(
db_cursor,
job_id,
Expand Down