Merge branch 'main' into feat(clp-s)-JSON-to-IRv2

components/core/CMakeLists.txt

@@ -275,6 +275,8 @@ set(SOURCE_FILES_clp_s_unitTest
     src/clp_s/FileReader.hpp
     src/clp_s/FileWriter.cpp
     src/clp_s/FileWriter.hpp
+    src/clp_s/InputConfig.cpp
+    src/clp_s/InputConfig.hpp
     src/clp_s/JsonConstructor.cpp
     src/clp_s/JsonConstructor.hpp
     src/clp_s/JsonFileIterator.cpp
@@ -613,7 +615,7 @@ target_include_directories(unitTest
 target_link_libraries(unitTest
         PRIVATE
         absl::flat_hash_map
-        Boost::filesystem Boost::iostreams Boost::program_options Boost::regex
+        Boost::filesystem Boost::iostreams Boost::program_options Boost::regex Boost::url
         ${CURL_LIBRARIES}
         fmt::fmt
         kql

components/core/src/clp/BoundedReader.hpp

@@ -61,8 +61,8 @@ class BoundedReader : public ReaderInterface {
      * @return ErrorCode_EndOfFile on EOF or trying to read after hitting checkpoint
      * @return ErrorCode_Success on success
      */
-    [[nodiscard]] auto
-    try_read(char* buf, size_t num_bytes_to_read, size_t& num_bytes_read) -> ErrorCode override;
+    [[nodiscard]] auto try_read(char* buf, size_t num_bytes_to_read, size_t& num_bytes_read)
+            -> ErrorCode override;
 
     /**
      * This function is unsupported because BoundedReader can not delegate to a potentially

components/core/src/clp/BufferReader.cpp

@@ -86,12 +86,9 @@ auto BufferReader::try_get_pos(size_t& pos) -> ErrorCode {
     return ErrorCode_Success;
 }
 
-auto BufferReader::try_read_to_delimiter(
-        char delim,
-        bool keep_delimiter,
-        bool append,
-        std::string& str
-) -> ErrorCode {
+auto
+BufferReader::try_read_to_delimiter(char delim, bool keep_delimiter, bool append, std::string& str)
+        -> ErrorCode {
     if (false == append) {
         str.clear();
     }

components/core/src/clp/BufferReader.hpp

@@ -63,8 +63,8 @@ class BufferReader : public ReaderInterface {
      * @return ErrorCode_EndOfFile if the buffer doesn't contain any more data
      * @return ErrorCode_Success on success
      */
-    [[nodiscard]] auto
-    try_read(char* buf, size_t num_bytes_to_read, size_t& num_bytes_read) -> ErrorCode override;
+    [[nodiscard]] auto try_read(char* buf, size_t num_bytes_to_read, size_t& num_bytes_read)
+            -> ErrorCode override;
 
     /**
      * Tries to seek to the given position, relative to the beginning of the buffer
@@ -88,12 +88,9 @@ class BufferReader : public ReaderInterface {
      * @param str Returns the content read from the buffer
      * @return Same as BufferReader::try_read_to_delimiter(char, bool, std::string&, bool&, size_t&)
      */
-    [[nodiscard]] auto try_read_to_delimiter(
-            char delim,
-            bool keep_delimiter,
-            bool append,
-            std::string& str
-    ) -> ErrorCode override;
+    [[nodiscard]] auto
+    try_read_to_delimiter(char delim, bool keep_delimiter, bool append, std::string& str)
+            -> ErrorCode override;
 
 private:
     // Methods

components/core/src/clp/BufferedFileReader.cpp

@@ -265,12 +265,9 @@ auto BufferedFileReader::try_read(char* buf, size_t num_bytes_to_read, size_t& n
     return ErrorCode_Success;
 }
 
-auto BufferedFileReader::try_read_to_delimiter(
-        char delim,
-        bool keep_delimiter,
-        bool append,
-        string& str
-) -> ErrorCode {
+auto
+BufferedFileReader::try_read_to_delimiter(char delim, bool keep_delimiter, bool append, string& str)
+        -> ErrorCode {
     if (-1 == m_fd) {
         return ErrorCode_NotInit;
     }

components/core/src/clp/BufferedFileReader.hpp

@@ -127,8 +127,8 @@ class BufferedFileReader : public ReaderInterface {
      * @return ErrorCode_NotInit if the file is not opened
      * @return ErrorCode_Success on success
      */
-    [[nodiscard]] auto
-    try_peek_buffered_data(char const*& buf, size_t& peek_size) const -> ErrorCode;
+    [[nodiscard]] auto try_peek_buffered_data(char const*& buf, size_t& peek_size) const
+            -> ErrorCode;
 
     /**
      * Peeks the remaining buffered content without advancing the read head.
@@ -191,8 +191,8 @@ class BufferedFileReader : public ReaderInterface {
      * @return ErrorCode_EndOfFile on EOF
      * @return ErrorCode_Success on success
      */
-    [[nodiscard]] auto
-    try_read(char* buf, size_t num_bytes_to_read, size_t& num_bytes_read) -> ErrorCode override;
+    [[nodiscard]] auto try_read(char* buf, size_t num_bytes_to_read, size_t& num_bytes_read)
+            -> ErrorCode override;
 
     /**
      * Tries to read up to an occurrence of the given delimiter
@@ -206,12 +206,9 @@ class BufferedFileReader : public ReaderInterface {
      * @return Same as BufferReader::try_read_to_delimiter if it fails
      * @return ErrorCode_Success on success
      */
-    [[nodiscard]] auto try_read_to_delimiter(
-            char delim,
-            bool keep_delimiter,
-            bool append,
-            std::string& str
-    ) -> ErrorCode override;
+    [[nodiscard]] auto
+    try_read_to_delimiter(char delim, bool keep_delimiter, bool append, std::string& str)
+            -> ErrorCode override;
 
 private:
     // Methods

components/core/src/clp/FileDescriptor.cpp

@@ -48,7 +48,7 @@ FileDescriptor::~FileDescriptor() {
 }
 
 auto FileDescriptor::get_size() const -> size_t {
-    struct stat stat_result {};
+    struct stat stat_result{};
 
     if (ErrorCode_Success != stat(stat_result)) {
         throw OperationFailed(

components/core/src/clp/FileDescriptorReader.hpp

@@ -65,8 +65,8 @@ class FileDescriptorReader : public ReaderInterface {
      * @return ErrorCode_EndOfFile on EOF
      * @return ErrorCode_Success on success
      */
-    [[nodiscard]] auto
-    try_read(char* buf, size_t num_bytes_to_read, size_t& num_bytes_read) -> ErrorCode override;
+    [[nodiscard]] auto try_read(char* buf, size_t num_bytes_to_read, size_t& num_bytes_read)
+            -> ErrorCode override;
 
     /**
      * Tries to seek to the given position, relative to the beginning of the file.

components/core/src/clp/Grep.cpp

@@ -890,10 +890,8 @@ bool Grep::get_bounds_of_next_potential_var(
     return (value_length != begin_pos);
 }
 
-void Grep::calculate_sub_queries_relevant_to_file(
-        File const& compressed_file,
-        vector<Query>& queries
-) {
+void
+Grep::calculate_sub_queries_relevant_to_file(File const& compressed_file, vector<Query>& queries) {
     for (auto& query : queries) {
         query.make_sub_queries_relevant_to_segment(compressed_file.get_segment_id());
     }

components/core/src/clp/NetworkReader.cpp

@@ -108,8 +108,8 @@ extern "C" auto curl_progress_callback(
  * @return On success, the number of bytes processed. If this is less than `nmemb`, the download
  * will be aborted.
  */
-extern "C" auto
-curl_write_callback(char* ptr, size_t size, size_t nmemb, void* reader_ptr) -> size_t {
+extern "C" auto curl_write_callback(char* ptr, size_t size, size_t nmemb, void* reader_ptr)
+        -> size_t {
     return static_cast<NetworkReader*>(reader_ptr)->buffer_downloaded_data({ptr, size * nmemb});
 }
 }  // namespace
@@ -252,10 +252,10 @@ auto NetworkReader::acquire_empty_buffer() -> void {
     }
     std::unique_lock<std::mutex> buffer_resource_lock{m_buffer_resource_mutex};
     while (m_filled_buffer_queue.size() == m_buffer_pool_size) {
-        m_downloader_cv.wait(buffer_resource_lock);
         if (is_abort_download_requested()) {
             return;
         }
+        m_downloader_cv.wait(buffer_resource_lock);
     }
     m_curr_downloader_buf.emplace(
             m_buffer_pool.at(m_curr_downloader_buf_idx).data(),
@@ -304,11 +304,9 @@ auto NetworkReader::get_filled_buffer() -> void {
     m_curr_reader_buf.emplace(next_reader_buffer);
 }
 
-auto NetworkReader::read_from_filled_buffers(
-        size_t num_bytes_to_read,
-        size_t& num_bytes_read,
-        char* dst
-) -> ErrorCode {
+auto
+NetworkReader::read_from_filled_buffers(size_t num_bytes_to_read, size_t& num_bytes_read, char* dst)
+        -> ErrorCode {
     num_bytes_read = 0;
     std::optional<BufferView> dst_view;
     if (nullptr != dst) {

components/core/src/clp/NetworkReader.hpp

@@ -132,8 +132,8 @@ class NetworkReader : public ReaderInterface {
      * @return ErrorCode_EndOfFile if there is no more buffered data.
      * @return ErrorCode_Success on success.
      */
-    [[nodiscard]] auto
-    try_read(char* buf, size_t num_bytes_to_read, size_t& num_bytes_read) -> ErrorCode override {
+    [[nodiscard]] auto try_read(char* buf, size_t num_bytes_to_read, size_t& num_bytes_read)
+            -> ErrorCode override {
         return read_from_filled_buffers(num_bytes_to_read, num_bytes_read, buf);
     }
 
@@ -306,11 +306,9 @@ class NetworkReader : public ReaderInterface {
      * @return ErrorCode_EndOfFile if the buffer doesn't contain any more data.
      * @return ErrorCode_Success on success.
      */
-    [[nodiscard]] auto read_from_filled_buffers(
-            size_t num_bytes_to_read,
-            size_t& num_bytes_read,
-            char* dst
-    ) -> ErrorCode;
+    [[nodiscard]] auto
+    read_from_filled_buffers(size_t num_bytes_to_read, size_t& num_bytes_read, char* dst)
+            -> ErrorCode;
 
     /**
      * Sets the download completion status with the return code from curl.

components/core/src/clp/Thread.hpp

@@ -28,7 +28,7 @@ class Thread {
     };
 
     // Constructors
-    Thread() : m_thread_running(false) {};
+    Thread() : m_thread_running(false) {}
 
     // Destructor
     virtual ~Thread();

components/core/src/clp/Utils.cpp

@@ -88,57 +88,6 @@ ErrorCode create_directory_structure(string const& path, mode_t mode) {
     return ErrorCode_Success;
 }
 
-string get_parent_directory_path(string const& path) {
-    string dirname = get_unambiguous_path(path);
-
-    size_t last_slash_pos = dirname.find_last_of('/');
-    if (0 == last_slash_pos) {
-        dirname = "/";
-    } else if (string::npos == last_slash_pos) {
-        dirname = ".";
-    } else {
-        dirname.resize(last_slash_pos);
-    }
-
-    return dirname;
-}
-
-string get_unambiguous_path(string const& path) {
-    string unambiguous_path;
-    if (path.empty()) {
-        return unambiguous_path;
-    }
-
-    // Break path into components
-    vector<string> path_components;
-    boost::split(path_components, path, boost::is_any_of("/"), boost::token_compress_on);
-
-    // Remove ambiguous components
-    list<string> unambiguous_components;
-    size_t num_components_to_ignore = 0;
-    for (size_t i = path_components.size(); i-- > 0;) {
-        if (".." == path_components[i]) {
-            ++num_components_to_ignore;
-        } else if ("." == path_components[i] || path_components[i].empty()) {
-            // Do nothing
-        } else if (num_components_to_ignore > 0) {
-            --num_components_to_ignore;
-        } else {
-            unambiguous_components.emplace_front(path_components[i]);
-        }
-    }
-
-    // Assemble unambiguous path from leading slash (if any) and the unambiguous components
-    if ('/' == path[0]) {
-        unambiguous_path += '/';
-    }
-    if (!unambiguous_components.empty()) {
-        unambiguous_path += boost::join(unambiguous_components, "/");
-    }
-
-    return unambiguous_path;
-}
-
 ErrorCode read_list_of_paths(string const& list_path, vector<string>& paths) {
     unique_ptr<FileReader> file_reader;
     try {

components/core/src/clp/Utils.hpp

@@ -35,28 +35,6 @@ ErrorCode create_directory(std::string const& path, mode_t mode, bool exist_ok);
  */
 ErrorCode create_directory_structure(std::string const& path, mode_t mode);
 
-/**
- * Gets the parent directory path for a given path
- * Corner cases:
- * - get_dirname("abc") = "."
- * - get_dirname(".") = "."
- * - get_dirname("..") = "."
- * - get_dirname("/") = "/"
- * - get_dirname("/.") = "/"
- * - get_dirname("/..") = "/"
- * - get_dirname("/abc") = "/"
- * @param path
- * @return Parent directory path
- */
-std::string get_parent_directory_path(std::string const& path);
-
-/**
- * Removes ".", "..", and consecutive "/" from a given path and returns the result
- * @param path The given path
- * @return The unambiguous path
- */
-std::string get_unambiguous_path(std::string const& path);
-
 /**
  * Read a list of paths from a file
  * @param list_path

components/core/src/clp/aws/AwsAuthenticationSigner.cpp

@@ -43,8 +43,8 @@ namespace {
  * @param timestamp
  * @return The formatted date string.
  */
-[[nodiscard]] auto get_formatted_date_string(std::chrono::system_clock::time_point const& timestamp
-) -> string;
+[[nodiscard]] auto get_formatted_date_string(std::chrono::system_clock::time_point const& timestamp)
+        -> string;
 
 /**
  * Gets the string to sign required by AWS Signature Version 4 protocol.
@@ -89,8 +89,8 @@ auto is_unreserved_characters(char c) -> bool {
     return is_alphabet(c) || is_decimal_digit(c) || c == '-' || c == '_' || c == '.' || c == '~';
 }
 
-auto get_formatted_timestamp_string(std::chrono::system_clock::time_point const& timestamp
-) -> string {
+auto get_formatted_timestamp_string(std::chrono::system_clock::time_point const& timestamp)
+        -> string {
     return fmt::format("{:%Y%m%dT%H%M%SZ}", timestamp);
 }
 
@@ -203,10 +203,9 @@ S3Url::S3Url(string const& url) {
     m_host = fmt::format("{}.s3.{}.{}", m_bucket, m_region, m_end_point);
 }
 
-auto AwsAuthenticationSigner::generate_presigned_url(
-        S3Url const& s3_url,
-        string& presigned_url
-) const -> ErrorCode {
+auto
+AwsAuthenticationSigner::generate_presigned_url(S3Url const& s3_url, string& presigned_url) const
+        -> ErrorCode {
     auto const s3_region = s3_url.get_region();
 
     auto const now = std::chrono::system_clock::now();
@@ -245,10 +244,9 @@ auto AwsAuthenticationSigner::generate_presigned_url(
     return ErrorCode_Success;
 }
 
-auto AwsAuthenticationSigner::get_canonical_query_string(
-        string_view scope,
-        string_view timestamp
-) const -> string {
+auto
+AwsAuthenticationSigner::get_canonical_query_string(string_view scope, string_view timestamp) const
+        -> string {
     auto const uri = fmt::format("{}/{}", m_access_key_id, scope);
     return fmt::format(
             "{}={}&{}={}&{}={}&{}={}&{}={}",

components/core/src/clp/aws/AwsAuthenticationSigner.hpp

@@ -82,8 +82,8 @@ class AwsAuthenticationSigner {
      * @return ErrorCode_Success on success.
      * @return Same as `get_sha256_hash` and `AwsAuthenticationSigner::get_signature` on failure.
      */
-    [[nodiscard]] auto
-    generate_presigned_url(S3Url const& s3_url, std::string& presigned_url) const -> ErrorCode;
+    [[nodiscard]] auto generate_presigned_url(S3Url const& s3_url, std::string& presigned_url) const
+            -> ErrorCode;
 
 private:
     /**
@@ -92,10 +92,9 @@ class AwsAuthenticationSigner {
      * @param timestamp
      * @return The canonical query string.
      */
-    [[nodiscard]] auto get_canonical_query_string(
-            std::string_view scope,
-            std::string_view timestamp
-    ) const -> std::string;
+    [[nodiscard]] auto
+    get_canonical_query_string(std::string_view scope, std::string_view timestamp) const
+            -> std::string;
 
     /**
      * Gets the signature signing key for the request.