Machinery for loading external Agent (encryptor) out of a shared library

cpp/src/arrow/util/io_util.cc

@@ -2278,4 +2278,24 @@ Result<void*> GetSymbol(void* handle, const char* name) {
 #endif
 }
 
+Status CloseDynamicLibrary(void* handle) {
+  if (handle == nullptr) {
+    return Status::Invalid("Attempting to close null library handle");
+  }
+#ifdef _WIN32
+  if (FreeLibrary(reinterpret_cast<HMODULE>(handle))) {
+    return Status::OK();
+  }
+  // win32 api doc: "If the function fails, the return value is zero."
+  return IOErrorFromWinError(GetLastError(), "FreeLibrary() failed");
+#else
+  if (dlclose(handle) == 0) {
+    return Status::OK();
+  }
+  // dlclose(3) man page: "On success, dlclose() returns 0; on error, it returns a nonzero value."
+  auto* error = dlerror();
+  return Status::IOError("dlclose() failed: ", error ? error : "unknown error");
+#endif
+}
+
 }  // namespace arrow::internal

cpp/src/arrow/util/io_util.h

@@ -443,6 +443,13 @@ ARROW_EXPORT Result<void*> LoadDynamicLibrary(const char* path);
 ///         returned; instead an error will be raised.
 ARROW_EXPORT Result<void*> GetSymbol(void* handle, const char* name);
 
+/// \brief Close a dynamic library
+///
+/// This wraps dlclose() except on Windows, where FreeLibrary() is called.
+///
+/// \return Status::OK() if the library was closed successfully, otherwise an error is returned.
+ARROW_EXPORT Status CloseDynamicLibrary(void* handle);
+
 template <typename T>
 Result<T*> GetSymbolAs(void* handle, const char* name) {
   ARROW_ASSIGN_OR_RAISE(void* sym, GetSymbol(handle, name));

cpp/src/parquet/CMakeLists.txt

@@ -238,8 +238,11 @@ endif()
 
 if(PARQUET_REQUIRE_ENCRYPTION)
   list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS ${ARROW_OPENSSL_LIBS})
+  list(APPEND PARQUET_STATIC_LINK_LIBS ${ARROW_OPENSSL_LIBS})
   set(PARQUET_SRCS ${PARQUET_SRCS} encryption/encryption_internal.cc
-                   encryption/openssl_internal.cc)
+                   encryption/openssl_internal.cc
+                   encryption/external/loadable_encryptor_utils.cc
+                   encryption/external/dbpa_library_wrapper.cc)
   # Encryption key management
   set(PARQUET_SRCS
       ${PARQUET_SRCS}

cpp/src/parquet/encryption/CMakeLists.txt

@@ -17,3 +17,28 @@
 
 # Headers: public api
 arrow_install_all_headers("parquet/encryption")
+
+if(ARROW_TESTING)
+  add_library(DBPATestAgent SHARED
+     external/dbpa_test_agent.cc
+  )
+
+  # DBPATestAgent configuration
+  target_link_libraries(DBPATestAgent PUBLIC
+    arrow_shared
+  )
+
+  set_target_properties(DBPATestAgent PROPERTIES OUTPUT_NAME "DBPATestAgent")
+
+  # Add test for DBPALibraryWrapper
+  add_parquet_test(dbpa-library-wrapper-test
+                   SOURCES external/dbpa_library_wrapper_test.cc
+                   LABELS "parquet-tests" "encryption-tests")
+
+  # Add test for LoadableEncryptorUtils
+  add_parquet_test(loadable-encryptor-utils-test
+                   SOURCES external/loadable_encryptor_utils_test.cc
+                   EXTRA_LINK_LIBS DBPATestAgent
+                   LABELS "parquet-tests" "encryption-tests")
+endif()
+

cpp/src/parquet/encryption/external/dbpa_interface.h

@@ -0,0 +1,32 @@
+//TODO: figure out the licensing.
+
+#pragma once
+
+#include <memory>
+#include "parquet/platform.h"
+#include "arrow/util/span.h"
+
+using ::arrow::util::span;
+
+namespace parquet::encryption::external {
+
+    //TODO: this will change once we have a solid defition of interfaces 
+
+    class EncryptionResult {
+    };
+
+    class DecryptionResult {
+    };
+
+    class PARQUET_EXPORT DataBatchProtectionAgentInterface {
+        public:
+         virtual std::unique_ptr<EncryptionResult> Encrypt(
+            span<const uint8_t> plaintext, 
+            span<uint8_t> ciphertext) = 0;
+
+        virtual std::unique_ptr<DecryptionResult> Decrypt(
+            span<const uint8_t> ciphertext) = 0;
+
+        virtual ~DataBatchProtectionAgentInterface() = default;
+    };
+}

cpp/src/parquet/encryption/external/dbpa_library_wrapper.cc

@@ -0,0 +1,80 @@
+//TODO: figure out the licensing.
+
+#include "parquet/encryption/external/dbpa_library_wrapper.h"
+#include "parquet/encryption/external/dbpa_interface.h"
+#include "arrow/util/span.h"
+#include <dlfcn.h>
+#include <stdexcept>
+#include <functional>
+
+#include <iostream>
+
+#include "arrow/util/io_util.h"
+
+using ::arrow::util::span;
+
+namespace parquet::encryption::external {
+
+// Default implementation for handle closing function
+void DefaultSharedLibraryClosingFn(void* library_handle) {
+  auto status = arrow::internal::CloseDynamicLibrary(library_handle);
+  if (!status.ok()) {
+    std::cerr << "Error closing library: " << status.message() << std::endl;
+  }
+}
+
+DBPALibraryWrapper::DBPALibraryWrapper(
+    std::unique_ptr<DataBatchProtectionAgentInterface> agent,
+    void* library_handle,
+    std::function<void(void*)> handle_closing_fn)
+    : wrapped_agent_(std::move(agent)), 
+      library_handle_(library_handle),
+      handle_closing_fn_(std::move(handle_closing_fn)) {
+  // Ensure the wrapped agent is not null
+  if (!wrapped_agent_) {
+    throw std::invalid_argument("DBPAWrapper: Cannot create wrapper with null agent");
+  }
+  if (!library_handle_) {
+    throw std::invalid_argument("DBPAWrapper: Cannot create wrapper with null library handle");
+  }
+  if (!handle_closing_fn_) {
+    throw std::invalid_argument("DBPAWrapper: Cannot create wrapper with null handle closing function");
+  }
+}
+
+// DBPALibraryWrapper destructor
+// This is the main reason for the decorator/wrapper.
+// This will (a) destroy the wrapped agent, and (b) close the shared library.
+// While the wrapped_agent_ would automatically be destroyed when this object is destroyed
+// we need to explicitly destroy **before** we are able to close the shared library.
+// Doing it in a different order, may cause issues, as by unloading the library may cause the class
+// definition to be unloaded before the destructor completes, and that is likely to cause issues 
+// (such as a segfault).
+DBPALibraryWrapper::~DBPALibraryWrapper() {
+  // Explicitly destroy the wrapped agent first
+  if (wrapped_agent_) {
+    DataBatchProtectionAgentInterface* wrapped_agent = wrapped_agent_.release();
+    delete wrapped_agent;
+  }
+
+  // Now we can close the shared library using the provided function
+  handle_closing_fn_(library_handle_);
+  library_handle_ = nullptr;
+}
+
+// Decorator implementation of Encrypt method
+std::unique_ptr<EncryptionResult> DBPALibraryWrapper::Encrypt(
+    span<const uint8_t> plaintext,
+    span<uint8_t> ciphertext) {
+
+  return wrapped_agent_->Encrypt(plaintext, ciphertext);
+}
+
+// Decorator implementation of Decrypt method
+std::unique_ptr<DecryptionResult> DBPALibraryWrapper::Decrypt(
+    span<const uint8_t> ciphertext) {
+
+    return wrapped_agent_->Decrypt(ciphertext);
+}
+
+}  // namespace parquet::encryption::external 

cpp/src/parquet/encryption/external/dbpa_library_wrapper.h

@@ -0,0 +1,59 @@
+//TODO: figure out the licensing.
+
+#pragma once
+
+#include <memory>
+#include <functional>
+
+#include "parquet/encryption/external/dbpa_interface.h"
+#include "arrow/util/span.h"
+
+using ::arrow::util::span;
+
+namespace parquet::encryption::external {
+
+// Default implementation for shared library closing function
+// This is passed into the constructor of DBPALibraryWrapper, 
+// and is used as the default function to close the shared library.
+void DefaultSharedLibraryClosingFn(void* library_handle);
+
+// Decorator/Wrapper class for the DataBatchProtectionAgentInterface
+// Its main purpose is to close the shared library when Arrow is about to destroy 
+// an intance of an DPBAgent
+//
+// In the constructor we allow to pass a function that will be used to close the shared library.
+// This simplifies testing, as we can use a mock function to avoid actually closing the shared library.
+class DBPALibraryWrapper : public DataBatchProtectionAgentInterface {
+ private:
+  std::unique_ptr<DataBatchProtectionAgentInterface> wrapped_agent_;
+  void* library_handle_;
+  std::function<void(void*)> handle_closing_fn_;
+
+ public:
+  // Constructor that takes ownership of the wrapped agent
+  explicit DBPALibraryWrapper(
+      std::unique_ptr<DataBatchProtectionAgentInterface> agent,
+      void* library_handle,
+      std::function<void(void*)> handle_closing_fn = &DefaultSharedLibraryClosingFn);
+
+  // Destructor
+  // This is the main reason for the decorator/wrapper.
+  // This will (a) destroy the wrapped agent, and (b) close the shared library.
+  // While the wrapped_agent_ would automatically be destroyed when this object is destroyed
+  // we need to explicitly destroy **before** we are able to close the shared library.
+  // Doing it in a different order, may cause issues, as by unloading the library may cause the class
+  // definition to be unloaded before the destructor completes, and that is likely to cause issues 
+  // (such as a segfault).
+  ~DBPALibraryWrapper() override;
+
+  // Decorator implementation of Encrypt method
+  std::unique_ptr<EncryptionResult> Encrypt(
+      span<const uint8_t> plaintext,
+      span<uint8_t> ciphertext) override;
+
+  // Decorator implementation of Decrypt method
+  std::unique_ptr<DecryptionResult> Decrypt(
+      span<const uint8_t> ciphertext) override;
+};
+
+}  // namespace parquet::encryption::external