feat: Add Python class Serializer to serialize log events into CLP …

…key-value pair IR format. (#83)

Co-authored-by: haiqi96 <[email protected]>

CMakeLists.txt

@@ -113,6 +113,8 @@ set(CLP_FFI_PY_LIB_IR_SOURCES
     ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyMetadata.hpp
     ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyQuery.cpp
     ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PyQuery.hpp
+    ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PySerializer.cpp
+    ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/PySerializer.hpp
     ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/Query.cpp
     ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/Query.hpp
     ${CLP_FFI_PY_LIB_SRC_DIR}/ir/native/serialization_methods.cpp

clp_ffi_py/ir/__init__.py

@@ -18,6 +18,7 @@
     "Metadata",  # native
     "Query",  # native
     "QueryBuilder",  # query_builder
+    "Serializer",  # native
     "ClpIrFileReader",  # readers
     "ClpIrStreamReader",  # readers
 ]

clp_ffi_py/ir/native.pyi

@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 from datetime import tzinfo
-from typing import Any, Dict, IO, List, Optional
+from types import TracebackType
+from typing import Any, Dict, IO, List, Optional, Type
 
 from clp_ffi_py.wildcard_query import WildcardQuery
 
@@ -86,4 +89,18 @@ class KeyValuePairLogEvent:
     def __init__(self, dictionary: Dict[Any, Any]): ...
     def to_dict(self) -> Dict[Any, Any]: ...
 
+class Serializer:
+    def __init__(self, output_stream: IO[bytes], buffer_size_limit: int = 65536): ...
+    def __enter__(self) -> Serializer: ...
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> None: ...
+    def serialize_log_event_from_msgpack_map(self, msgpack_map: bytes) -> int: ...
+    def get_num_bytes_serialized(self) -> int: ...
+    def flush(self) -> None: ...
+    def close(self) -> None: ...
+
 class IncompleteStreamError(Exception): ...

src/clp_ffi_py/PyObjectCast.hpp

@@ -118,6 +118,7 @@ class PyKeyValuePairLogEvent;
 class PyLogEvent;
 class PyMetadata;
 class PyQuery;
+class PySerializer;
 }  // namespace ir::native
 
 CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyDeserializerBuffer);
@@ -126,6 +127,7 @@ CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyKeyValuePairLogEvent);
 CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyLogEvent);
 CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyMetadata);
 CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PyQuery);
+CLP_FFI_PY_MARK_AS_PYOBJECT(ir::native::PySerializer);
 CLP_FFI_PY_MARK_AS_PYOBJECT(PyBytesObject);
 CLP_FFI_PY_MARK_AS_PYOBJECT(PyDictObject);
 CLP_FFI_PY_MARK_AS_PYOBJECT(PyTypeObject);

src/clp_ffi_py/PyObjectUtils.hpp

@@ -5,6 +5,8 @@
 
 #include <memory>
 
+#include <clp_ffi_py/PyExceptionContext.hpp>
+
 namespace clp_ffi_py {
 /**
  * A specialized deleter for PyObjectPtr which decrements the pointed PyObject reference count when
@@ -47,6 +49,32 @@ using PyObjectPtr = std::unique_ptr<PyObjectType, PyObjectDeleter<PyObjectType>>
  */
 template <typename PyObjectType>
 using PyObjectStaticPtr = std::unique_ptr<PyObjectType, PyObjectTrivialDeleter<PyObjectType>>;
+
+/**
+ * A guard class for Python exceptions. In certain CPython methods, such as `tp_finalize`,
+ * the exception state must remain unchanged throughout execution. This class saves the current
+ * exception state upon initialization and restores it upon destruction, ensuring the exception
+ * status is preserved.
+ * Docs: https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_finalize
+ */
+class PyErrGuard {
+public:
+    // Constructor
+    PyErrGuard() = default;
+
+    // Destructor
+    ~PyErrGuard() { m_exception_context.restore(); }
+
+    // Delete copy/move constructor and assignment
+    PyErrGuard(PyErrGuard const&) = delete;
+    PyErrGuard(PyErrGuard&&) = delete;
+    auto operator=(PyErrGuard const&) -> PyErrGuard& = delete;
+    auto operator=(PyErrGuard&&) -> PyErrGuard& = delete;
+
+private:
+    // Variables
+    PyExceptionContext m_exception_context;
+};
 }  // namespace clp_ffi_py
 
 #endif  // CLP_FFI_PY_PY_OBJECT_UTILS_HPP

src/clp_ffi_py/Python.hpp

@@ -13,8 +13,11 @@
 #ifdef CLP_FFI_PY_ENABLE_LINTING
 // Inform IWYU of the headers that we use that are exported by Python.h
 // IWYU pragma: begin_exports
+#include <abstract.h>
 #include <bytesobject.h>
 #include <dictobject.h>
+#include <longobject.h>
+#include <memoryobject.h>
 #include <methodobject.h>
 #include <modsupport.h>
 #include <object.h>