Preliminarily support Redis ChunkStore in Python, remove exceptions from PyBind11 submodules

Author: hpnkv

CMakeLists.txt

@@ -79,17 +79,12 @@ add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/cppack)
 
 include(FetchContent)
 set(BUILD_TESTING_BACKUP ${BUILD_TESTING})
-set(BUILD_TESTING OFF)
 set(USE_SYSTEM_ABSEIL_BACKUP ${USE_SYSTEM_ABSEIL})
 set(USE_SYSTEM_PYBIND_BACKUP ${USE_SYSTEM_PYBIND})
+set(BUILD_TESTING OFF)
 set(USE_SYSTEM_ABSEIL ON)
 set(USE_SYSTEM_PYBIND ON)
-FetchContent_Declare(
-        pybind11_abseil
-        GIT_REPOSITORY https://github.com/pybind/pybind11_abseil.git
-        GIT_TAG master
-)
-FetchContent_MakeAvailable(pybind11_abseil)
+add_subdirectory(${EGLT_THIRD_PARTY_ROOT}/pybind11_abseil)
 set(BUILD_TESTING ${BUILD_TESTING_BACKUP})
 set(USE_SYSTEM_ABSEIL ${USE_SYSTEM_ABSEIL_BACKUP})
 set(USE_SYSTEM_PYBIND ${USE_SYSTEM_PYBIND_BACKUP})

build_pybindings_for_dev.sh

@@ -7,10 +7,12 @@ repo_root=$(pwd)
 echo "Building project..."
 cd build
 CC=clang CXX=clang++ cmake --build . --parallel "${nproc}" --target evergreen_pybind11
+CC=clang CXX=clang++ cmake --build . --parallel "${nproc}" --target pybind11_abseil
 
 echo "Moving compiled files to Python code directory..."
 cd "$repo_root"
 mv build/src/evergreen_pybind11*.so py/evergreen/
+mv build/src/pybind11_abseil_module/pybind11_abseil*.so py/evergreen/
 rm -rf install
 
 if [[ "$1" == "--only-rebuild-pybind11" ]]; then

py/evergreen/__init__.py

@@ -9,6 +9,7 @@
 from evergreen import stream as eg_stream
 from evergreen import data
 from evergreen import pydantic_helpers
+from evergreen import redis
 from evergreen import utils
 from evergreen import webrtc
 from evergreen import websockets

py/evergreen/redis.py

@@ -0,0 +1,4 @@
+from evergreen.evergreen_pybind11 import redis as redis_pybind11
+
+Redis = redis_pybind11.Redis
+ChunkStore = redis_pybind11.ChunkStore

src/CMakeLists.txt

@@ -49,6 +49,7 @@ add_subdirectory(eglt/redis)
 add_subdirectory(eglt/service)
 add_subdirectory(eglt/stores)
 add_subdirectory(eglt/util)
+add_subdirectory(pybind11_abseil_module)
 
 # ------------------------------------------------------------------
 

src/eglt/actions/actions_pybind11.cc

@@ -110,17 +110,11 @@ void BindAction(py::handle scope, std::string_view name) {
         return std::make_shared<Action>(std::move(schema), id);
       }))
       .def("run",
-           [](const std::shared_ptr<Action>& action) {
-             if (const absl::Status status = action->Run(); !status.ok()) {
-               throw std::runtime_error(status.ToString());
-             }
-           })
+           [](const std::shared_ptr<Action>& action) { return action->Run(); })
       .def(
           "call",
           [](const std::shared_ptr<Action>& action) {
-            if (const absl::Status status = action->Call(); !status.ok()) {
-              throw py::value_error(status.ToString());
-            }
+            return action->Call();
           },
           py::call_guard<py::gil_scoped_release>())
       .def(

src/eglt/data/data_pybind11.cc

@@ -18,9 +18,14 @@
 #include <string_view>
 #include <vector>
 
+#include <pybind11/pybind11.h>
+#include <pybind11_abseil/status_caster.h>
+#include <pybind11_abseil/statusor_caster.h>
+
 #include "eglt/data/eg_structs.h"
 #include "eglt/data/serialization.h"
 #include "eglt/pybind11_headers.h"
+#include "eglt/util/status_macros.h"
 #include "eglt/util/utils_pybind11.h"
 
 namespace eglt::pybindings {
@@ -187,20 +192,24 @@ void BindSerializerRegistry(py::handle scope, std::string_view name) {
       py::class_<SerializerRegistry, std::shared_ptr<SerializerRegistry>>(
           scope, std::string(name).c_str());
   registry.def(MakeSameObjectRefConstructor<SerializerRegistry>());
+
   registry.def(
       py::init([]() { return std::make_shared<SerializerRegistry>(); }));
+
   registry.def_property_readonly(
       "_mimetype_to_type", [](const std::shared_ptr<SerializerRegistry>& self) {
         return GetMimetypeToTypeDict(self.get());
       });
+
   registry.def_property_readonly(
       "_type_to_mimetype", [](const std::shared_ptr<SerializerRegistry>& self) {
         return GetTypeToMimetypeDict(self.get());
       });
+
   registry.def(
       "serialize",
       [](const std::shared_ptr<SerializerRegistry>& self, py::handle value,
-         std::string_view mimetype) -> py::bytes {
+         std::string_view mimetype) -> absl::StatusOr<py::bytes> {
         auto mimetype_str = std::string(mimetype);
         if (mimetype_str.empty()) {
           auto type_to_mimetype = GetTypeToMimetypeDict(self.get());
@@ -218,38 +227,28 @@ void BindSerializerRegistry(py::handle scope, std::string_view name) {
           py::gil_scoped_release release_gil;
           serialized = self->Serialize(value, mimetype_str);
           if (!serialized.ok()) {
-            auto cpp_any = CastPyObjectToAny(std::move(value), mimetype_str);
-            if (!cpp_any.ok()) {
-              throw py::value_error(
-                  absl::StrCat("Failed to convert value to std::any: ",
-                               cpp_any.status().message()));
-            }
-
-            serialized = self->Serialize(*std::move(cpp_any), mimetype_str);
+            ASSIGN_OR_RETURN(std::any cpp_any,
+                             CastPyObjectToAny(std::move(value), mimetype_str));
+            serialized = self->Serialize(std::move(cpp_any), mimetype_str);
           }
         }
-        if (!serialized.ok()) {
-          throw py::value_error(absl::StrCat("Failed to serialize value: ",
-                                             serialized.status().message()));
-        }
+        RETURN_IF_ERROR(serialized.status());
         return py::bytes(std::move(*serialized));
       },
       py::arg("value"), py::arg_v("mimetype", ""));
+
   registry.def(
       "deserialize",
       [](const std::shared_ptr<SerializerRegistry>& self, py::bytes data,
-         std::string_view mimetype) -> py::object {
+         std::string_view mimetype) -> absl::StatusOr<py::object> {
         absl::StatusOr<std::any> deserialized;
         {
           py::gil_scoped_release release_gil;
           deserialized = self->Deserialize(std::move(data), mimetype);
         }
-        if (!deserialized.ok()) {
-          throw py::value_error(absl::StrCat("Failed to deserialize data: ",
-                                             deserialized.status().message()));
-        }
+        RETURN_IF_ERROR(deserialized.status());
         if (std::any_cast<py::object>(&*deserialized) == nullptr) {
-          throw py::type_error(
+          return absl::InvalidArgumentError(
               absl::StrCat("Deserialized object is not a py::object, but a ",
                            deserialized->type().name(),
                            ". Cannot convert to py::object because it's not "
@@ -259,14 +258,15 @@ void BindSerializerRegistry(py::handle scope, std::string_view name) {
         return std::any_cast<py::object>(*std::move(deserialized));
       },
       py::arg("data"), py::arg_v("mimetype", ""));
+
   registry.def(
       "register_serializer",
       [](const std::shared_ptr<SerializerRegistry>& self,
          std::string_view mimetype, const py::function& serializer,
          const py::object& obj_type = py::none()) {
         if (!obj_type.is_none()) {
           if (!py::isinstance<py::type>(obj_type)) {
-            throw py::type_error(
+            return absl::InvalidArgumentError(
                 "obj_type must be a type, not an instance or other object.");
           }
           // Register the mimetype with the type.
@@ -276,6 +276,7 @@ void BindSerializerRegistry(py::handle scope, std::string_view name) {
         }
         self->RegisterSerializer(std::string(mimetype),
                                  PySerializerToCppSerializer(serializer));
+        return absl::OkStatus();
       },
       py::arg("mimetype"), py::arg("serializer"),
       py::arg_v("obj_type", py::none()), py::keep_alive<1, 3>());
@@ -286,9 +287,8 @@ void BindSerializerRegistry(py::handle scope, std::string_view name) {
          const py::object& obj_type = py::none()) {
         if (!obj_type.is_none()) {
           if (!py::isinstance<py::type>(obj_type)) {
-            throw py::type_error(
-                "obj_type must be a type, not an instance or other "
-                "object.");
+            return absl::InvalidArgumentError(
+                "obj_type must be a type, not an instance or other object.");
           }
           // Register the mimetype with the type.
           auto mimetype_str = std::string(mimetype);
@@ -298,6 +298,7 @@ void BindSerializerRegistry(py::handle scope, std::string_view name) {
         self->RegisterDeserializer(
             std::string(mimetype),
             PyDeserializerToCppDeserializer(deserializer));
+        return absl::OkStatus();
       },
       py::keep_alive<1, 3>());
   registry.def("__del__", [](const std::shared_ptr<SerializerRegistry>& self) {
@@ -326,25 +327,27 @@ py::module_ MakeDataModule(py::module_ scope, std::string_view module_name) {
   data.def(
       "to_bytes",
       [](py::handle obj, std::string_view mimetype = "",
-         SerializerRegistry* registry = nullptr) -> py::bytes {
-        return pybindings::PyToChunk(std::move(obj), mimetype, registry).data;
+         SerializerRegistry* registry = nullptr) -> absl::StatusOr<py::bytes> {
+        ASSIGN_OR_RETURN(Chunk chunk, pybindings::PyToChunk(
+                                          std::move(obj), mimetype, registry));
+        return std::move(chunk.data);
       },
       py::arg("obj"), py::arg_v("mimetype", ""),
       py::arg_v("registry", nullptr));
 
   data.def(
       "to_chunk",
       [](py::handle obj, std::string_view mimetype = "",
-         SerializerRegistry* registry = nullptr) {
-        return pybindings::PyToChunk(std::move(obj), mimetype, registry);
+         SerializerRegistry* registry = nullptr) -> absl::StatusOr<Chunk> {
+        return pybindings::PyToChunk(obj, mimetype, registry);
       },
       py::arg("obj"), py::arg_v("mimetype", ""),
       py::arg_v("registry", nullptr));
 
   data.def(
       "from_chunk",
       [](Chunk chunk, std::string_view mimetype = "",
-         const SerializerRegistry* registry = nullptr) -> py::object {
+         const SerializerRegistry* registry = nullptr) {
         return pybindings::PyFromChunk(std::move(chunk), mimetype, registry);
       },
       py::arg("chunk"), py::arg_v("mimetype", ""),

src/eglt/data/data_pybind11.h

@@ -18,8 +18,11 @@
 #include <string_view>
 
 #include <pybind11/pybind11.h>
+#include <pybind11_abseil/status_caster.h>
+#include <pybind11_abseil/statusor_caster.h>
 
 #include "eglt/data/serialization.h"
+#include "eglt/util/status_macros.h"
 
 namespace eglt {
 namespace py = ::pybind11;
@@ -238,8 +241,9 @@ inline absl::Status EgltAssignInto(pybind11::handle obj, std::string* dest) {
 
 namespace eglt::pybindings {
 
-inline Chunk PyToChunk(py::handle obj, std::string_view mimetype = "",
-                       SerializerRegistry* registry = nullptr) {
+inline absl::StatusOr<Chunk> PyToChunk(py::handle obj,
+                                       std::string_view mimetype = "",
+                                       SerializerRegistry* registry = nullptr) {
   auto mimetype_str = std::string(mimetype);
 
   if (registry == nullptr) {
@@ -260,16 +264,10 @@ inline Chunk PyToChunk(py::handle obj, std::string_view mimetype = "",
   }
 
   if (mimetype_str.empty()) {
-    absl::StatusOr<Chunk> chunk;
     {
       py::gil_scoped_release release;  // Release GIL for serialization.
-      chunk = ConvertTo<Chunk>(std::move(obj));
+      return ConvertTo<Chunk>(obj);
     }
-    if (!chunk.ok()) {
-      throw py::value_error(absl::StrCat("Failed to convert object to Chunk: ",
-                                         chunk.status().message()));
-    }
-    return *std::move(chunk);
   }
 
   absl::StatusOr<Chunk> serialized_chunk;
@@ -279,33 +277,30 @@ inline Chunk PyToChunk(py::handle obj, std::string_view mimetype = "",
     if (!serialized_chunk.ok()) {
       serialized_chunk =
           ToChunk(ConvertToOrDie<std::any>(pybindings::PySerializationArgs{
-                      .object = std::move(obj), .mimetype = mimetype_str}),
+                      .object = obj, .mimetype = mimetype_str}),
                   mimetype_str, registry);
     }
   }
 
-  if (!serialized_chunk.ok()) {
-    throw py::value_error(std::string(serialized_chunk.status().message()));
-  }
+  RETURN_IF_ERROR(serialized_chunk.status());
   return *std::move(serialized_chunk);
 }
 
-inline py::object PyFromChunk(Chunk chunk, std::string_view mimetype = "",
-                              const SerializerRegistry* registry = nullptr) {
+inline absl::StatusOr<py::object> PyFromChunk(
+    Chunk chunk, std::string_view mimetype = "",
+    const SerializerRegistry* registry = nullptr) {
   absl::StatusOr<std::any> obj;
   {
     py::gil_scoped_release release;  // Release GIL for deserialization.
     obj = FromChunk(std::move(chunk), mimetype, registry);
   }
-  if (!obj.ok()) {
-    throw py::value_error(absl::StrCat("Failed to convert Chunk to object: ",
-                                       obj.status().message()));
-  }
+  RETURN_IF_ERROR(obj.status());
 
   if (std::any_cast<py::object>(&*obj) == nullptr) {
-    throw py::type_error(absl::StrCat(
+    return absl::InvalidArgumentError(absl::StrCat(
         "Deserialized object is not a py::object, but a ", obj->type().name(),
-        ". Cannot convert to py::object because it's not implemented yet."));
+        ". Cannot convert to py::object because it's not "
+        "implemented yet."));
   }
 
   return std::any_cast<py::object>(*std::move(obj));

src/eglt/evergreen_pybind11.cc

@@ -18,6 +18,7 @@
 #include <pybind11/detail/common.h>
 #include <pybind11/pybind11.h>
 #include <pybind11/pytypes.h>
+#include <pybind11_abseil/check_status_module_imported.h>
 
 #include "eglt/actions/actions_pybind11.h"
 #include "eglt/data/data_pybind11.h"
@@ -33,6 +34,12 @@ namespace eglt {
 /// @private
 PYBIND11_MODULE(evergreen_pybind11, m) {
   absl::InstallFailureSignalHandler({});
+  if (!pybind11::google::internal::IsStatusModuleImported()) {
+    py::module_::import("evergreen.pybind11_abseil.status");
+    // importing under a custom path/name, so just in case check that the
+    // library understands our import.
+    py::google::internal::CheckStatusModuleImported();
+  }
 
   py::module_ data = pybindings::MakeDataModule(m, "data");