more orbbec work

ossia · Dec 13, 2024 · 9ca2f98 · 9ca2f98
1 parent 826a2b8
commit 9ca2f98
Show file tree

Hide file tree

Showing 8 changed files with 243 additions and 160 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
 
+return()
 if(NOT (WIN32 OR APPLE OR LINUX))
   return()
 endif()
@@ -14,7 +15,13 @@ set(OB_BUILD_EXAMPLES 0)
 set(OB_BUILD_TESTS 0)
 set(OB_BUILD_DOCS 0)
 set(OB_BUILD_TOOLS 0)
+set(OB_BUILD_EXAMPLES 0 CACHE "" INTERNAL)
+set(OB_BUILD_TESTS 0 CACHE "" INTERNAL)
+set(OB_BUILD_DOCS 0 CACHE "" INTERNAL)
+set(OB_BUILD_TOOLS 0 CACHE "" INTERNAL)
 block()
+  set(BUILD_SHARED_LIBS 0)
+  set(CMAKE_UNITY_BUILD 0)
   add_compile_options(-Ofast -march=native)
   add_subdirectory(3rdparty/OrbbecSDK_v2 SYSTEM)
 endblock()
@@ -23,6 +30,8 @@ score_common_setup()
 
 # Creation of the library
 add_library(score_addon_orbbec
+  Orbbec/ApplicationPlugin.hpp
+  Orbbec/ApplicationPlugin.cpp
   Orbbec/OrbbecInputStream.hpp
   Orbbec/OrbbecInputStream.cpp
   Orbbec/OrbbecInputDevice.hpp

diff --git a/Orbbec/ApplicationPlugin.cpp b/Orbbec/ApplicationPlugin.cpp
@@ -0,0 +1,31 @@
+#include "ApplicationPlugin.hpp"
+
+#include <QGuiApplication>
+#include <QThread>
+
+#include <wobjectimpl.h>
+
+W_OBJECT_IMPL(Gfx::Orbbec::ApplicationPlugin)
+namespace Gfx::Orbbec
+{
+ApplicationPlugin::ApplicationPlugin(const score::GUIApplicationContext& ctx)
+    : score::GUIApplicationPlugin{ctx}
+{
+  orbbec.setDeviceChangedCallback([this](
+                                      std::shared_ptr<ob::DeviceList> removedList,
+                                      std::shared_ptr<ob::DeviceList> deviceList) {
+    QMetaObject::invokeMethod(this, [this, removedList, deviceList] {
+      for(uint32_t index = 0, N = deviceList->getCount(); index < N; index++)
+      {
+        m_known_devices.push_back(deviceList->getDevice(index));
+        deviceAdded(m_known_devices.back());
+      }
+      for(uint32_t index = 0, N = removedList->getCount(); index < N; index++)
+      {
+        ossia::remove_erase(m_known_devices, deviceList->getDevice(index));
+        deviceRemoved(m_known_devices.back());
+      }
+    });
+  });
+}
+}
diff --git a/Orbbec/ApplicationPlugin.hpp b/Orbbec/ApplicationPlugin.hpp
@@ -3,15 +3,24 @@
 
 #include <libobsensor/ObSensor.hpp>
 
+#include <verdigris>
 namespace Gfx::Orbbec
 {
 class ApplicationPlugin
     : public QObject
     , public score::GUIApplicationPlugin
 {
+  W_OBJECT(ApplicationPlugin)
 public:
-  using GUIApplicationPlugin::GUIApplicationPlugin;
+  explicit ApplicationPlugin(const score::GUIApplicationContext& ctx);
 
   ob::Context orbbec;
+  std::vector<std::shared_ptr<ob::Device>> m_known_devices;
+
+  void deviceAdded(std::shared_ptr<ob::Device> dev) W_SIGNAL(deviceAdded, dev);
+  void deviceRemoved(std::shared_ptr<ob::Device> dev) W_SIGNAL(deviceRemoved, dev);
 };
 }
+
+Q_DECLARE_METATYPE(std::shared_ptr<ob::Device>)
+W_REGISTER_ARGTYPE(std::shared_ptr<ob::Device>)
diff --git a/Orbbec/OrbbecInputDevice.cpp b/Orbbec/OrbbecInputDevice.cpp
@@ -1,64 +1,15 @@
 #include "OrbbecInputDevice.hpp"
-#include <Orbbec/ApplicationPlugin.hpp>
-#include <Orbbec/OrbbecInputStream.hpp>
 
 #include <Gfx/GfxApplicationPlugin.hpp>
 
-#include <QLabel>
-#include <QFormLayout>
-
-#include <wobjectimpl.h>
-/*
-#include <Device/Protocol/DeviceInterface.hpp>
-#include <Device/Protocol/DeviceSettings.hpp>
-
-
-#include <ossia/gfx/texture_parameter.hpp>
-#include <ossia/network/base/device.hpp>
-#include <ossia/network/base/protocol.hpp>
-#include <QLineEdit>
-*/
-
-
-/*
-
-#include <State/Widgets/AddressFragmentLineEdit.hpp>
-
-#include <Gfx/GfxExecContext.hpp>
-#include <Gfx/Graph/VideoNode.hpp>
-#include <Video/CameraInput.hpp>
-#include <Video/FrameQueue.hpp>
-#include <Video/GStreamerCompatibility.hpp>
-#include <Video/GpuFormats.hpp>
-#include <Video/VideoInterface.hpp>
-
-#include <score/serialization/MimeVisitor.hpp>
-
-#include <ossia/detail/flicks.hpp>
-#include <ossia/detail/fmt.hpp>
-#include <magic_enum/magic_enum.hpp>
-
-#include <ossia-qt/name_utils.hpp>
-
-#include <QComboBox>
-#include <QDebug>
-#include <QElapsedTimer>
 #include <QFormLayout>
 #include <QLabel>
-#include <QMenu>
-#include <QMimeData>
-
-#include <wobjectimpl.h>
 
-#include <functional>
+#include <Orbbec/ApplicationPlugin.hpp>
+#include <Orbbec/OrbbecInputStream.hpp>
+#include <Orbbec/OrbbecProtocol.hpp>
 
-extern "C" {
-#include <libavformat/avformat.h>
-#include <libavcodec/avcodec.h>
-#include <libavformat/avformat.h>
-#include <libavutil/imgutils.h>
-}
-*/
+#include <wobjectimpl.h>
 
 namespace Gfx::Orbbec
 {
@@ -74,8 +25,8 @@ class InputDevice final : public Gfx::GfxInputDevice
   bool reconnect() override;
   ossia::net::device_base* getDevice() const override { return m_dev.get(); }
 
-  Gfx::video_texture_input_protocol* m_protocol{};
-  mutable std::unique_ptr<Gfx::video_texture_input_device> m_dev;
+  ossia::net::protocol_base* m_protocol;
+  std::unique_ptr<ossia::net::device_base> m_dev;
 };
 
 }
@@ -104,6 +55,7 @@ bool InputDevice::reconnect()
 
       {
         auto deviceList = orbbec_app.orbbec.queryDeviceList();
+        qDebug() << "Count?" << (int)deviceList->deviceCount();
         // for(int i = 0; i < (int)deviceList->deviceCount(); i++)
         // {
         //   auto dev = deviceList->getDevice(i);
@@ -142,12 +94,10 @@ bool InputDevice::reconnect()
 
         }
       }
-      auto stream = std::make_shared<InputStream>(config, device);
-
-      m_protocol = new Gfx::video_texture_input_protocol{std::move(stream), plug->exec};
-      m_dev = std::make_unique<Gfx::video_texture_input_device>(
-          std::unique_ptr<ossia::net::protocol_base>(m_protocol),
-          this->settings().name.toStdString());
+      auto proto = std::make_unique<orbbec_protocol>(config, device, set);
+      m_protocol = proto.get();
+      m_dev = std::make_unique<orbbec_device>(
+          set, plug->exec, std::move(proto), this->settings().name.toStdString());
     }
   }
   catch(std::exception& e)
@@ -197,41 +147,80 @@ class OrbbecEnumerator : public Device::DeviceEnumerator
 {
 public:
   const score::GUIApplicationContext& context;
-  const ob::Context& orbbec;
+  const ApplicationPlugin& orbbec;
+  std::string connection_type;
 
-  explicit OrbbecEnumerator(const score::GUIApplicationContext& ctx)
+  explicit OrbbecEnumerator(
+      const score::GUIApplicationContext& ctx, std::string connection_type)
       : context{ctx}
-      , orbbec{ctx.guiApplicationPlugin<Orbbec::ApplicationPlugin>().orbbec}
+      , orbbec{ctx.guiApplicationPlugin<Orbbec::ApplicationPlugin>()}
+      , connection_type{connection_type}
+  {
+    connect(
+        &orbbec, &ApplicationPlugin::deviceAdded, this,
+        [this](std::shared_ptr<ob::Device> dev) {
+      if(auto set = deviceInfo(dev))
+        deviceAdded(set->name, *set);
+    });
+    connect(
+        &orbbec, &ApplicationPlugin::deviceRemoved, this,
+        [this](std::shared_ptr<ob::Device> dev) { deviceRemoved(deviceName(dev)); });
+  }
+
+  static QString deviceName(std::shared_ptr<ob::Device> device)
+  {
+    return deviceName(device->getDeviceInfo());
+  }
+  static QString deviceName(std::shared_ptr<ob::DeviceInfo> deviceInfo)
   {
+    auto name = QString("%1 (%2)")
+                    .arg(deviceInfo->getName())
+                    .arg(deviceInfo->getSerialNumber());
+    name.remove("orbbec ", Qt::CaseInsensitive);
+    return name;
+  }
+
+  std::optional<Device::DeviceSettings>
+  deviceInfo(std::shared_ptr<ob::Device> device) const
+  {
+    auto deviceInfo = device->getDeviceInfo();
+    qDebug() << "Got device: " << deviceInfo->connectionType();
+    if(deviceInfo->connectionType() != connection_type)
+      return std::nullopt;
+
+    Device::DeviceSettings set;
+    SharedInputSettings specif;
+    set.name = deviceName(deviceInfo);
+    set.protocol = InputFactory::static_concreteKey();
+    specif.path = deviceInfo->getSerialNumber();
+    set.deviceSpecificSettings = QVariant::fromValue(specif);
+    return set;
   }
 
   void enumerate(std::function<void(const QString&, const Device::DeviceSettings&)> f)
       const override
   {
-    auto deviceList = orbbec.queryDeviceList();
-    for(uint32_t index = 0, N = deviceList->getCount(); index < N; index++)
+    qDebug() << "oy: " << this->orbbec.m_known_devices.size();
+    for(auto& dev : this->orbbec.m_known_devices)
     {
-      auto device = deviceList->getDevice(index);
-      auto deviceInfo = device->getDeviceInfo();
-
-      Device::DeviceSettings set;
-      SharedInputSettings specif;
-      set.name = QString("%1 (%2)")
-                     .arg(deviceInfo->getName())
-                     .arg(deviceInfo->getSerialNumber());
-      set.name.remove("orbbec ", Qt::CaseInsensitive);
-      set.protocol = InputFactory::static_concreteKey();
-      specif.path = deviceInfo->getSerialNumber();
-      set.deviceSpecificSettings = QVariant::fromValue(specif);
-      f(set.name, set);
+      if(auto set = deviceInfo(dev))
+        f(set->name, *set);
     }
   }
 };
 
 Device::DeviceEnumerators
 InputFactory::getEnumerators(const score::DocumentContext& ctx) const
 {
-  return {{"Cameras", new OrbbecEnumerator{ctx.app}}};
+
+  Device::DeviceEnumerators es;
+  for(const char* con :
+      {"USB", "USB1.0", "USB1.1", "USB2.0", "USB2.1", "USB3.0", "USB3.1", "USB3.2",
+       "Ethernet"})
+  {
+    es.push_back({con, new OrbbecEnumerator{ctx.app, con}});
+  }
+  return es;
 }
 
 Device::ProtocolSettingsWidget* InputFactory::makeSettingsWidget()

diff --git a/Orbbec/OrbbecInputStream.cpp b/Orbbec/OrbbecInputStream.cpp
@@ -6,23 +6,31 @@
 namespace Gfx::Orbbec
 {
 
-InputStream::InputStream(std::shared_ptr<ob::Config> conf, std::shared_ptr<ob::Device> device) noexcept
+InputStream::InputStream(
+    std::shared_ptr<ob::Config> conf, std::shared_ptr<ob::Device> device) noexcept
     : m_config{conf}
     , m_device{device}
 {
-  realTime = true;
   initH26XCodecs();
   m_pipeline = std::make_unique<ob::Pipeline>();
   m_pointCloud = std::make_shared<ob::PointCloudFilter>();
   m_pointCloud->setCreatePointFormat(OB_FORMAT_RGB_POINT);
 }
 
+InputStreamExtractor::InputStreamExtractor(
+    std::shared_ptr<InputStream> s, ::Video::FrameQueue& q) noexcept
+    : m_stream{s}
+    , m_queue{q}
+{
+  realTime = true;
+}
+
 InputStream::~InputStream() noexcept { stop(); }
 
 bool InputStream::start() noexcept
 {
   if(m_running)
-    return false;
+    return true;
 
   m_pipeline->start(m_config, [&](std::shared_ptr<ob::FrameSet> output) {
     {
@@ -46,11 +54,28 @@ void InputStream::stop() noexcept
 
   // Remove frames that were in flight
   m_rgb_frames.drain();
+  m_ir_frames.drain();
+  m_depth_frames.drain();
+  m_pcl_frames.drain();
 }
 
-AVFrame *InputStream::dequeue_frame() noexcept { return m_rgb_frames.dequeue(); }
+bool InputStreamExtractor::start() noexcept
+{
+  return m_stream->start();
+}
+void InputStreamExtractor::stop() noexcept
+{
+  m_stream->stop();
+}
+AVFrame* InputStreamExtractor::dequeue_frame() noexcept
+{
+  return m_queue.dequeue();
+}
 
-void InputStream::release_frame(AVFrame *frame) noexcept { m_rgb_frames.release(frame); }
+void InputStreamExtractor::release_frame(AVFrame* frame) noexcept
+{
+  m_queue.release(frame);
+}
 
 void InputStream::initH26XCodecs()
 {
@@ -74,9 +99,7 @@ AVFrame *InputStream::decodeFrame(uint8_t *myData, int dataSize, OBFormat fmt){
   packet.data = myData;
   packet.size = dataSize;
 
-  // Allocate an AVFrame for decoded data
   AVFrame* frame = av_frame_alloc();
-
   AVCodecContext* codecContext{};
   switch(fmt)
   {
@@ -230,9 +253,26 @@ void InputStream::on_data()
     try {
       std::shared_ptr<ob::Frame> pointCloudFrame = m_pointCloud->process(m_frameset);
       auto* points = reinterpret_cast<const OBColorPoint*>(pointCloudFrame->getData());
-      const auto N =  pointCloudFrame->getDataSize() / double(sizeof(OBColorPoint));
-
-      qDebug() << magic_enum::enum_name(pointCloudFrame->getFormat()) << N;
+      const auto bytes = pointCloudFrame->getDataSize();
+      const auto N = bytes / double(sizeof(OBColorPoint));
+
+      AVFrame* frame = av_frame_alloc();
+      frame->buf[0] = av_buffer_alloc(bytes);
+      frame->data[0] = frame->buf[0]->data;
+      memcpy(frame->data[0], points, bytes);
+
+      frame->linesize[0] = pointCloudFrame->getDataSize();
+      if(pointCloudFrame->getFormat() == OBFormat::OB_FORMAT_RGB_POINT)
+      {
+        frame->format = 0x585954a3; // 'XYZC';
+      }
+      else
+      {
+        frame->format = 0x58595400; // 'XYZ\0';
+      }
+      m_pcl_frames.enqueue(frame);
+
+      //qDebug() << magic_enum::enum_name(pointCloudFrame->getFormat()) << N;
       // pointCloudToMesh(m_frameset->depthFrame(), m_frameset->colorFrame());
     }
     catch(std::exception &e) {