Add OpenVino Detector (blakeblackshear#3768)

* Initial work for adding OpenVino detector.  Not functional

* Load model and submit for inference.

Sucessfully load model and initialize OpenVino engine with either CPU or GPU as device.
Does not parse results for objects.

* Detection working with ssdlite_mobilenetv2 FP16 model

* Add OpenVIno support and model to docker image

* Add documentation for OpenVino detector configuration

* Adds support for ARM32/ARM64 and the Myriad X hardware

-  Use custom-built openvino wheel for all platforms
-  Add libusb build without udev for NCS2 support

* Add documentation around Intel CPU requirements and NCS2 setup

* Print all available output tensors

* Update documentation for config parameters

Author: NateMeyer

Dockerfile

@@ -5,6 +5,8 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 FROM debian:11 AS base
 
+FROM --platform=linux/amd64 debian:11 AS base_amd64
+
 FROM debian:11-slim AS slim-base
 
 FROM blakeblackshear/frigate-nginx:1.0.2 AS nginx
@@ -24,13 +26,62 @@ WORKDIR /rootfs/usr/local/go2rtc/bin
 RUN wget -qO go2rtc "https://github.com/AlexxIT/go2rtc/releases/download/v0.1-rc.3/go2rtc_linux_${TARGETARCH}" \
     && chmod +x go2rtc
 
+# Download and Convert OpenVino model
+FROM base_amd64 AS ov-converter
+ARG DEBIAN_FRONTEND
+
+# Install OpenVino Runtime and Dev library
+COPY requirements-ov.txt /requirements-ov.txt
+RUN apt-get -qq update \
+    && apt-get -qq install -y wget python3 python3-distutils \
+    && wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
+    && python3 get-pip.py "pip" \
+    && pip install -r /requirements-ov.txt
+
+# Get OpenVino Model
+RUN mkdir /models \
+    && cd /models && omz_downloader --name ssdlite_mobilenet_v2 \
+    && cd /models && omz_converter --name ssdlite_mobilenet_v2 --precision FP16
+
+
+# libUSB - No Udev
+FROM wget as libusb-build
+ARG TARGETARCH
+ARG DEBIAN_FRONTEND
+
+# Build libUSB without udev.  Needed for Openvino NCS2 support
+WORKDIR /opt
+RUN apt-get update && apt-get install -y unzip build-essential automake libtool
+RUN wget -q https://github.com/libusb/libusb/archive/v1.0.25.zip -O v1.0.25.zip && \
+    unzip v1.0.25.zip && cd libusb-1.0.25 && \
+    ./bootstrap.sh && \
+    ./configure --disable-udev --enable-shared && \
+    make -j $(nproc --all)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends libusb-1.0-0-dev && \
+    rm -rf /var/lib/apt/lists/*
+WORKDIR /opt/libusb-1.0.25/libusb
+RUN /bin/mkdir -p '/usr/local/lib' && \
+    /bin/bash ../libtool  --mode=install /usr/bin/install -c libusb-1.0.la '/usr/local/lib' && \
+    /bin/mkdir -p '/usr/local/include/libusb-1.0' && \
+    /usr/bin/install -c -m 644 libusb.h '/usr/local/include/libusb-1.0' && \
+    /bin/mkdir -p '/usr/local/lib/pkgconfig' && \
+    cd  /opt/libusb-1.0.25/ && \
+    /usr/bin/install -c -m 644 libusb-1.0.pc '/usr/local/lib/pkgconfig' && \
+    ldconfig
+
+
 
 FROM wget AS models
 
 # Get model and labels
 RUN wget -qO edgetpu_model.tflite https://github.com/google-coral/test_data/raw/release-frogfish/ssdlite_mobiledet_coco_qat_postprocess_edgetpu.tflite
 RUN wget -qO cpu_model.tflite https://github.com/google-coral/test_data/raw/release-frogfish/ssdlite_mobiledet_coco_qat_postprocess.tflite
 COPY labelmap.txt .
+# Copy OpenVino model
+COPY --from=ov-converter /models/public/ssdlite_mobilenet_v2/FP16 openvino-model
+RUN wget -q https://github.com/openvinotoolkit/open_model_zoo/raw/master/data/dataset_classes/coco_91cl_bkgr.txt -O openvino-model/coco_91cl_bkgr.txt
+
 
 
 FROM wget AS s6-overlay
@@ -85,6 +136,7 @@ RUN pip3 wheel --wheel-dir=/wheels -r requirements-wheels.txt
 FROM scratch AS deps-rootfs
 COPY --from=nginx /usr/local/nginx/ /usr/local/nginx/
 COPY --from=go2rtc /rootfs/ /
+COPY --from=libusb-build /usr/local/lib /usr/local/lib
 COPY --from=s6-overlay /rootfs/ /
 COPY --from=models /rootfs/ /
 COPY docker/rootfs/ /
@@ -112,6 +164,8 @@ RUN --mount=type=bind,from=wheels,source=/wheels,target=/deps/wheels \
 
 COPY --from=deps-rootfs / /
 
+RUN ldconfig
+
 EXPOSE 5000
 EXPOSE 1935
 EXPOSE 8554

docker/install_deps.sh

@@ -52,7 +52,9 @@ if [[ "${TARGETARCH}" == "amd64" ]]; then
     # Use debian testing repo only for hwaccel packages
     echo 'deb http://deb.debian.org/debian testing main non-free' >/etc/apt/sources.list.d/debian-testing.list
     apt-get -qq update
+    # intel-opencl-icd specifically for GPU support in OpenVino
     apt-get -qq install --no-install-recommends --no-install-suggests -y \
+        intel-opencl-icd \
         mesa-va-drivers libva-drm2 intel-media-va-driver-non-free i965-va-driver libmfx1 radeontop intel-gpu-tools
     rm -f /etc/apt/sources.list.d/debian-testing.list
 fi

docs/docs/configuration/detectors.md

@@ -77,3 +77,64 @@ detectors:
 ```
 
 When using CPU detectors, you can add a CPU detector per camera. Adding more detectors than the number of cameras should not improve performance.
+
+## OpenVINO
+
+The OpenVINO detector allows Frigate to run an OpenVINO IR model on Intel CPU, GPU and VPU hardware.
+
+### OpenVINO Devices
+
+The OpenVINO detector supports the Intel-supplied device plugins and can specify one or more devices in the configuration.  See OpenVINO's device naming conventions in the [Device Documentation](https://docs.openvino.ai/latest/openvino_docs_OV_UG_Working_with_devices.html) for more detail. Other supported devices could be `AUTO`, `CPU`, `GPU`, `MYRIAD`, etc.
+
+```yaml
+detectors:
+  ov_detector:
+    type: openvino
+    device: GPU
+```
+
+OpenVINO is supported on 6th Gen Intel platforms (Skylake) and newer.  A supported Intel platform is required to use the GPU device with OpenVINO.  The `MYRIAD` device may be run on any platform, including Arm devices.  For detailed system requirements, see [OpenVINO System Requirements](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/system-requirements.html)
+
+#### Intel NCS2 VPU and Myriad X Setup
+
+Intel produces a neural net inference accelleration chip called Myriad X.  This chip was sold in their Neural Compute Stick 2 (NCS2) which has been discontinued.  If intending to use the MYRIAD device for accelleration, additional setup is required to pass through the USB device.  The host needs a udev rule installed to handle the NCS2 device.  
+
+```bash
+sudo usermod -a -G users "$(whoami)"
+cat <<EOF > 97-myriad-usbboot.rules
+SUBSYSTEM=="usb", ATTRS{idProduct}=="2485", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1"
+SUBSYSTEM=="usb", ATTRS{idProduct}=="f63b", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1"
+EOF
+sudo cp 97-myriad-usbboot.rules /etc/udev/rules.d/
+sudo udevadm control --reload-rules
+sudo udevadm trigger
+```
+
+Additionally, the Frigate docker container needs to run with the following configuration:
+
+```bash
+--device-cgroup-rule='c 189:\* rmw' -v /dev/bus/usb:/dev/bus/usb
+```
+or in your compose file:
+
+```yml
+device_cgroup_rules:
+  - 'c 189:* rmw'
+volumes:
+  - /dev/bus/usb:/dev/bus/usb
+```
+
+### OpenVINO Models
+
+The included model for an OpenVINO detector comes from Intel's Open Model Zoo [SSDLite MobileNet V2](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/ssdlite_mobilenet_v2) and is converted to an FP16 precision IR model.  Use the model configuration shown below when using the OpenVINO detector.
+
+```yaml
+model:
+  path: /openvino-model/ssdlite_mobilenet_v2.xml
+  width: 300
+  height: 300
+  input_tensor: nhwc
+  input_pixel_format: bgr
+  labelmap_path: /openvino-model/coco_91cl_bkgr.txt
+
+```

docs/docs/configuration/index.md

@@ -76,9 +76,9 @@ detectors:
   # Required: name of the detector
   coral:
     # Required: type of the detector
-    # Valid values are 'edgetpu' (requires device property below) and 'cpu'.
+    # Valid values are 'edgetpu' (requires device property below)  `openvino` (see Detectors documentation), and 'cpu'.
     type: edgetpu
-    # Optional: device name as defined here: https://coral.ai/docs/edgetpu/multiple-edgetpu/#using-the-tensorflow-lite-python-api
+    # Optional: Edgetpu or OpenVino device name
     device: usb
     # Optional: num_threads value passed to the tflite.Interpreter (default: shown below)
     # This value is only used for CPU types
@@ -104,7 +104,7 @@ model:
   input_pixel_format: rgb
   # Optional: Object detection model input tensor format
   # Valid values are nhwc or nchw (default: shown below)
-  input_tensor: "nhwc"
+  input_tensor: nhwc
   # Optional: Label name modifications. These are merged into the standard labelmap.
   labelmap:
     2: vehicle

frigate/config.py

@@ -54,6 +54,7 @@ class Config:
 
 class DetectorTypeEnum(str, Enum):
     edgetpu = "edgetpu"
+    openvino = "openvino"
     cpu = "cpu"
 
 

frigate/detectors/openvino.py

@@ -0,0 +1,54 @@
+import logging
+import numpy as np
+import openvino.runtime as ov
+
+from frigate.detectors.detection_api import DetectionApi
+
+
+logger = logging.getLogger(__name__)
+
+
+class OvDetector(DetectionApi):
+    def __init__(self, det_device=None, model_config=None, num_threads=1):
+        self.ov_core = ov.Core()
+        self.ov_model = self.ov_core.read_model(model_config.path)
+
+        self.interpreter = self.ov_core.compile_model(
+            model=self.ov_model, device_name=det_device
+        )
+        logger.info(f"Model Input Shape: {self.interpreter.input(0).shape}")
+        self.output_indexes = 0
+        while True:
+            try:
+                tensor_shape = self.interpreter.output(self.output_indexes).shape
+                logger.info(f"Model Output-{self.output_indexes} Shape: {tensor_shape}")
+                self.output_indexes += 1
+            except:
+                logger.info(f"Model has {self.output_indexes} Output Tensors")
+                break
+
+    def detect_raw(self, tensor_input):
+
+        infer_request = self.interpreter.create_infer_request()
+        infer_request.infer([tensor_input])
+
+        results = infer_request.get_output_tensor()
+
+        detections = np.zeros((20, 6), np.float32)
+        i = 0
+        for object_detected in results.data[0, 0, :]:
+            if object_detected[0] != -1:
+                logger.debug(object_detected)
+            if object_detected[2] < 0.1 or i == 20:
+                break
+            detections[i] = [
+                object_detected[1],  # Label ID
+                float(object_detected[2]),  # Confidence
+                object_detected[4],  # y_min
+                object_detected[3],  # x_min
+                object_detected[6],  # y_max
+                object_detected[5],  # x_max
+            ]
+            i += 1
+
+        return detections

frigate/object_detection.py

@@ -12,6 +12,7 @@
 
 from frigate.config import DetectorTypeEnum, InputTensorEnum
 from frigate.detectors.edgetpu_tfl import EdgeTpuTfl
+from frigate.detectors.openvino import OvDetector
 from frigate.detectors.cpu_tfl import CpuTfl
 
 from frigate.util import EventsPerSecond, SharedMemoryFrameManager, listen, load_labels
@@ -57,6 +58,10 @@ def __init__(
             self.detect_api = EdgeTpuTfl(
                 det_device=det_device, model_config=model_config
             )
+        elif det_type == DetectorTypeEnum.openvino:
+            self.detect_api = OvDetector(
+                det_device=det_device, model_config=model_config
+            )
         else:
             logger.warning(
                 "CPU detectors are not recommended and should only be used for testing or for trial purposes."

requirements-ov.txt

@@ -0,0 +1,3 @@
+numpy == 1.19.*
+openvino == 2022.*
+openvino-dev[tensorflow2] == 2022.*

requirements-wheels.txt

@@ -18,3 +18,7 @@ scipy == 1.8.*
 setproctitle == 1.2.*
 ws4py == 0.5.*
 zeroconf == 0.39.4
+# Openvino Library - Custom built with MYRIAD support
+openvino @ https://github.com/NateMeyer/openvino-wheels/releases/download/multi-arch_2022.2.0/openvino-2022.2.0-000-cp39-cp39-manylinux_2_31_x86_64.whl; platform_machine == 'x86_64'
+openvino @ https://github.com/NateMeyer/openvino-wheels/releases/download/multi-arch_2022.2.0/openvino-2022.2.0-000-cp39-cp39-linux_aarch64.whl; platform_machine == 'aarch64'
+openvino @ https://github.com/NateMeyer/openvino-wheels/releases/download/multi-arch_2022.2.0/openvino-2022.2.0-000-cp39-cp39-linux_armv7l.whl; platform_machine == 'armv7l'