Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[BUG] depthai-sdk Replay module - depth and color video asynrouse #881

Open
li195111 opened this issue Dec 22, 2022 · 1 comment
Open

[BUG] depthai-sdk Replay module - depth and color video asynrouse #881

li195111 opened this issue Dec 22, 2022 · 1 comment
Assignees
@Erol444
Labels
bug Something isn't working

Comments

@li195111
Copy link

Describe the bug
I used the depthai-sdk to replay the video after using it to record but I got the async video between depth and color frames as follows
image

My pencil is already down but the depth not following.

Minimal Reproducible Example

  • MRE record
import os

from depthai_sdk import OakCamera, RecordType

if __name__ == '__main__':
  record_dir = 'record_outputs/sdk'
  record_dir = os.path.abspath(record_dir)
  if not os.path.exists(record_dir):
    os.makedirs(record_dir, exist_ok=True)

  with OakCamera() as oak:
    color = oak.create_camera('color',
                              resolution='1080P',
                              fps=30,
                              encode='H264')
    left = oak.create_camera('left', resolution='480P', fps=30, encode='H264')
    right = oak.create_camera('right',
                              resolution='480P',
                              fps=30,
                              encode='H264')
    depth = oak.create_stereo(resolution='1080P',
                              fps=30,
                              left=left,
                              right=right)
    # Synchronize & save all (encoded) streams
    oak.record([
        color.out.encoded, left.out.encoded, right.out.encoded,
        # depth.out.encoded
    ], record_dir, RecordType.VIDEO)
    # Show color stream
    oak.visualize([color.out.camera, depth.out.main], scale=2 / 3, fps=True)
    # Blocking is necessary
    oak.start(blocking=True)
  • MRE replay
import os

from depthai_sdk import OakCamera

if __name__ == '__main__':
  record_dir = 'record_outputs/sdk/3-1844301041180E1300'
  record_dir = os.path.abspath(record_dir)

  with OakCamera(replay=record_dir) as oak:
    # Created CameraComponent/StereoComponent will use streams from the recording
    color = oak.create_camera('color',
                              resolution='1080P',
                              fps=30,
                              encode='H264')
    left = oak.create_camera('left', resolution='480P', fps=30, encode='H264')
    right = oak.create_camera('right',
                              resolution='480P',
                              fps=30,
                              encode='H264')
    depth = oak.create_stereo(resolution='480p',
                              fps=30,
                              left=left,
                              right=right)
    # Show color stream
    oak.visualize([color.out.camera, depth.out.main], scale=1 / 3, fps=True)
    # Blocking is necessary
    oak.start(blocking=True)

Expected behavior
The depth image must correspond to the RGB image synchronously.

Pipeline Graph

  • record
    image

  • replay
    image

Attach system log
log_system_information.json

{
    "architecture": "64bit WindowsPE",
    "machine": "AMD64",
    "platform": "Windows-10-10.0.22621-SP0",
    "processor": "Intel64 Family 6 Model 158 Stepping 10, GenuineIntel",
    "python_build": "default Nov 24 2022 14:38:14",
    "python_compiler": "MSC v.1916 64 bit (AMD64)",
    "python_implementation": "CPython",
    "python_version": "3.8.15",
    "release": "10",
    "system": "Windows",
    "version": "10.0.22621",
    "win32_ver": "10 10.0.22621 SP0 Multiprocessor Free",
    "uname": "Windows LI-DESKTOP 10 10.0.22621 AMD64 Intel64 Family 6 Model 158 Stepping 10, GenuineIntel",
    "packages": [
        "anyio==3.6.2",
        "argon2-cffi==21.3.0",
        "argon2-cffi-bindings==21.2.0",
        "arrow==1.2.3",
        "asttokens==2.2.1",
        "attrs==22.1.0",
        "av==10.0.0",
        "backcall==0.2.0",
        "beautifulsoup4==4.11.1",
        "bleach==5.0.1",
        "blobconverter==1.3.0",
        "boto3==1.26.34",
        "botocore==1.29.34",
        "certifi @ file:///C:/b/abs_85o_6fm0se/croot/certifi_1671487778835/work/certifi",
        "cffi==1.15.1",
        "charset-normalizer==2.1.1",
        "colorama==0.4.6",
        "coloredlogs==15.0.1",
        "comm==0.1.2",
        "contourpy==1.0.6",
        "cycler==0.11.0",
        "debugpy==1.6.4",
        "decorator==5.1.1",
        "defusedxml==0.7.1",
        "depthai==2.19.1.0",
        "depthai-pipeline-graph @ git+https://github.com/geaxgx/depthai_pipeline_graph.git@b5f89bb06b5c421574d991bbdb31f1ebd42118e6",
        "depthai-sdk==1.9.1",
        "distinctipy==1.2.2",
        "entrypoints==0.4",
        "executing==1.2.0",
        "fastjsonschema==2.16.2",
        "flatbuffers==22.12.6",
        "fonttools==4.38.0",
        "fqdn==1.5.1",
        "humanfriendly==10.0",
        "idna==3.4",
        "importlib-metadata==5.2.0",
        "importlib-resources==5.10.1",
        "imutils==0.5.4",
        "ipykernel==6.19.4",
        "ipython==8.7.0",
        "ipython-genutils==0.2.0",
        "ipywidgets==8.0.3",
        "isoduration==20.11.0",
        "jedi==0.18.2",
        "Jinja2==3.1.2",
        "jmespath==1.0.1",
        "jsonpointer==2.3",
        "jsonschema==4.17.3",
        "jupyter==1.0.0",
        "jupyter-console==6.4.4",
        "jupyter-events==0.5.0",
        "jupyter_client==7.4.8",
        "jupyter_core==5.1.0",
        "jupyter_server==2.0.2",
        "jupyter_server_terminals==0.4.3",
        "jupyterlab-pygments==0.2.2",
        "jupyterlab-widgets==3.0.4",
        "kiwisolver==1.4.4",
        "MarkupSafe==2.1.1",
        "marshmallow==3.17.0",
        "matplotlib==3.6.2",
        "matplotlib-inline==0.1.6",
        "mistune==2.0.4",
        "mpmath==1.2.1",
        "nbclassic==0.4.8",
        "nbclient==0.7.2",
        "nbconvert==7.2.7",
        "nbformat==5.7.1",
        "nest-asyncio==1.5.6",
        "notebook==6.5.2",
        "notebook_shim==0.2.2",
        "numpy==1.24.0",
        "onnxruntime-gpu==1.13.1",
        "opencv-contrib-python==4.6.0.66",
        "opencv-python==4.6.0.66",
        "packaging==22.0",
        "pandas==1.5.2",
        "pandocfilters==1.5.0",
        "parso==0.8.3",
        "pickleshare==0.7.5",
        "Pillow==9.3.0",
        "pip==22.3.1",
        "pkgutil_resolve_name==1.3.10",
        "platformdirs==2.6.0",
        "prometheus-client==0.15.0",
        "prompt-toolkit==3.0.36",
        "protobuf==4.21.12",
        "psutil==5.9.4",
        "pure-eval==0.2.2",
        "pycparser==2.21",
        "pydantic==1.10.2",
        "pydbow==0.1.2",
        "Pygments==2.13.0",
        "pyparsing==3.0.9",
        "pyreadline3==3.4.1",
        "pyrsistent==0.19.2",
        "PySide2==5.15.2.1",
        "python-dateutil==2.8.2",
        "python-json-logger==2.0.4",
        "pytube==12.1.2",
        "PyTurboJPEG==1.6.4",
        "pytz==2022.7",
        "pyusb==1.2.1",
        "pywin32==305",
        "pywinpty==2.0.9",
        "PyYAML==6.0",
        "pyzmq==24.0.1",
        "Qt.py==1.3.7",
        "qtconsole==5.4.0",
        "QtPy==2.3.0",
        "requests==2.28.1",
        "rfc3339-validator==0.1.4",
        "rfc3986-validator==0.1.1",
        "s3transfer==0.6.0",
        "scipy==1.9.3",
        "seaborn==0.12.1",
        "Send2Trash==1.8.0",
        "setuptools==65.5.0",
        "shiboken2==5.15.2.1",
        "six==1.16.0",
        "sniffio==1.3.0",
        "soupsieve==2.3.2.post1",
        "stack-data==0.6.2",
        "sympy==1.11.1",
        "tensorrt @ file:///G:/TensorRT-8.4.3.1/python/tensorrt-8.4.3.1-cp38-none-win_amd64.whl",
        "terminado==0.17.1",
        "tinycss2==1.2.1",
        "torch==1.13.1+cu116",
        "torchaudio==0.13.1+cu116",
        "torchvision==0.14.1+cu116",
        "tornado==6.2",
        "tqdm==4.64.1",
        "traitlets==5.8.0",
        "typing_extensions==4.4.0",
        "uri-template==1.2.0",
        "urllib3==1.26.13",
        "wcwidth==0.2.5",
        "webcolors==1.12",
        "webencodings==0.5.1",
        "websocket-client==1.4.2",
        "wheel==0.37.1",
        "widgetsnbextension==4.0.4",
        "wincertstore==0.2",
        "xmltodict==0.13.0",
        "yapf==0.32.0",
        "zipp==3.11.0"
    ],
    "usb": [
        "No USB backend found"
    ]
}
@li195111 li195111 added the bug Something isn't working label Dec 22, 2022
@Erol444 Erol444 self-assigned this Dec 23, 2022
@Erol444
Copy link
Member

Erol444 commented Mar 1, 2023 •
edited
Loading

Hi @li195111 ,
We apologize for the delay. After some debugging, I can confirm this issue, and it seems like it happens at Replay stage (not at Recording stage). I recorded a stopwatch app on a screen, then overlayed left/color frames with script below, and recording seems to be in sync:
image

We will investigate the reason behind Replay out-of-sync issue.
Thanks, Erik

import cv2

cap_color = cv2.VideoCapture('recordings/2-184430102127631200/color.mp4')
cap_mono = cv2.VideoCapture('recordings/2-184430102127631200/left.mp4')

read = True
while True:
    if read:
        ret_color, color_frame = cap_color.read()
        ret_mono, mono_frame = cap_mono.read()

    if ret_color and ret_mono:
        color_frame = cv2.cvtColor(color_frame, cv2.COLOR_BGR2RGB)
        mono_frame = cv2.cvtColor(mono_frame, cv2.COLOR_BGR2RGB)
        mono_frame = cv2.resize(mono_frame, (color_frame.shape[1], color_frame.shape[0]))
        print(color_frame.shape, mono_frame.shape)
        overlay = cv2.addWeighted(color_frame, 0.5, mono_frame, 0.5, 0)
        cv2.imshow('Overlay', cv2.pyrDown(overlay))
    else:
        break

    key = cv2.waitKey(33)
    if key == ord('q'):
        break

    if key == ord(' '):
        read = not read

@Erol444 Erol444 changed the title [BUG] depthai-sdk depth and color video asynrouse [BUG] depthai-sdk Replay module - depth and color video asynrouse Mar 1, 2023
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@Erol444 Erol444

Labels
bug Something isn't working
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
2 participants
@Erol444 @li195111