Add orientation transforms to stream

homeassistant/components/camera/__init__.py

@@ -65,6 +65,8 @@
     DATA_CAMERA_PREFS,
     DATA_RTSP_TO_WEB_RTC,
     DOMAIN,
+    PREF_ORIENTATION,
+    PREF_PRELOAD_STREAM,
     SERVICE_RECORD,
     STREAM_TYPE_HLS,
     STREAM_TYPE_WEB_RTC,
@@ -874,7 +876,8 @@ async def websocket_get_prefs(
     {
         vol.Required("type"): "camera/update_prefs",
         vol.Required("entity_id"): cv.entity_id,
-        vol.Optional("preload_stream"): bool,
+        vol.Optional(PREF_PRELOAD_STREAM): bool,
+        vol.Optional(PREF_ORIENTATION): vol.All(int, vol.Range(min=1, max=8)),
     }
 )
 @websocket_api.async_response
@@ -888,9 +891,10 @@ async def websocket_update_prefs(
     changes.pop("id")
     changes.pop("type")
     entity_id = changes.pop("entity_id")
-    await prefs.async_update(entity_id, **changes)
-
-    connection.send_result(msg["id"], prefs.get(entity_id).as_dict())
+    if isinstance(entity_prefs := await prefs.async_update(entity_id, **changes), dict):
+        connection.send_result(msg["id"], entity_prefs)
+    else:
+        connection.send_error(msg["id"], "update_failed", entity_prefs)
 
 
 async def async_handle_snapshot_service(
@@ -959,6 +963,7 @@ async def _async_stream_endpoint_url(
     # Update keepalive setting which manages idle shutdown
     camera_prefs = hass.data[DATA_CAMERA_PREFS].get(camera.entity_id)
     stream.keepalive = camera_prefs.preload_stream
+    stream.orientation = camera_prefs.orientation
 
     stream.add_provider(fmt)
     await stream.start()

homeassistant/components/camera/const.py

@@ -9,6 +9,7 @@
 DATA_RTSP_TO_WEB_RTC: Final = "rtsp_to_web_rtc"
 
 PREF_PRELOAD_STREAM: Final = "preload_stream"
+PREF_ORIENTATION: Final = "orientation"
 
 SERVICE_RECORD: Final = "record"
 

homeassistant/components/camera/prefs.py

@@ -1,13 +1,14 @@
 """Preference management for camera component."""
 from __future__ import annotations
 
-from typing import Final
+from typing import Final, Union, cast
 
 from homeassistant.core import HomeAssistant
+from homeassistant.helpers import entity_registry as er
 from homeassistant.helpers.storage import Store
 from homeassistant.helpers.typing import UNDEFINED, UndefinedType
 
-from .const import DOMAIN, PREF_PRELOAD_STREAM
+from .const import DOMAIN, PREF_ORIENTATION, PREF_PRELOAD_STREAM
 
 STORAGE_KEY: Final = DOMAIN
 STORAGE_VERSION: Final = 1
@@ -16,18 +17,23 @@
 class CameraEntityPreferences:
     """Handle preferences for camera entity."""
 
-    def __init__(self, prefs: dict[str, bool]) -> None:
+    def __init__(self, prefs: dict[str, bool | int]) -> None:
         """Initialize prefs."""
         self._prefs = prefs
 
-    def as_dict(self) -> dict[str, bool]:
+    def as_dict(self) -> dict[str, bool | int]:
         """Return dictionary version."""
         return self._prefs
 
     @property
     def preload_stream(self) -> bool:
         """Return if stream is loaded on hass start."""
-        return self._prefs.get(PREF_PRELOAD_STREAM, False)
+        return cast(bool, self._prefs.get(PREF_PRELOAD_STREAM, False))
+
+    @property
+    def orientation(self) -> int:
+        """Return the current stream orientation settings."""
+        return self._prefs.get(PREF_ORIENTATION, 1)
 
 
 class CameraPreferences:
@@ -36,10 +42,13 @@ class CameraPreferences:
     def __init__(self, hass: HomeAssistant) -> None:
         """Initialize camera prefs."""
         self._hass = hass
-        self._store = Store[dict[str, dict[str, bool]]](
+        # The orientation prefs are stored in in the entity registry options
+        # The preload_stream prefs are stored in this Store
+        self._store = Store[dict[str, dict[str, Union[bool, int]]]](
             hass, STORAGE_VERSION, STORAGE_KEY
         )
-        self._prefs: dict[str, dict[str, bool]] | None = None
+        # Local copy of the preload_stream prefs
+        self._prefs: dict[str, dict[str, bool | int]] | None = None
 
     async def async_initialize(self) -> None:
         """Finish initializing the preferences."""
@@ -53,22 +62,34 @@ async def async_update(
         entity_id: str,
         *,
         preload_stream: bool | UndefinedType = UNDEFINED,
+        orientation: int | UndefinedType = UNDEFINED,
         stream_options: dict[str, str] | UndefinedType = UNDEFINED,
-    ) -> None:
-        """Update camera preferences."""
-        # Prefs already initialized.
-        assert self._prefs is not None
-        if not self._prefs.get(entity_id):
-            self._prefs[entity_id] = {}
-
-        for key, value in ((PREF_PRELOAD_STREAM, preload_stream),):
-            if value is not UNDEFINED:
-                self._prefs[entity_id][key] = value
-
-        await self._store.async_save(self._prefs)
+    ) -> dict[str, bool | int] | str:
+        """Update camera preferences.
+
+        Returns a dict with the preferences on success or a string on error.
+        """
+        if preload_stream is not UNDEFINED:
+            # Prefs already initialized.
+            assert self._prefs is not None
+            if not self._prefs.get(entity_id):
+                self._prefs[entity_id] = {}
+            self._prefs[entity_id][PREF_PRELOAD_STREAM] = preload_stream
+            await self._store.async_save(self._prefs)
+
+        if orientation is not UNDEFINED:
+            if (registry := er.async_get(self._hass)).async_get(entity_id):
+                registry.async_update_entity_options(
+                    entity_id, DOMAIN, {PREF_ORIENTATION: orientation}
+                )
+            else:
+                return "Orientation is only supported on entities set up through config flows"
+        return self.get(entity_id).as_dict()
 
     def get(self, entity_id: str) -> CameraEntityPreferences:
         """Get preferences for an entity."""
         # Prefs are already initialized.
         assert self._prefs is not None
-        return CameraEntityPreferences(self._prefs.get(entity_id, {}))
+        reg_entry = er.async_get(self._hass).async_get(entity_id)
+        er_prefs = reg_entry.options.get(DOMAIN, {}) if reg_entry else {}
+        return CameraEntityPreferences(self._prefs.get(entity_id, {}) | er_prefs)

homeassistant/components/stream/__init__.py

@@ -229,6 +229,7 @@ async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
             part_target_duration=conf[CONF_PART_DURATION],
             hls_advance_part_limit=max(int(3 / conf[CONF_PART_DURATION]), 3),
             hls_part_timeout=2 * conf[CONF_PART_DURATION],
+            orientation=1,
         )
     else:
         hass.data[DOMAIN][ATTR_SETTINGS] = STREAM_SETTINGS_NON_LL_HLS
@@ -280,7 +281,7 @@ def __init__(
         self._thread_quit = threading.Event()
         self._outputs: dict[str, StreamOutput] = {}
         self._fast_restart_once = False
-        self._keyframe_converter = KeyFrameConverter(hass)
+        self._keyframe_converter = KeyFrameConverter(hass, stream_settings)
         self._available: bool = True
         self._update_callback: Callable[[], None] | None = None
         self._logger = (
@@ -290,6 +291,16 @@ def __init__(
         )
         self._diagnostics = Diagnostics()
 
+    @property
+    def orientation(self) -> int:
+        """Return the current orientation setting."""
+        return self._stream_settings.orientation
+
+    @orientation.setter
+    def orientation(self, value: int) -> None:
+        """Set the stream orientation setting."""
+        self._stream_settings.orientation = value
+
     def endpoint_url(self, fmt: str) -> str:
         """Start the stream and returns a url for the output format."""
         if fmt not in self._outputs:
@@ -401,6 +412,7 @@ def _run_worker(self) -> None:
             start_time = time.time()
             self.hass.add_job(self._async_update_state, True)
             self._diagnostics.set_value("keepalive", self.keepalive)
+            self._diagnostics.set_value("orientation", self.orientation)
             self._diagnostics.increment("start_worker")
             try:
                 stream_worker(

homeassistant/components/stream/core.py

@@ -11,6 +11,7 @@
 from aiohttp import web
 import async_timeout
 import attr
+import numpy as np
 
 from homeassistant.components.http.view import HomeAssistantView
 from homeassistant.core import CALLBACK_TYPE, HomeAssistant, callback
@@ -43,6 +44,7 @@ class StreamSettings:
     part_target_duration: float = attr.ib()
     hls_advance_part_limit: int = attr.ib()
     hls_part_timeout: float = attr.ib()
+    orientation: int = attr.ib()
 
 
 STREAM_SETTINGS_NON_LL_HLS = StreamSettings(
@@ -51,6 +53,7 @@ class StreamSettings:
     part_target_duration=TARGET_SEGMENT_DURATION_NON_LL_HLS,
     hls_advance_part_limit=3,
     hls_part_timeout=TARGET_SEGMENT_DURATION_NON_LL_HLS,
+    orientation=1,
 )
 
 
@@ -397,7 +400,7 @@ class KeyFrameConverter:
     If unsuccessful, get_image will return the previous image
     """
 
-    def __init__(self, hass: HomeAssistant) -> None:
+    def __init__(self, hass: HomeAssistant, stream_settings: StreamSettings) -> None:
         """Initialize."""
 
         # Keep import here so that we can import stream integration without installing reqs
@@ -410,6 +413,7 @@ def __init__(self, hass: HomeAssistant) -> None:
         self._turbojpeg = TurboJPEGSingleton.instance()
         self._lock = asyncio.Lock()
         self._codec_context: CodecContext | None = None
+        self._stream_settings = stream_settings
 
     def create_codec_context(self, codec_context: CodecContext) -> None:
         """
@@ -430,6 +434,28 @@ def create_codec_context(self, codec_context: CodecContext) -> None:
         self._codec_context.skip_frame = "NONKEY"
         self._codec_context.thread_type = "NONE"
 
+    @staticmethod
+    def transform_image(image: np.ndarray, orientation: int) -> np.ndarray:
+        """Transform image to a given orientation.
+
+        Adapted from https://github.com/lilohuang/PyTurboJPEG.
+        """
+        if orientation == 1:
+            return image
+        if orientation == 2:
+            return np.fliplr(image).copy()
+        if orientation == 3:
+            return np.rot90(image, 2).copy()
+        if orientation == 4:
+            return np.flipud(image).copy()
+        if orientation == 5:
+            return np.rot90(np.flipud(image), -1).copy()
+        if orientation == 6:
+            return np.rot90(image).copy()
+        if orientation == 7:
+            return np.rot90(np.flipud(image)).copy()
+        return np.rot90(image, -1).copy()
+
     def _generate_image(self, width: int | None, height: int | None) -> None:
         """
         Generate the keyframe image.
@@ -462,8 +488,13 @@ def _generate_image(self, width: int | None, height: int | None) -> None:
         if frames:
             frame = frames[0]
             if width and height:
-                frame = frame.reformat(width=width, height=height)
-            bgr_array = frame.to_ndarray(format="bgr24")
+                if self._stream_settings.orientation >= 5:
+                    frame = frame.reformat(width=height, height=width)
+                else:
+                    frame = frame.reformat(width=width, height=height)
+            bgr_array = self.transform_image(
+                frame.to_ndarray(format="bgr24"), self._stream_settings.orientation
+            )
             self._image = bytes(self._turbojpeg.encode(bgr_array))
 
     async def async_get_image(

homeassistant/components/stream/fmp4utils.py

@@ -5,7 +5,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from io import BytesIO
+    from io import BufferedIOBase
 
 
 def find_box(
@@ -141,9 +141,55 @@ def get_codec_string(mp4_bytes: bytes) -> str:
     return ",".join(codecs)
 
 
-def read_init(bytes_io: BytesIO) -> bytes:
+def read_init(bytes_io: BufferedIOBase) -> bytes:
     """Read the init from a mp4 file."""
     bytes_io.seek(24)
     moov_len = int.from_bytes(bytes_io.read(4), byteorder="big")
     bytes_io.seek(0)
     return bytes_io.read(24 + moov_len)
+
+
+ZERO32 = b"\x00\x00\x00\x00"
+ONE32 = b"\x00\x01\x00\x00"
+NEGONE32 = b"\xFF\xFF\x00\x00"
+XYW_ROW = ZERO32 + ZERO32 + b"\x40\x00\x00\x00"
+ROTATE_RIGHT = (ZERO32 + ONE32 + ZERO32) + (NEGONE32 + ZERO32 + ZERO32)
+ROTATE_LEFT = (ZERO32 + NEGONE32 + ZERO32) + (ONE32 + ZERO32 + ZERO32)
+ROTATE_180 = (NEGONE32 + ZERO32 + ZERO32) + (ZERO32 + NEGONE32 + ZERO32)
+MIRROR = (NEGONE32 + ZERO32 + ZERO32) + (ZERO32 + ONE32 + ZERO32)
+FLIP = (ONE32 + ZERO32 + ZERO32) + (ZERO32 + NEGONE32 + ZERO32)
+# The two below do not seem to get applied properly
+ROTATE_LEFT_FLIP = (ZERO32 + NEGONE32 + ZERO32) + (NEGONE32 + ZERO32 + ZERO32)
+ROTATE_RIGHT_FLIP = (ZERO32 + ONE32 + ZERO32) + (ONE32 + ZERO32 + ZERO32)
+
+TRANSFORM_MATRIX_TOP = [
+    # The first two entries are just to align the indices with the EXIF orientation tags
+    b"",
+    b"",
+    MIRROR,
+    ROTATE_180,
+    FLIP,
+    ROTATE_LEFT_FLIP,
+    ROTATE_LEFT,
+    ROTATE_RIGHT_FLIP,
+    ROTATE_RIGHT,
+]
+
+
+def transform_init(init: bytes, orientation: int) -> bytes:
+    """Change the transformation matrix in the header."""
+    if orientation == 1:
+        return init
+    # Find moov
+    moov_location = next(find_box(init, b"moov"))
+    mvhd_location = next(find_box(init, b"trak", moov_location))
+    tkhd_location = next(find_box(init, b"tkhd", mvhd_location))
+    tkhd_length = int.from_bytes(
+        init[tkhd_location : tkhd_location + 4], byteorder="big"
+    )
+    return (
+        init[: tkhd_location + tkhd_length - 44]
+        + TRANSFORM_MATRIX_TOP[orientation]
+        + XYW_ROW
+        + init[tkhd_location + tkhd_length - 8 :]
+    )

homeassistant/components/stream/hls.py

@@ -24,7 +24,7 @@
     StreamSettings,
     StreamView,
 )
-from .fmp4utils import get_codec_string
+from .fmp4utils import get_codec_string, transform_init
 
 if TYPE_CHECKING:
     from . import Stream
@@ -339,7 +339,7 @@ async def handle(
         if not (segments := track.get_segments()) or not (body := segments[0].init):
             return web.HTTPNotFound()
         return web.Response(
-            body=body,
+            body=transform_init(body, stream.orientation),
             headers={"Content-Type": "video/mp4"},
         )