feat(dataviewer): enhance HDF5 video handling and nested dataset support

.gitignore

@@ -465,4 +465,4 @@ AGENTS.md
 # Terraform Environments
 envs/
 
-src/dataviewer/datasets/
+datasets/

src/dataviewer/backend/.env.azure.example

@@ -0,0 +1,14 @@
+# Local development against Azure Blob Storage (osmorbt3-dev-001 environment)
+#
+# Usage:
+#   1. Copy to .env:  cp .env.azure.example .env
+#   2. Log in:        az login  (or source deploy/000-prerequisites/az-sub-init.sh)
+#   3. Start:         npm run dev:backend  (from src/dataviewer/)
+#
+# Your Azure AD user must have Storage Blob Data Reader (or Contributor) on the account.
+# Do NOT set AZURE_CLIENT_ID locally — it overrides CLI credentials with managed identity.
+
+HMI_STORAGE_BACKEND=azure
+AZURE_STORAGE_ACCOUNT_NAME=<insert-storage-account-name>
+AZURE_STORAGE_DATASET_CONTAINER=datasets
+AZURE_STORAGE_ANNOTATION_CONTAINER=annotations

src/dataviewer/backend/.env.example

@@ -22,8 +22,14 @@ HMI_DATA_PATH=../../../datasets
 # ─────────────────────────────────────────────────────────────────────────────
 # Azure Blob Storage (required when HMI_STORAGE_BACKEND=azure)
 # ─────────────────────────────────────────────────────────────────────────────
+# Azure Blob Storage (required when HMI_STORAGE_BACKEND=azure)
+# ─────────────────────────────────────────────────────────────────────────────
+# Prerequisite: run `az login` or `source deploy/000-prerequisites/az-sub-init.sh`
+# Your Azure AD user needs Storage Blob Data Reader (or Contributor) on the account.
+# Do NOT set AZURE_CLIENT_ID locally — it overrides CLI credentials.
+# See .env.azure.example for a ready-to-copy config.
 
-# Azure Storage account name (e.g. mystorageaccount)
+# Azure Storage account name
 # AZURE_STORAGE_ACCOUNT_NAME=
 
 # Blob container that holds dataset files (videos, parquet, HDF5).

src/dataviewer/backend/Dockerfile

@@ -4,6 +4,10 @@ FROM python:3.11-slim AS base
 
 WORKDIR /app
 
+# Install system dependencies (ffmpeg for HDF5 video generation)
+RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
+
 # Install uv
 RUN pip install --no-cache-dir uv
 

src/dataviewer/backend/src/api/main.py

@@ -2,6 +2,8 @@
 
 import logging
 import os
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
 from pathlib import Path
 
 from dotenv import load_dotenv
@@ -20,6 +22,10 @@
     format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
 )
 
+# Suppress verbose Azure SDK HTTP request logging
+logging.getLogger("azure").setLevel(logging.WARNING)
+logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(logging.WARNING)
+
 # Load .env before any config or service singletons are initialized so that
 # all env vars are available to get_app_config() on first access.
 env_path = Path(__file__).parent.parent.parent / ".env"
@@ -31,10 +37,28 @@
 
 _config = load_config()
 
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(_app: FastAPI) -> AsyncGenerator[None]:
+    """Clean up blob sync temp directories on shutdown."""
+    yield
+    from .services.dataset_service import get_dataset_service
+
+    try:
+        service = get_dataset_service()
+        service.cleanup_temp_dirs()
+        logger.info("Cleaned up blob sync temp directories")
+    except Exception:
+        pass
+
+
 app = FastAPI(
     title="LeRobot Annotation API",
     description="API for episode annotation in robot demonstration datasets",
     version="0.1.0",
+    lifespan=lifespan,
     openapi_tags=[
         {"name": "auth", "description": "Authentication utilities"},
     ],

src/dataviewer/backend/src/api/routers/datasets.py

@@ -5,6 +5,7 @@
 and accessing episode information with HDF5 and LeRobot parquet support.
 """
 
+import asyncio
 from pathlib import Path
 
 from fastapi import APIRouter, Depends, HTTPException, Query, Request
@@ -13,7 +14,7 @@
 
 from ..models.datasources import DatasetInfo, EpisodeData, EpisodeMeta, TrajectoryPoint
 from ..services.dataset_service import DatasetService, get_dataset_service
-from ..validation import validate_path_containment, validated_camera_name, validated_dataset_id
+from ..validation import validated_camera_name, validated_dataset_id
 
 router = APIRouter()
 
@@ -248,10 +249,12 @@ async def get_episode_video(
 
     Note: camera parameter can include dots (e.g., 'observation.images.color')
     """
-    video_path = service.get_video_file_path(dataset_id, episode_idx, camera)
+    video_path = await asyncio.to_thread(service.get_video_file_path, dataset_id, episode_idx, camera)
 
     if video_path is not None:
-        video_file = validate_path_containment(Path(video_path), Path(service.base_path))
+        if not service.is_safe_video_path(video_path):
+            raise HTTPException(status_code=400, detail="Path traversal detected: resolved path escapes base directory")
+        video_file = Path(video_path)
         if not video_file.exists():
             raise HTTPException(
                 status_code=404,
@@ -363,8 +366,7 @@ async def warm_cache(
     Preload the first N episodes into the LRU cache.
 
     Designed to be called on dataset selection so the initial episode
-    loads are instant. Runs synchronously to give the caller confidence
-    that warm-up is complete before navigating.
+    loads are instant.
     """
     dataset = await service.get_dataset(dataset_id)
     if dataset is None:

src/dataviewer/backend/src/api/routers/labels.py

@@ -4,10 +4,13 @@
 and managing the set of available label options per dataset.
 """
 
+from __future__ import annotations
+
 import json
 import logging
 import os
 from pathlib import Path
+from typing import TYPE_CHECKING, Protocol
 
 import aiofiles
 import aiofiles.os
@@ -16,8 +19,12 @@
 
 from ..csrf import require_csrf_token
 from ..services.dataset_service import DatasetService, get_dataset_service
+from ..storage.paths import dataset_id_to_blob_prefix
 from ..validation import validate_path_containment, validated_dataset_id
 
+if TYPE_CHECKING:
+    from ..storage.blob_dataset import BlobDatasetProvider
+
 logger = logging.getLogger(__name__)
 
 router = APIRouter()
@@ -55,47 +62,140 @@
     return label.strip().upper()
 
 
+# ============================================================================
+# Label Storage Backends
+# ============================================================================
+
+
+class LabelStorage(Protocol):
+    """Protocol for label persistence backends."""
+
+    async def load(self, dataset_id: str) -> DatasetLabelsFile: ...
+    async def save(self, dataset_id: str, labels_file: DatasetLabelsFile) -> None: ...
+
+
+class LocalLabelStorage:
+    """Filesystem-backed label storage."""
+
+    def __init__(self, base_path: str) -> None:
+        self._base_path = base_path
+
+    def _path(self, dataset_id: str) -> Path:
+        return _labels_path_for_base(dataset_id, self._base_path)
+
+    async def load(self, dataset_id: str) -> DatasetLabelsFile:
+        path = self._path(dataset_id)
+        safe_base = os.path.realpath(self._base_path)
+        resolved = os.path.realpath(str(path))
+        if not resolved.startswith(safe_base + os.sep):
+            raise HTTPException(status_code=400, detail="Path traversal detected")
+        path = Path(resolved)
+        if not await aiofiles.os.path.exists(path):
+            return DatasetLabelsFile(dataset_id=dataset_id)
+        async with aiofiles.open(path, encoding="utf-8") as f:
+            data = json.loads(await f.read())
+            return DatasetLabelsFile.model_validate(data)
+
+    async def save(self, dataset_id: str, labels_file: DatasetLabelsFile) -> None:
+        path = self._path(dataset_id)
+        safe_base = os.path.realpath(self._base_path)
+        resolved = os.path.realpath(str(path))
+        if not resolved.startswith(safe_base + os.sep):
+            raise HTTPException(status_code=400, detail="Path traversal detected")
+        path = Path(resolved)
+        await aiofiles.os.makedirs(path.parent, exist_ok=True)
+        content = json.dumps(labels_file.model_dump(), indent=2)
+        async with aiofiles.open(path, "w", encoding="utf-8") as f:
+            await f.write(content)
+
+
+class BlobLabelStorage:
+    """Azure Blob Storage-backed label storage. Stores in datasets container."""
+
+    def __init__(self, blob_provider: BlobDatasetProvider) -> None:
+        self._provider = blob_provider
+
+    def _blob_path(self, dataset_id: str) -> str:
+        return f"{dataset_id_to_blob_prefix(dataset_id)}/meta/episode_labels.json"
+
+    async def load(self, dataset_id: str) -> DatasetLabelsFile:
+        data = await self._provider._read_blob_bytes(self._blob_path(dataset_id))
+        if data is None:
+            return DatasetLabelsFile(dataset_id=dataset_id)
+        try:
+            return DatasetLabelsFile.model_validate(json.loads(data.decode("utf-8")))
+        except (json.JSONDecodeError, Exception):
+            logger.warning("Invalid labels blob for %s, returning defaults", dataset_id)
+            return DatasetLabelsFile(dataset_id=dataset_id)
+
+    async def save(self, dataset_id: str, labels_file: DatasetLabelsFile) -> None:
+        try:
+            from azure.storage.blob import ContentSettings
+
+            client = await self._provider._get_client()
+            container = client.get_container_client(self._provider.container_name)
+            blob_client = container.get_blob_client(self._blob_path(dataset_id))
+            content = json.dumps(labels_file.model_dump(), indent=2).encode("utf-8")
+            await blob_client.upload_blob(
+                content,
+                overwrite=True,
+                content_settings=ContentSettings(content_type="application/json"),
+            )
+        except Exception as e:
+            logger.error("Failed to save labels blob for %s: %s", dataset_id, e)
+            raise HTTPException(status_code=500, detail="Failed to save labels") from e
+
+
+def _create_label_storage(
+    storage_backend: str = "local",
+    blob_provider: BlobDatasetProvider | None = None,
+) -> LabelStorage:
+    """Create label storage backend based on config."""
+    if storage_backend == "azure" and blob_provider is not None:
+        return BlobLabelStorage(blob_provider)
+    return LocalLabelStorage(os.environ.get("HMI_DATA_PATH", "./data"))
+
+
+_label_storage: LabelStorage | None = None
+
+
+def _get_label_storage() -> LabelStorage:
+    """Get or create the global label storage singleton."""
+    global _label_storage
+    if _label_storage is None:
+        from ..config import get_app_config
+
+        config = get_app_config()
+        blob_provider = None
+        if config.storage_backend == "azure":
+            from ..config import create_blob_dataset_provider
+
+            blob_provider = create_blob_dataset_provider(config)
+        _label_storage = _create_label_storage(config.storage_backend, blob_provider)
+    return _label_storage
+
+
 def _get_base_path() -> str:
     return os.environ.get("HMI_DATA_PATH", "./data")
 
 
+def _labels_path_for_base(dataset_id: str, base_path: str) -> Path:
+    """Build labels path, resolving -- to nested directories."""
+    base = Path(base_path)
+    parts = dataset_id.split("--") if "--" in dataset_id else [dataset_id]
+    return validate_path_containment(base.joinpath(*parts, "meta", "episode_labels.json"), base)
+
+
 def _labels_path(dataset_id: str) -> Path:
-    base = Path(_get_base_path())
-    path = base / dataset_id / "meta" / "episode_labels.json"
-    return validate_path_containment(path, base)
+    return _labels_path_for_base(dataset_id, _get_base_path())
 
 
 async def _load_labels(dataset_id: str) -> DatasetLabelsFile:
-    path = _labels_path(dataset_id)
-    safe_base = os.path.realpath(_get_base_path())
-    resolved = os.path.realpath(str(path))
-    if not resolved.startswith(safe_base + os.sep):
-        raise HTTPException(
-            status_code=400,
-            detail="Path traversal detected: labels path escapes base directory",
-        )
-    path = Path(resolved)
-    if not await aiofiles.os.path.exists(path):
-        return DatasetLabelsFile(dataset_id=dataset_id)
-    async with aiofiles.open(path, encoding="utf-8") as f:
-        data = json.loads(await f.read())
-        return DatasetLabelsFile.model_validate(data)
+    return await _get_label_storage().load(dataset_id)
 
 
 async def _save_labels(dataset_id: str, labels_file: DatasetLabelsFile) -> None:
-    path = _labels_path(dataset_id)
-    safe_base = os.path.realpath(_get_base_path())
-    resolved = os.path.realpath(str(path))
-    if not resolved.startswith(safe_base + os.sep):
-        raise HTTPException(
-            status_code=400,
-            detail="Path traversal detected: labels path escapes base directory",
-        )
-    path = Path(resolved)
-    await aiofiles.os.makedirs(path.parent, exist_ok=True)
-    content = json.dumps(labels_file.model_dump(), indent=2)
-    async with aiofiles.open(path, "w", encoding="utf-8") as f:
-        await f.write(content)
+    await _get_label_storage().save(dataset_id, labels_file)
 
 
 @router.get("/{dataset_id}/labels")