feat(data): add camera selector to annotation workspace and fix AV1 frame extraction

.cspell/general-technical.txt

@@ -1926,6 +1926,7 @@ incluster
 instancetype
 instancetypes
 jtzh
+libdav
 libgl
 libglib
 lnkmpfy

data-management/viewer/README.md

@@ -42,8 +42,8 @@ Copy `backend/.env.example` to `backend/.env` and set values for your environmen
 ### Local File Storage (default)
 
 ```env
-HMI_STORAGE_BACKEND=local
-HMI_DATA_PATH=/path/to/your/datasets
+STORAGE_BACKEND=local
+DATA_DIR=/path/to/your/datasets
 ```
 
 ### Azure Blob Storage
@@ -54,7 +54,7 @@ which supports managed identity, workload identity, and Azure CLI credentials
 automatically — no SAS token required in AKS or Container Apps.
 
 ```env
-HMI_STORAGE_BACKEND=azure
+STORAGE_BACKEND=azure
 AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
 AZURE_STORAGE_DATASET_CONTAINER=datasets
 AZURE_STORAGE_ANNOTATION_CONTAINER=annotations
@@ -75,8 +75,8 @@ Expected blob structure:
 
 | Variable                             | Default         | Description                                                    |
 |--------------------------------------|-----------------|----------------------------------------------------------------|
-| `HMI_STORAGE_BACKEND`                | `local`         | Storage backend: `local` or `azure`                            |
-| `HMI_DATA_PATH`                      | `./data`        | Local dataset directory (local mode)                           |
+| `STORAGE_BACKEND`                    | `local`         | Storage backend: `local` or `azure`                            |
+| `DATA_DIR`                      | `./data`        | Local dataset directory (local mode)                           |
 | `AZURE_STORAGE_ACCOUNT_NAME`         | —               | Azure Storage account name (azure mode)                        |
 | `AZURE_STORAGE_DATASET_CONTAINER`    | —               | Blob container for dataset files                               |
 | `AZURE_STORAGE_ANNOTATION_CONTAINER` | —               | Blob container for annotations (defaults to dataset container) |
@@ -205,7 +205,7 @@ The application will be available at `http://localhost:5173`.
 HMI_LOCAL_DATA_PATH=/path/to/datasets docker compose up --build
 
 # Azure Blob Storage mode
-export HMI_STORAGE_BACKEND=azure
+export STORAGE_BACKEND=azure
 export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
 export AZURE_STORAGE_DATASET_CONTAINER=datasets
 export AZURE_STORAGE_ANNOTATION_CONTAINER=annotations
@@ -217,7 +217,7 @@ docker compose up --build
 For AKS with workload identity or Container Apps with managed identity, set:
 
 ```env
-HMI_STORAGE_BACKEND=azure
+STORAGE_BACKEND=azure
 AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
 AZURE_STORAGE_DATASET_CONTAINER=datasets
 BACKEND_HOST=0.0.0.0

data-management/viewer/backend/.env.azure.example

@@ -8,7 +8,7 @@
 # Your Azure AD user must have Storage Blob Data Reader (or Contributor) on the account.
 # Do NOT set AZURE_CLIENT_ID locally — it overrides CLI credentials with managed identity.
 
-HMI_STORAGE_BACKEND=azure
+STORAGE_BACKEND=azure
 AZURE_STORAGE_ACCOUNT_NAME=<insert-storage-account-name>
 AZURE_STORAGE_DATASET_CONTAINER=datasets
 AZURE_STORAGE_ANNOTATION_CONTAINER=annotations

data-management/viewer/backend/.env.example

@@ -7,22 +7,22 @@
 
 # Active storage backend: local (default) | azure
 # Use 'azure' to connect directly to Azure Blob Storage via DefaultAzureCredential
-# or a SAS token. Use 'local' for local filesystem access via HMI_DATA_PATH.
-HMI_STORAGE_BACKEND=local
+# or a SAS token. Use 'local' for local filesystem access via DATA_DIR.
+STORAGE_BACKEND=local
 
 # ─────────────────────────────────────────────────────────────────────────────
-# Local storage (used when HMI_STORAGE_BACKEND=local)
+# Local storage (used when STORAGE_BACKEND=local)
 # ─────────────────────────────────────────────────────────────────────────────
 
 # Path to the directory containing datasets.
 # Each subdirectory is treated as a dataset_id.
 # Relative paths are resolved from this file's directory (backend/).
-HMI_DATA_PATH=../../../datasets
+DATA_DIR=../../../datasets
 
 # ─────────────────────────────────────────────────────────────────────────────
-# Azure Blob Storage (required when HMI_STORAGE_BACKEND=azure)
+# Azure Blob Storage (required when STORAGE_BACKEND=azure)
 # ─────────────────────────────────────────────────────────────────────────────
-# Azure Blob Storage (required when HMI_STORAGE_BACKEND=azure)
+# Azure Blob Storage (required when STORAGE_BACKEND=azure)
 # ─────────────────────────────────────────────────────────────────────────────
 # Prerequisite: run `az login` or `source infrastructure/terraform/prerequisites/az-sub-init.sh`
 # Your Azure AD user needs Storage Blob Data Reader (or Contributor) on the account.

data-management/viewer/backend/Dockerfile

@@ -46,8 +46,8 @@ EXPOSE 8000
 # BACKEND_HOST=0.0.0.0 binds to all interfaces (required in containers)
 ENV BACKEND_HOST=0.0.0.0 \
     BACKEND_PORT=8000 \
-    HMI_STORAGE_BACKEND=local \
-    HMI_DATA_PATH=/data
+    STORAGE_BACKEND=local \
+    DATA_DIR=/data
 
 # Health check
 HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \

data-management/viewer/backend/src/api/config.py

@@ -28,7 +28,7 @@ class AppConfig:
     """Active storage backend: 'local' or 'azure'."""
 
     data_path: str
-    """Local dataset directory (HMI_DATA_PATH). Used when storage_backend='local'."""
+    """Local dataset directory (DATA_DIR). Used when storage_backend='local'."""
 
     azure_account_name: str | None
     """Azure Storage account name. Required when storage_backend='azure'."""
@@ -73,8 +73,8 @@ def load_config(env_path: Path | None = None) -> AppConfig:
 
         load_dotenv(env_path)
 
-    storage_backend = os.environ.get("HMI_STORAGE_BACKEND", "local").lower()
-    data_path = os.environ.get("HMI_DATA_PATH", "./data")
+    storage_backend = os.environ.get("STORAGE_BACKEND", "local").lower()
+    data_path = os.environ.get("DATA_DIR", "./data")
 
     azure_account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME") or None
     azure_dataset_container = os.environ.get("AZURE_STORAGE_DATASET_CONTAINER") or None
@@ -126,12 +126,12 @@ def create_annotation_storage(config: AppConfig):
 
     if config.storage_backend == "azure":
         if not config.azure_account_name:
-            raise ValueError("AZURE_STORAGE_ACCOUNT_NAME is required when HMI_STORAGE_BACKEND=azure")
+            raise ValueError("AZURE_STORAGE_ACCOUNT_NAME is required when STORAGE_BACKEND=azure")
         annotation_container = config.azure_annotation_container or config.azure_dataset_container
         if not annotation_container:
             raise ValueError(
                 "AZURE_STORAGE_ANNOTATION_CONTAINER or AZURE_STORAGE_DATASET_CONTAINER is required "
-                "when HMI_STORAGE_BACKEND=azure"
+                "when STORAGE_BACKEND=azure"
             )
 
         from .storage.azure import AzureBlobStorageAdapter

data-management/viewer/backend/src/api/models/annotations.py

@@ -200,6 +200,37 @@ class AnomalyAnnotation(SanitizedModel):
     anomalies: list[Anomaly] = Field(default_factory=list)
 
 
+# ============================================================================
+# Language Instruction Types (VLA)
+# ============================================================================
+
+
+class InstructionSource(StrEnum):
+    """Source of the language instruction annotation."""
+
+    HUMAN = "human"
+    TEMPLATE = "template"
+    LLM_GENERATED = "llm-generated"
+    RETROACTIVE = "retroactive"
+
+
+class LanguageInstructionAnnotation(SanitizedModel):
+    """Natural language instruction for VLA-conditioned training.
+
+    Stores the primary task instruction plus optional paraphrases and
+    subtask decomposition used for data augmentation and hierarchical
+    policy conditioning.
+    """
+
+    instruction: str = Field(min_length=1, max_length=1000)
+    source: InstructionSource
+    language: str = Field(default="en", max_length=10)
+    paraphrases: list[str] = Field(default_factory=list)
+    subtask_instructions: list[str] = Field(default_factory=list)
+
+    model_config: ClassVar = {"use_enum_values": True}
+
+
 # ============================================================================
 # Combined Episode Annotation Types
 # ============================================================================
@@ -214,6 +245,7 @@ class EpisodeAnnotation(SanitizedModel):
     trajectory_quality: TrajectoryQualityAnnotation
     data_quality: DataQualityAnnotation
     anomalies: AnomalyAnnotation
+    language_instruction: LanguageInstructionAnnotation | None = None
     notes: str | None = None
 
 

data-management/viewer/backend/src/api/routers/joint_config.py

@@ -76,7 +76,7 @@ class JointConfigUpdate(SanitizedModel):
 
 
 def _get_base_path() -> str:
-    return os.environ.get("HMI_DATA_PATH", "./data")
+    return os.environ.get("DATA_DIR", "./data")
 
 
 def _dataset_config_path(dataset_id: str) -> Path:

data-management/viewer/backend/src/api/routers/labels.py

@@ -169,7 +169,7 @@ def _create_label_storage(
     """Create label storage backend based on config."""
     if storage_backend == "azure" and blob_provider is not None:
         return BlobLabelStorage(blob_provider)
-    return LocalLabelStorage(os.environ.get("HMI_DATA_PATH", "./data"))
+    return LocalLabelStorage(os.environ.get("DATA_DIR", "./data"))
 
 
 _label_storage: LabelStorage | None = None
@@ -192,7 +192,7 @@ def _get_label_storage() -> LabelStorage:
 
 
 def _get_base_path() -> str:
-    return os.environ.get("HMI_DATA_PATH", "./data")
+    return os.environ.get("DATA_DIR", "./data")
 
 
 def _labels_path_for_base(dataset_id: str, base_path: str) -> Path:

data-management/viewer/backend/src/api/services/dataset_service/base.py

@@ -33,9 +33,24 @@ def build_trajectory(
     Convert raw numpy arrays into a list of TrajectoryPoint models.
 
     Works for both LeRobot and HDF5 data by accepting optional arrays
-    with sensible defaults (zeros) for missing fields.
+    with sensible defaults (zeros) for missing fields. When ``joint_velocities``
+    is not provided, it is estimated via finite differences of
+    ``joint_positions`` over ``timestamps`` so the velocity view in the UI is
+    populated for datasets that only record positions.
     """
     num_joints = joint_positions.shape[1] if joint_positions.ndim > 1 else 6
+
+    if joint_velocities is None and joint_positions.ndim == 2 and length > 1:
+        # Backwards/forwards differences along the time axis. Guard against
+        # zero or non-monotonic timestamps by clamping dt to a small positive
+        # value.
+        dt = np.diff(timestamps)
+        dt = np.where(dt > 1e-6, dt, 1e-6)
+        diffs = np.diff(joint_positions, axis=0) / dt[:, None]
+        joint_velocities = np.empty_like(joint_positions)
+        joint_velocities[:-1] = diffs
+        joint_velocities[-1] = diffs[-1]
+
     points: list[TrajectoryPoint] = []
 
     for i in range(length):

data-management/viewer/backend/src/api/services/dataset_service/lerobot_handler.py

@@ -13,7 +13,7 @@
 import subprocess
 from pathlib import Path
 
-from ...models.datasources import DatasetInfo, EpisodeData, EpisodeMeta, FeatureSchema, TrajectoryPoint
+from ...models.datasources import DatasetInfo, EpisodeData, EpisodeMeta, FeatureSchema, TaskInfo, TrajectoryPoint
 from .base import build_trajectory
 
 logger = logging.getLogger(__name__)
@@ -112,7 +112,7 @@ def discover(self, dataset_id: str, dataset_path: Path) -> DatasetInfo | None:
                 total_episodes=lr_info.total_episodes,
                 fps=lr_info.fps,
                 features=features,
-                tasks=[],
+                tasks=[TaskInfo(task_index=idx, description=desc) for idx, desc in sorted(loader.get_tasks().items())],
             )
         except Exception as e:
             logger.warning(
@@ -241,17 +241,33 @@ def get_frame_image(
 
         return self._extract_frame_cv2(str(video_path), frame_idx)
 
+    @staticmethod
+    def _resolve_ffmpeg() -> str | None:
+        """Locate a usable ffmpeg binary.
+
+        Prefers the imageio-ffmpeg static binary (ships with libdav1d for AV1
+        and libx264; not affected by host libGL breakage) over a system ffmpeg
+        on PATH.
+        """
+        try:
+            import imageio_ffmpeg
+
+            return imageio_ffmpeg.get_ffmpeg_exe()
+        except Exception:
+            return shutil.which("ffmpeg")
+
     @staticmethod
     def _extract_frame_ffmpeg(video_path: str, frame_idx: int, fps: float) -> bytes | None:
         """Extract a single frame as JPEG using ffmpeg."""
-        if shutil.which("ffmpeg") is None:
+        ffmpeg = LeRobotFormatHandler._resolve_ffmpeg()
+        if ffmpeg is None:
             return None
 
         seek_time = frame_idx / fps
         try:
             proc = subprocess.run(
                 [
-                    "ffmpeg",
+                    ffmpeg,
                     "-ss",
                     f"{seek_time:.6f}",
                     "-i",

data-management/viewer/backend/src/api/services/dataset_service/service.py

@@ -66,7 +66,7 @@ def __init__(
         episode_cache_max_mb: int = 100,
     ):
         if base_path is None:
-            base_path = os.environ.get("HMI_DATA_PATH", "./data")
+            base_path = os.environ.get("DATA_DIR", "./data")
         self.base_path = base_path
         self._datasets: dict[str, DatasetInfo] = {}
         if storage_adapter is not None:
@@ -799,7 +799,7 @@ def get_dataset_service() -> DatasetService:
     Get the global dataset service instance.
 
     On first call, reads application config and creates the appropriate
-    storage adapter and optional BlobDatasetProvider based on HMI_STORAGE_BACKEND.
+    storage adapter and optional BlobDatasetProvider based on STORAGE_BACKEND.
 
     Returns:
         DatasetService singleton.