feat(infra): add blob storage lifecycle policies and folder structure

.cspell/azure-services.txt

@@ -56,6 +56,7 @@ deviceregistry
 eastu
 eastus
 entra
+eventhub
 functionalization
 keyvault
 onedrive
@@ -67,5 +68,3 @@ sharepoint
 southeastasia
 wasbs
 westus
-
-eventhub

data-management/tools/blob_path_validator.py

@@ -0,0 +1,72 @@
+"""Blob path validation for Azure Storage folder structure.
+
+Validates blob paths against naming conventions:
+- Lowercase only (no uppercase)
+- Hyphens for separators (no underscores or spaces in folder names)
+- Date format: YYYY-MM-DD
+- Timestamp format: YYYYMMDD_HHMMSS
+- File extensions: .mcap, .bag, .json, .npz, .mp4, .pt, .onnx, .jit
+
+Validation enforces folder structure and naming patterns for:
+- raw: ROS bags from edge devices
+- converted: LeRobot datasets
+- reports: Validation and inference reports
+- checkpoints: Model checkpoints from training
+"""
+
+import re
+from typing import Literal
+
+DataType = Literal["raw", "converted", "reports", "checkpoints"]
+
+# Regex patterns for each data type
+PATTERNS = {
+    "raw": r"^raw/[a-z0-9-]+/\d{4}-\d{2}-\d{2}/[a-z0-9-]+\.(mcap|bag)$",
+    "converted": r"^converted/[a-z0-9-]+(-v\d+)?/(meta|data|videos)/.+$",
+    "reports": r"^reports/[a-z0-9-]+/\d{4}-\d{2}-\d{2}/[a-z0-9-_]+\.(json|npz|mp4)$",
+    "checkpoints": r"^checkpoints/[a-z0-9-]+/\d{8}_\d{6}(_step_\d+)?\.(pt|onnx|jit)$",
+}
+
+
+def validate_blob_path(blob_name: str, data_type: DataType) -> bool:
+    """Validate blob path follows naming conventions.
+
+    Args:
+        blob_name: Full blob path (e.g., "raw/robot-01/2026-03-05/episode-001.mcap")
+        data_type: Type of data ("raw", "converted", "reports", "checkpoints")
+
+    Returns:
+        True if path is valid, False otherwise
+
+    Raises:
+        ValueError: If data_type is not recognized
+    """
+    if data_type not in PATTERNS:
+        raise ValueError(f"Unknown data type: {data_type}. Must be one of {list(PATTERNS.keys())}")
+
+    return bool(re.match(PATTERNS[data_type], blob_name))
+
+
+def get_validation_error(blob_name: str, data_type: DataType) -> str | None:
+    """Get validation error message if path is invalid.
+
+    Args:
+        blob_name: Full blob path
+        data_type: Type of data
+
+    Returns:
+        Error message if invalid, None if valid
+    """
+    if validate_blob_path(blob_name, data_type):
+        return None
+
+    errors = []
+    if any(c.isupper() for c in blob_name):
+        errors.append("contains uppercase characters (must be lowercase only)")
+    if " " in blob_name:
+        errors.append("contains spaces (use hyphens instead)")
+
+    error_detail = ", ".join(errors) if errors else "does not match expected pattern"
+    return (
+        f"Invalid blob path '{blob_name}': {error_detail}. See docs/cloud/blob-storage-structure.md for path patterns."
+    )

data-management/tools/tests/test_blob_path_validator.py

@@ -0,0 +1,143 @@
+"""Unit tests for blob path validator."""
+
+import pytest
+from blob_path_validator import (
+    get_validation_error,
+    validate_blob_path,
+)
+
+
+class TestRawBagValidation:
+    """Test validation for raw ROS bag paths."""
+
+    def test_valid_raw_mcap(self):
+        assert validate_blob_path("raw/robot-01/2026-03-05/episode-001.mcap", "raw")
+
+    def test_valid_raw_bag(self):
+        assert validate_blob_path("raw/ur10e-arm/2026-03-04/pick-task-001.bag", "raw")
+
+    def test_valid_raw_mobile_manipulator(self):
+        assert validate_blob_path("raw/mobile-manipulator-03/2026-03-01/navigation-001.mcap", "raw")
+
+    def test_invalid_raw_uppercase_device(self):
+        assert not validate_blob_path("raw/Robot-01/2026-03-05/episode-001.mcap", "raw")
+
+    def test_invalid_raw_uppercase_filename(self):
+        assert not validate_blob_path("raw/robot-01/2026-03-05/Episode-001.mcap", "raw")
+
+    def test_invalid_raw_spaces(self):
+        assert not validate_blob_path("raw/robot-01/2026-03-05/episode 001.mcap", "raw")
+
+    def test_invalid_raw_wrong_date_format(self):
+        assert not validate_blob_path("raw/robot-01/03-05-2026/episode-001.mcap", "raw")
+
+    def test_invalid_raw_wrong_extension(self):
+        assert not validate_blob_path("raw/robot-01/2026-03-05/episode-001.txt", "raw")
+
+    def test_invalid_raw_missing_date(self):
+        assert not validate_blob_path("raw/robot-01/episode-001.mcap", "raw")
+
+
+class TestConvertedDatasetValidation:
+    """Test validation for converted LeRobot dataset paths."""
+
+    def test_valid_converted_meta_info(self):
+        assert validate_blob_path("converted/pick-place-v1/meta/info.json", "converted")
+
+    def test_valid_converted_meta_stats(self):
+        assert validate_blob_path("converted/pick-place-v1/meta/stats.json", "converted")
+
+    def test_valid_converted_data_parquet(self):
+        assert validate_blob_path("converted/pick-place-v1/data/chunk-000/episode_000000.parquet", "converted")
+
+    def test_valid_converted_video(self):
+        assert validate_blob_path(
+            "converted/pick-place-v1/videos/observation.image/chunk-000/episode_0000.mp4",
+            "converted",
+        )
+
+    def test_valid_converted_no_version(self):
+        assert validate_blob_path("converted/navigation/meta/info.json", "converted")
+
+    def test_invalid_converted_uppercase(self):
+        assert not validate_blob_path("converted/Pick-Place-v1/meta/info.json", "converted")
+
+    def test_invalid_converted_wrong_subfolder(self):
+        assert not validate_blob_path("converted/pick-place-v1/invalid/info.json", "converted")
+
+
+class TestReportsValidation:
+    """Test validation for validation report paths."""
+
+    def test_valid_reports_json(self):
+        assert validate_blob_path("reports/pick-place-v1/2026-03-05/eval_results.json", "reports")
+
+    def test_valid_reports_npz(self):
+        assert validate_blob_path("reports/pick-place-v1/2026-03-05/ep000_predictions.npz", "reports")
+
+    def test_valid_reports_mp4(self):
+        assert validate_blob_path("reports/pick-place-v1/2026-03-05/inference_video.mp4", "reports")
+
+    def test_invalid_reports_uppercase(self):
+        assert not validate_blob_path("reports/Pick-Place-v1/2026-03-05/eval_results.json", "reports")
+
+    def test_invalid_reports_wrong_date_format(self):
+        assert not validate_blob_path("reports/pick-place-v1/03-05-2026/eval_results.json", "reports")
+
+    def test_invalid_reports_missing_date(self):
+        assert not validate_blob_path("reports/pick-place-v1/eval_results.json", "reports")
+
+
+class TestCheckpointsValidation:
+    """Test validation for model checkpoint paths."""
+
+    def test_valid_checkpoints_with_step(self):
+        assert validate_blob_path("checkpoints/act-policy/20260305_143022_step_1000.pt", "checkpoints")
+
+    def test_valid_checkpoints_no_step(self):
+        assert validate_blob_path("checkpoints/velocity-anymal/20260301_120000.pt", "checkpoints")
+
+    def test_valid_checkpoints_onnx(self):
+        assert validate_blob_path("checkpoints/diffusion-policy/20260304_091500.onnx", "checkpoints")
+
+    def test_valid_checkpoints_jit(self):
+        assert validate_blob_path("checkpoints/rsl-rl/20260305_100000.jit", "checkpoints")
+
+    def test_invalid_checkpoints_uppercase(self):
+        assert not validate_blob_path("checkpoints/ACT-Policy/20260305_143022_step_1000.pt", "checkpoints")
+
+    def test_invalid_checkpoints_wrong_timestamp_format(self):
+        assert not validate_blob_path("checkpoints/act-policy/2026-03-05_14-30-22_step_1000.pt", "checkpoints")
+
+    def test_invalid_checkpoints_wrong_extension(self):
+        assert not validate_blob_path("checkpoints/act-policy/20260305_143022_step_1000.pth", "checkpoints")
+
+
+class TestValidationErrors:
+    """Test validation error messages."""
+
+    def test_error_message_uppercase(self):
+        error = get_validation_error("raw/Robot-01/2026-03-05/episode-001.mcap", "raw")
+        assert error is not None
+        assert "uppercase" in error
+        assert "docs/cloud/blob-storage-structure.md" in error
+
+    def test_error_message_spaces(self):
+        error = get_validation_error("raw/robot-01/2026-03-05/episode 001.mcap", "raw")
+        assert error is not None
+        assert "spaces" in error
+        assert "hyphens" in error
+
+    def test_error_message_multiple_issues(self):
+        error = get_validation_error("raw/Robot-01/2026-03-05/Episode 001.mcap", "raw")
+        assert error is not None
+        assert "uppercase" in error
+        assert "spaces" in error
+
+    def test_valid_path_no_error(self):
+        error = get_validation_error("raw/robot-01/2026-03-05/episode-001.mcap", "raw")
+        assert error is None
+
+    def test_unknown_data_type_raises_error(self):
+        with pytest.raises(ValueError, match="Unknown data type"):
+            validate_blob_path("some/path/file.txt", "invalid")  # type: ignore