diff --git a/.cspell/general-technical.txt b/.cspell/general-technical.txt index f9c83e2f..657630eb 100644 --- a/.cspell/general-technical.txt +++ b/.cspell/general-technical.txt @@ -62,6 +62,7 @@ asgi asgs assetguidance asyncio +atheris attestations attunity audiologs @@ -544,6 +545,7 @@ Hypervelocity hypervisor iaas ibm +hypothesis ibpms icmp iconmonstr @@ -585,6 +587,7 @@ interceptable interconnectivity interdependencies interdomain +interp interoperate interoperates interpretability diff --git a/.github/workflows/dataviewer-backend-pytests.yml b/.github/workflows/dataviewer-backend-pytests.yml index fce06122..5a1a0282 100644 --- a/.github/workflows/dataviewer-backend-pytests.yml +++ b/.github/workflows/dataviewer-backend-pytests.yml @@ -43,21 +43,21 @@ jobs: run: uv sync --extra dev --extra analysis --extra hdf5 --extra export --extra auth - name: Run pytest with coverage - run: uv run pytest -v --cov=src --cov-report=xml --cov-report=term-missing + run: uv run pytest -v --cov=src --cov-report=xml:../../../logs/coverage-dataviewer.xml --cov-report=term-missing - name: Upload coverage.xml artifact if: ${{ inputs.code-coverage && always() }} uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: pytest-dataviewer-coverage-xml - path: data-management/viewer/backend/coverage.xml + path: logs/coverage-dataviewer.xml retention-days: 30 - name: Upload coverage to Codecov if: ${{ inputs.code-coverage && always() }} uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 with: - files: coverage.xml + files: logs/coverage-dataviewer.xml use_oidc: true fail_ci_if_error: false verbose: true diff --git a/.github/workflows/fuzz-regression-tests.yml b/.github/workflows/fuzz-regression-tests.yml new file mode 100644 index 00000000..7a40621e --- /dev/null +++ b/.github/workflows/fuzz-regression-tests.yml @@ -0,0 +1,59 @@ +name: Fuzz Regression Tests + +on: + workflow_call: + inputs: + code-coverage: + description: 'Enable Codecov coverage upload' + required: false + default: false + type: boolean + +permissions: + contents: read + +jobs: + fuzz-regression: + name: Fuzz Regression + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + + - name: Setup uv + uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 + + - name: Install dependencies + run: uv sync --group dev + + - name: Run fuzz regression tests + run: uv run pytest tests/fuzz_harness.py -v --cov=tests --cov=data-management/viewer/backend/src --cov=training --cov-report=xml:logs/coverage-fuzz.xml --cov-report=term-missing + + - name: Upload coverage.xml artifact + if: ${{ inputs.code-coverage && always() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: pytest-fuzz-coverage-xml + path: logs/coverage-fuzz.xml + retention-days: 30 + + - name: Upload coverage to Codecov + if: ${{ inputs.code-coverage && always() }} + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + with: + files: logs/coverage-fuzz.xml + use_oidc: true + fail_ci_if_error: false + verbose: true + flags: pytest-fuzz + name: pytest-fuzz-coverage diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 28f1b7cc..822eda0b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -127,6 +127,16 @@ jobs: contents: read id-token: write + # Fuzz regression via deterministic corpus-based tests + fuzz-regression-tests: + name: Fuzz Regression Tests + uses: ./.github/workflows/fuzz-regression-tests.yml + with: + code-coverage: true + permissions: + contents: read + id-token: write + # Python linting using ruff python-lint: name: Python Lint diff --git a/.github/workflows/pester-tests.yml b/.github/workflows/pester-tests.yml index ee8679d7..c224c7e1 100644 --- a/.github/workflows/pester-tests.yml +++ b/.github/workflows/pester-tests.yml @@ -194,7 +194,7 @@ jobs: uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: coverage-report-${{ matrix.os }} - path: logs/coverage.xml + path: logs/coverage-pester.xml retention-days: 30 - name: Upload to Codecov @@ -202,7 +202,7 @@ jobs: uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 with: use_oidc: true - files: logs/coverage.xml + files: logs/coverage-pester.xml flags: pester name: pester-coverage verbose: true @@ -211,8 +211,8 @@ jobs: - name: Coverage Threshold Check if: matrix.os == 'ubuntu-latest' && inputs.code-coverage && always() && steps.pester.outcome != 'skipped' run: | - if (Test-Path logs/coverage.xml) { - [xml]$coverage = Get-Content logs/coverage.xml + if (Test-Path logs/coverage-pester.xml) { + [xml]$coverage = Get-Content logs/coverage-pester.xml $lineRate = $coverage.report.counter | Where-Object { $_.type -eq 'LINE' } | ForEach-Object { diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 5253de30..c9de0249 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -147,6 +147,16 @@ jobs: contents: read id-token: write + # Fuzz regression via deterministic corpus-based tests + fuzz-regression-tests: + name: Fuzz Regression Tests + uses: ./.github/workflows/fuzz-regression-tests.yml + with: + code-coverage: true + permissions: + contents: read + id-token: write + # Python linting using ruff python-lint: name: Python Lint diff --git a/.github/workflows/pytest-tests.yml b/.github/workflows/pytest-tests.yml index 2357c0f6..6ccd3b8b 100644 --- a/.github/workflows/pytest-tests.yml +++ b/.github/workflows/pytest-tests.yml @@ -44,14 +44,14 @@ jobs: uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: pytest-coverage-xml - path: coverage.xml + path: logs/coverage.xml retention-days: 30 - name: Upload coverage to Codecov if: ${{ inputs.code-coverage && always() }} uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 with: - files: coverage.xml + files: logs/coverage.xml use_oidc: true fail_ci_if_error: false verbose: true diff --git a/codecov.yml b/codecov.yml index 7c3b9991..c11f6f3d 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,7 +3,9 @@ # Flags: # go — Go tests covering infrastructure/terraform/e2e/ # pester — PowerShell tests covering scripts/ -# pytest — Python tests covering src/ +# pytest — Python tests covering training/ +# pytest-dataviewer — Dataviewer backend tests covering data-management/viewer/backend/src/ +# pytest-fuzz — Python fuzz regression tests covering tests/, backend, and training # terraform — Terraform tests covering infrastructure/terraform/ # vitest — Vitest tests covering data-management/viewer/frontend/src/ @@ -40,15 +42,53 @@ coverage: - pytest-dataviewer target: auto threshold: 1% + terraform: + flags: + - terraform + target: auto + threshold: 1% vitest: flags: - vitest target: auto threshold: 1% + pytest-fuzz: + flags: + - pytest-fuzz + target: auto + threshold: 1% patch: default: target: auto threshold: 5% + go: + flags: + - go + informational: true + pester: + flags: + - pester + informational: true + pytest: + flags: + - pytest + informational: true + pytest-dataviewer: + flags: + - pytest-dataviewer + informational: true + terraform: + flags: + - terraform + informational: true + vitest: + flags: + - vitest + informational: true + pytest-fuzz: + flags: + - pytest-fuzz + informational: true flags: go: @@ -61,7 +101,7 @@ flags: carryforward: true pytest: paths: - - src/ + - training/ carryforward: true pytest-dataviewer: paths: @@ -75,6 +115,12 @@ flags: paths: - data-management/viewer/frontend/src/ carryforward: true + pytest-fuzz: + paths: + - tests/ + - data-management/viewer/backend/src/ + - training/ + carryforward: true parsers: jacoco: diff --git a/data-management/viewer/backend/pyproject.toml b/data-management/viewer/backend/pyproject.toml index ce8388b8..f200dbe2 100644 --- a/data-management/viewer/backend/pyproject.toml +++ b/data-management/viewer/backend/pyproject.toml @@ -22,6 +22,7 @@ dev = [ "pytest-asyncio==1.3.0", "pytest-cov==7.1.0", "httpx==0.28.1", + "hypothesis==6.151.11", "schemathesis==4.14.3", ] azure = [ diff --git a/data-management/viewer/backend/tests/test_property_based.py b/data-management/viewer/backend/tests/test_property_based.py new file mode 100644 index 00000000..92caf41d --- /dev/null +++ b/data-management/viewer/backend/tests/test_property_based.py @@ -0,0 +1,828 @@ +"""Property-based tests for dataviewer backend pure functions. + +Uses Hypothesis to verify invariants across large input spaces for +validation, sanitization, caching, serialization, and path utilities. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from datetime import UTC, datetime + +import hypothesis.strategies as st +import numpy as np +from hypothesis import assume, given, settings +from hypothesis.extra.numpy import arrays +from numpy.typing import NDArray + +from src.api.services.dataset_service.service import _validate_dataset_id +from src.api.services.episode_cache import CacheStats, EpisodeCache +from src.api.services.frame_interpolation import ( + interpolate_frame_data, + interpolate_image, +) +from src.api.services.trajectory_analysis import TrajectoryAnalyzer +from src.api.storage.paths import dataset_id_to_blob_prefix +from src.api.storage.serializers import DateTimeEncoder +from src.api.validation import ( + SAFE_CAMERA_NAME_PATTERN, + SAFE_DATASET_ID_PATTERN, + _sanitize_nested_value, + sanitize_user_string, + validate_safe_string, +) + +# =================================================================== +# Strategies +# =================================================================== + +_valid_dataset_ids = st.from_regex(re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,50}"), fullmatch=True) + +_valid_camera_names = st.from_regex(re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,30}"), fullmatch=True) + +_nested_json = st.recursive( + st.one_of(st.text(max_size=50), st.integers(), st.floats(allow_nan=False), st.none()), + lambda children: st.one_of( + st.lists(children, max_size=5), + st.tuples(children), + st.dictionaries(st.text(max_size=10), children, max_size=5), + ), + max_leaves=20, +) + + +# =================================================================== +# sanitize_user_string +# =================================================================== + + +class TestSanitizeUserStringProperties: + @given(text=st.text(max_size=500)) + def test_output_never_contains_cr_or_lf(self, text: str) -> None: + result = sanitize_user_string(text) + assert "\r" not in result + assert "\n" not in result + + @given(text=st.text(max_size=500)) + def test_idempotent(self, text: str) -> None: + once = sanitize_user_string(text) + twice = sanitize_user_string(once) + assert once == twice + + @given(text=st.text(max_size=500)) + def test_preserves_non_crlf_characters(self, text: str) -> None: + result = sanitize_user_string(text) + expected = text.replace("\r", "").replace("\n", "") + assert result == expected + + @given(text=st.text(max_size=500)) + def test_length_never_increases(self, text: str) -> None: + assert len(sanitize_user_string(text)) <= len(text) + + +# =================================================================== +# _sanitize_nested_value +# =================================================================== + + +class TestSanitizeNestedValueProperties: + @given(value=_nested_json) + def test_preserves_container_type(self, value: object) -> None: + result = _sanitize_nested_value(value) + assert type(result) is type(value) + + @given(value=st.text(max_size=200)) + def test_string_leaves_sanitized(self, value: str) -> None: + result = _sanitize_nested_value(value) + assert isinstance(result, str) + assert "\r" not in result + assert "\n" not in result + + @given(items=st.lists(st.text(max_size=50), max_size=10)) + def test_list_elements_all_sanitized(self, items: list[str]) -> None: + result = _sanitize_nested_value(items) + assert isinstance(result, list) + for item in result: + assert "\r" not in item + assert "\n" not in item + + @given(mapping=st.dictionaries(st.text(max_size=20), st.text(max_size=50), max_size=10)) + def test_dict_keys_and_values_sanitized(self, mapping: dict[str, str]) -> None: + result = _sanitize_nested_value(mapping) + assert isinstance(result, dict) + for key, val in result.items(): + assert "\r" not in key and "\n" not in key + assert "\r" not in val and "\n" not in val + + @given(value=st.one_of(st.integers(), st.floats(allow_nan=False), st.none())) + def test_non_string_passthrough(self, value: int | float | None) -> None: + assert _sanitize_nested_value(value) == value + + +# =================================================================== +# validate_safe_string +# =================================================================== + + +class TestValidateSafeStringProperties: + @given(value=_valid_dataset_ids) + def test_valid_dataset_ids_accepted(self, value: str) -> None: + result = validate_safe_string(value, pattern=SAFE_DATASET_ID_PATTERN, label="dataset_id") + assert result == value + + @given(value=_valid_camera_names) + def test_valid_camera_names_accepted(self, value: str) -> None: + result = validate_safe_string(value, pattern=SAFE_CAMERA_NAME_PATTERN, label="camera") + assert result == value + + @given(value=st.text(min_size=1, max_size=100)) + def test_null_bytes_always_rejected(self, value: str) -> None: + injected = value[:1] + "\x00" + value[1:] + from fastapi import HTTPException + + try: + validate_safe_string(injected, pattern=SAFE_DATASET_ID_PATTERN, label="test") + except HTTPException as exc: + assert exc.status_code == 400 + return + raise AssertionError("Expected HTTPException for null byte injection") + + @given(prefix=st.text(min_size=1, max_size=50)) + def test_slash_always_rejected(self, prefix: str) -> None: + assume("\x00" not in prefix and prefix not in (".", "..")) + from fastapi import HTTPException + + for char in ("/", "\\"): + try: + validate_safe_string(prefix + char, pattern=SAFE_DATASET_ID_PATTERN, label="test") + except HTTPException as exc: + assert exc.status_code == 400 + else: + raise AssertionError(f"Expected HTTPException for {char!r} injection") + + @given(value=_valid_dataset_ids) + def test_idempotent_for_valid_inputs(self, value: str) -> None: + first = validate_safe_string(value, pattern=SAFE_DATASET_ID_PATTERN, label="test") + second = validate_safe_string(first, pattern=SAFE_DATASET_ID_PATTERN, label="test") + assert first == second + + +# =================================================================== +# _validate_dataset_id +# =================================================================== + + +class TestValidateDatasetIdProperties: + @given( + parts=st.lists( + st.from_regex(re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,20}"), fullmatch=True), + min_size=1, + max_size=5, + ) + ) + def test_valid_nested_ids_accepted(self, parts: list[str]) -> None: + dataset_id = "--".join(parts) + result = _validate_dataset_id(dataset_id) + assert result == dataset_id + + @given( + parts=st.lists( + st.from_regex(re.compile(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,10}"), fullmatch=True), + min_size=6, + max_size=10, + ) + ) + def test_deep_nesting_rejected(self, parts: list[str]) -> None: + dataset_id = "--".join(parts) + try: + _validate_dataset_id(dataset_id) + except ValueError: + return + raise AssertionError("Expected ValueError for deep nesting") + + @given(value=st.text(min_size=1, max_size=100)) + def test_slash_always_rejected(self, value: str) -> None: + for char in ("/", "\\"): + try: + _validate_dataset_id(value + char) + except ValueError: + pass + else: + raise AssertionError(f"Expected ValueError for {char!r}") + + @given( + prefix=st.from_regex(re.compile(r"[a-zA-Z0-9]{1,10}"), fullmatch=True), + ) + def test_dot_parts_rejected(self, prefix: str) -> None: + for dot_part in (".", ".."): + dataset_id = f"{prefix}--{dot_part}" + try: + _validate_dataset_id(dataset_id) + except ValueError: + pass + else: + raise AssertionError(f"Expected ValueError for part={dot_part!r}") + + +# =================================================================== +# dataset_id_to_blob_prefix +# =================================================================== + + +class TestDatasetIdToBlobPrefixProperties: + @given(value=st.text(max_size=200)) + def test_no_double_dash_in_output(self, value: str) -> None: + result = dataset_id_to_blob_prefix(value) + assert "--" not in result + + @given(value=st.text(max_size=200)) + def test_dash_count_equals_slash_count(self, value: str) -> None: + dd_count = value.count("--") + result = dataset_id_to_blob_prefix(value) + new_slashes = result.count("/") - value.count("/") + assert new_slashes == dd_count + + @given( + parts=st.lists(st.text(min_size=1, max_size=20), min_size=1, max_size=5), + ) + def test_roundtrip_with_join_split(self, parts: list[str]) -> None: + assume(all("--" not in p and "/" not in p and "-" not in p for p in parts)) + dataset_id = "--".join(parts) + blob_prefix = dataset_id_to_blob_prefix(dataset_id) + assert blob_prefix == "/".join(parts) + + @given(value=st.text(max_size=200)) + def test_idempotent_when_no_double_dashes(self, value: str) -> None: + assume("--" not in value) + assert dataset_id_to_blob_prefix(value) == value + + +# =================================================================== +# DateTimeEncoder +# =================================================================== + + +class TestDateTimeEncoderProperties: + @given( + dt=st.datetimes( + min_value=datetime(1, 1, 1), + max_value=datetime(9999, 12, 31), + timezones=st.just(UTC), + ) + ) + def test_datetime_produces_iso_string(self, dt: datetime) -> None: + result = json.loads(json.dumps({"ts": dt}, cls=DateTimeEncoder)) + assert isinstance(result["ts"], str) + parsed = datetime.fromisoformat(result["ts"]) + assert parsed == dt + + @given( + dt=st.datetimes( + min_value=datetime(1, 1, 1), + max_value=datetime(9999, 12, 31), + timezones=st.just(UTC), + ) + ) + def test_roundtrip_preserves_value(self, dt: datetime) -> None: + encoded = json.dumps({"ts": dt}, cls=DateTimeEncoder) + decoded = json.loads(encoded) + restored = datetime.fromisoformat(decoded["ts"]) + assert restored == dt + + @given(value=st.one_of(st.integers(), st.text(max_size=50), st.booleans())) + def test_non_datetime_passes_through(self, value: int | str | bool) -> None: + result = json.loads(json.dumps({"v": value}, cls=DateTimeEncoder)) + assert result["v"] == value + + +# =================================================================== +# CacheStats +# =================================================================== + + +class TestCacheStatsProperties: + @given( + hits=st.integers(min_value=0, max_value=10_000), + misses=st.integers(min_value=0, max_value=10_000), + ) + def test_hit_rate_in_unit_interval(self, hits: int, misses: int) -> None: + stats = CacheStats(capacity=32, size=0, hits=hits, misses=misses, total_bytes=0, max_memory_bytes=0) + assert 0.0 <= stats.hit_rate <= 1.0 + + @given(hits=st.integers(min_value=0, max_value=10_000)) + def test_zero_misses_gives_perfect_rate(self, hits: int) -> None: + assume(hits > 0) + stats = CacheStats(capacity=32, size=0, hits=hits, misses=0, total_bytes=0, max_memory_bytes=0) + assert stats.hit_rate == 1.0 + + def test_zero_total_gives_zero_rate(self) -> None: + stats = CacheStats(capacity=32, size=0, hits=0, misses=0, total_bytes=0, max_memory_bytes=0) + assert stats.hit_rate == 0.0 + + +# =================================================================== +# EpisodeCache (stateful) +# =================================================================== + + +class TestEpisodeCacheProperties: + @given( + capacity=st.integers(min_value=1, max_value=20), + n_puts=st.integers(min_value=1, max_value=50), + ) + def test_capacity_never_exceeded(self, capacity: int, n_puts: int) -> None: + cache = EpisodeCache(capacity=capacity, max_memory_bytes=0) + for i in range(n_puts): + cache.put("ds", i, _make_minimal_episode(i)) + assert len(cache._entries) <= capacity + + @given(index=st.integers(min_value=0, max_value=100)) + def test_get_after_put_returns_same_object(self, index: int) -> None: + cache = EpisodeCache(capacity=32, max_memory_bytes=0) + episode = _make_minimal_episode(index) + cache.put("ds", index, episode) + retrieved = cache.get("ds", index) + assert retrieved is episode + + @given(index=st.integers(min_value=0, max_value=100)) + def test_miss_returns_none(self, index: int) -> None: + cache = EpisodeCache(capacity=32, max_memory_bytes=0) + assert cache.get("ds", index) is None + + @given(index=st.integers(min_value=0, max_value=100)) + def test_miss_increments_miss_counter(self, index: int) -> None: + cache = EpisodeCache(capacity=32, max_memory_bytes=0) + cache.get("ds", index) + assert cache._misses == 1 + + @given(index=st.integers(min_value=0, max_value=100)) + def test_hit_increments_hit_counter(self, index: int) -> None: + cache = EpisodeCache(capacity=32, max_memory_bytes=0) + cache.put("ds", index, _make_minimal_episode(index)) + cache.get("ds", index) + assert cache._hits == 1 + + @given( + capacity=st.integers(min_value=2, max_value=10), + extra=st.integers(min_value=1, max_value=5), + ) + def test_lru_eviction_order(self, capacity: int, extra: int) -> None: + cache = EpisodeCache(capacity=capacity, max_memory_bytes=0) + total = capacity + extra + for i in range(total): + cache.put("ds", i, _make_minimal_episode(i)) + for i in range(extra): + assert cache.get("ds", i) is None + for i in range(extra, total): + assert cache.get("ds", i) is not None + + def test_disabled_cache_always_misses(self) -> None: + cache = EpisodeCache(capacity=0) + cache.put("ds", 0, _make_minimal_episode(0)) + assert cache.get("ds", 0) is None + + @given( + capacity=st.integers(min_value=1, max_value=10), + n_puts=st.integers(min_value=0, max_value=20), + ) + def test_stats_hits_plus_misses_equals_total_gets(self, capacity: int, n_puts: int) -> None: + cache = EpisodeCache(capacity=capacity, max_memory_bytes=0) + total_gets = 0 + for i in range(n_puts): + cache.put("ds", i, _make_minimal_episode(i)) + for i in range(n_puts + 5): + cache.get("ds", i) + total_gets += 1 + stats = cache.stats() + assert stats.hits + stats.misses == total_gets + + +# =================================================================== +# Helpers +# =================================================================== + + +@dataclass(frozen=True) +class _MinimalTrajectoryPoint: + timestamp: float = 0.0 + frame: int = 0 + gripper_state: float = 0.0 + joint_positions: list[float] = field(default_factory=list) + joint_velocities: list[float] = field(default_factory=list) + end_effector_pose: list[float] = field(default_factory=list) + + +@dataclass(frozen=True) +class _MinimalEpisodeMeta: + episode_index: int = 0 + length: int = 0 + dataset_id: str = "" + + +@dataclass(frozen=True) +class _MinimalEpisode: + meta: _MinimalEpisodeMeta = field(default_factory=_MinimalEpisodeMeta) + video_urls: dict[str, str] = field(default_factory=dict) + trajectory_data: list[_MinimalTrajectoryPoint] = field(default_factory=list) + + +def _make_minimal_episode(index: int, length: int = 5) -> _MinimalEpisode: + """Build a lightweight episode-like object for cache tests.""" + return _MinimalEpisode( + meta=_MinimalEpisodeMeta(episode_index=index, length=length, dataset_id="ds"), + video_urls={}, + trajectory_data=[_MinimalTrajectoryPoint() for _ in range(length)], + ) + + +# =================================================================== +# Strategies — Frame Interpolation & Trajectory Analysis +# =================================================================== + +_uint8_images = st.shared( + st.tuples( + st.integers(min_value=1, max_value=64), + st.integers(min_value=1, max_value=64), + st.integers(min_value=1, max_value=4), + ), + key="img_shape", +).flatmap( + lambda shape: st.tuples( + arrays(np.uint8, shape, elements=st.integers(0, 255)), + arrays(np.uint8, shape, elements=st.integers(0, 255)), + ) +) + +_interp_factor = st.floats(min_value=0.0, max_value=1.0, allow_nan=False, allow_infinity=False) + +_small_float_array = st.integers(min_value=3, max_value=50).flatmap( + lambda n: st.tuples( + st.just(n), + arrays(np.float64, (n, 3), elements=st.floats(-1e3, 1e3, allow_nan=False, allow_infinity=False)), + ) +) + + +# =================================================================== +# Frame Interpolation — Property Tests +# =================================================================== + + +class TestInterpolateImageProperties: + """Property tests for interpolate_image.""" + + @given(images=_uint8_images, t=_interp_factor) + @settings(max_examples=80) + def test_output_shape_matches_input(self, images: tuple, t: float) -> None: + img1, img2 = images + result = interpolate_image(img1, img2, t) + assert result.shape == img1.shape + + @given(images=_uint8_images, t=_interp_factor) + @settings(max_examples=80) + def test_output_dtype_is_uint8(self, images: tuple, t: float) -> None: + img1, img2 = images + result = interpolate_image(img1, img2, t) + assert result.dtype == np.uint8 + + @given(images=_uint8_images) + @settings(max_examples=40) + def test_t_zero_returns_first_image(self, images: tuple) -> None: + img1, img2 = images + result = interpolate_image(img1, img2, t=0.0) + np.testing.assert_array_equal(result, img1) + + @given(images=_uint8_images) + @settings(max_examples=40) + def test_t_one_returns_second_image(self, images: tuple) -> None: + img1, img2 = images + result = interpolate_image(img1, img2, t=1.0) + np.testing.assert_array_equal(result, img2) + + @given( + shape_a=st.tuples(st.integers(1, 8), st.integers(1, 8), st.just(3)), + shape_b=st.tuples(st.integers(1, 8), st.integers(1, 8), st.just(3)), + ) + def test_shape_mismatch_raises_value_error(self, shape_a: tuple, shape_b: tuple) -> None: + assume(shape_a != shape_b) + img1 = np.zeros(shape_a, dtype=np.uint8) + img2 = np.zeros(shape_b, dtype=np.uint8) + try: + interpolate_image(img1, img2) + raise AssertionError("Expected ValueError") + except ValueError: + pass + + @given(images=_uint8_images, t=_interp_factor) + @settings(max_examples=60) + def test_output_values_in_valid_range(self, images: tuple, t: float) -> None: + img1, img2 = images + result = interpolate_image(img1, img2, t) + assert result.min() >= 0 + assert result.max() <= 255 + + +class TestInterpolateFrameDataProperties: + """Property tests for interpolate_frame_data.""" + + @given( + n=st.integers(min_value=2, max_value=30), + cols=st.integers(min_value=1, max_value=6), + t=_interp_factor, + ) + @settings(max_examples=80) + def test_output_shape_matches_row(self, n: int, cols: int, t: float) -> None: + data = np.random.default_rng(42).standard_normal((n, cols)) + idx = np.random.default_rng(0).integers(0, n - 1) + result = interpolate_frame_data(data, idx, t) + assert result.shape == (cols,) + + @given( + n=st.integers(min_value=2, max_value=20), + t=_interp_factor, + ) + @settings(max_examples=60) + def test_integer_dtype_preserved(self, n: int, t: float) -> None: + data = np.random.default_rng(42).integers(0, 100, size=(n, 3)).astype(np.int32) + result = interpolate_frame_data(data, 0, t) + assert result.dtype == np.int32 + + @given( + n=st.integers(min_value=2, max_value=20), + t=_interp_factor, + ) + @settings(max_examples=60) + def test_float_dtype_returns_float(self, n: int, t: float) -> None: + data = np.random.default_rng(42).standard_normal((n, 3)).astype(np.float64) + result = interpolate_frame_data(data, 0, t) + assert np.issubdtype(result.dtype, np.floating) + + @given(n=st.integers(min_value=2, max_value=20)) + @settings(max_examples=40) + def test_negative_index_raises_index_error(self, n: int) -> None: + data = np.zeros((n, 3)) + try: + interpolate_frame_data(data, -1) + raise AssertionError("Expected IndexError") + except IndexError: + pass + + @given(n=st.integers(min_value=2, max_value=20)) + @settings(max_examples=40) + def test_out_of_range_index_raises_index_error(self, n: int) -> None: + data = np.zeros((n, 3)) + try: + interpolate_frame_data(data, n - 1) + raise AssertionError("Expected IndexError") + except IndexError: + pass + + @given( + n=st.integers(min_value=2, max_value=20), + cols=st.integers(min_value=1, max_value=6), + ) + @settings(max_examples=40) + def test_t_zero_returns_first_frame(self, n: int, cols: int) -> None: + data = np.random.default_rng(42).standard_normal((n, cols)) + result = interpolate_frame_data(data, 0, t=0.0) + np.testing.assert_allclose(result, data[0], atol=1e-10) + + @given( + n=st.integers(min_value=2, max_value=20), + cols=st.integers(min_value=1, max_value=6), + ) + @settings(max_examples=40) + def test_t_one_returns_second_frame(self, n: int, cols: int) -> None: + data = np.random.default_rng(42).standard_normal((n, cols)) + result = interpolate_frame_data(data, 0, t=1.0) + np.testing.assert_allclose(result, data[1], atol=1e-10) + + +# =================================================================== +# Trajectory Analysis — Property Tests +# =================================================================== + + +class TestComputeSmoothnessProperties: + """Property tests for TrajectoryAnalyzer._compute_smoothness.""" + + @given( + jerk=arrays( + np.float64, + st.tuples(st.integers(1, 50), st.integers(1, 6)), + elements=st.floats(-1e4, 1e4, allow_nan=False, allow_infinity=False), + ), + ) + @settings(max_examples=80) + def test_output_in_unit_interval(self, jerk: NDArray) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._compute_smoothness(jerk) + assert 0.0 <= result <= 1.0 + + def test_empty_jerk_returns_one(self) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._compute_smoothness(np.array([]).reshape(0, 3)) + assert result == 1.0 + + def test_zero_jerk_returns_one(self) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._compute_smoothness(np.zeros((10, 3))) + assert result == 1.0 + + +class TestComputeEfficiencyProperties: + """Property tests for TrajectoryAnalyzer._compute_efficiency.""" + + @given( + positions=arrays( + np.float64, + st.tuples(st.integers(2, 50), st.integers(1, 6)), + elements=st.floats(-1e3, 1e3, allow_nan=False, allow_infinity=False), + ), + ) + @settings(max_examples=80) + def test_output_in_unit_interval(self, positions: NDArray) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._compute_efficiency(positions) + assert 0.0 <= result <= 1.0 + + def test_single_point_returns_one(self) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._compute_efficiency(np.array([[1.0, 2.0, 3.0]])) + assert result == 1.0 + + @given( + start=arrays(np.float64, (3,), elements=st.floats(-100, 100, allow_nan=False, allow_infinity=False)), + end=arrays(np.float64, (3,), elements=st.floats(-100, 100, allow_nan=False, allow_infinity=False)), + n=st.integers(min_value=2, max_value=20), + ) + @settings(max_examples=60) + def test_straight_line_efficiency_near_one(self, start: NDArray, end: NDArray, n: int) -> None: + assume(np.linalg.norm(end - start) > 1e-4) + t_values = np.linspace(0.0, 1.0, n) + positions = np.array([start + t * (end - start) for t in t_values]) + analyzer = TrajectoryAnalyzer() + result = analyzer._compute_efficiency(positions) + assert result > 0.99 + + +class TestDetermineFlagsProperties: + """Property tests for TrajectoryAnalyzer._determine_flags.""" + + @given( + smoothness=st.floats(0.0, 1.0, allow_nan=False), + jitter=st.floats(0.0, 1.0, allow_nan=False), + hesitation_count=st.integers(0, 20), + correction_count=st.integers(0, 20), + ) + @settings(max_examples=100) + def test_returns_list_of_strings( + self, + smoothness: float, + jitter: float, + hesitation_count: int, + correction_count: int, + ) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._determine_flags(smoothness, jitter, hesitation_count, correction_count) + assert isinstance(result, list) + assert all(isinstance(f, str) for f in result) + + def test_good_metrics_no_flags(self) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._determine_flags(smoothness=0.9, jitter=0.1, hesitation_count=0, correction_count=0) + assert result == [] + + @given(smoothness=st.floats(0.0, 0.499, allow_nan=False)) + @settings(max_examples=40) + def test_low_smoothness_flags_jittery(self, smoothness: float) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._determine_flags(smoothness, jitter=0.0, hesitation_count=0, correction_count=0) + assert "jittery" in result + + @given(jitter=st.floats(0.301, 1.0, allow_nan=False)) + @settings(max_examples=40) + def test_high_jitter_flags_noise(self, jitter: float) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._determine_flags(smoothness=0.9, jitter=jitter, hesitation_count=0, correction_count=0) + assert "high_frequency_noise" in result + + @given(hesitation=st.integers(min_value=3, max_value=20)) + @settings(max_examples=40) + def test_many_hesitations_flags_hesitant(self, hesitation: int) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._determine_flags(smoothness=0.9, jitter=0.0, hesitation_count=hesitation, correction_count=0) + assert "hesitant" in result + + @given(corrections=st.integers(min_value=6, max_value=30)) + @settings(max_examples=40) + def test_many_corrections_flags_excessive(self, corrections: int) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._determine_flags(smoothness=0.9, jitter=0.0, hesitation_count=0, correction_count=corrections) + assert "excessive_corrections" in result + + +class TestComputeOverallScoreProperties: + """Property tests for TrajectoryAnalyzer._compute_overall_score.""" + + @given( + smoothness=st.floats(0.0, 1.0, allow_nan=False), + efficiency=st.floats(0.0, 1.0, allow_nan=False), + jitter=st.floats(0.0, 1.0, allow_nan=False), + hesitation_count=st.integers(0, 20), + correction_count=st.integers(0, 30), + ) + @settings(max_examples=120) + def test_score_in_valid_range( + self, + smoothness: float, + efficiency: float, + jitter: float, + hesitation_count: int, + correction_count: int, + ) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._compute_overall_score(smoothness, efficiency, jitter, hesitation_count, correction_count) + assert result in {1, 2, 3, 4, 5} + + def test_perfect_metrics_return_five(self) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._compute_overall_score( + smoothness=1.0, + efficiency=1.0, + jitter=0.0, + hesitation_count=0, + correction_count=0, + ) + assert result == 5 + + def test_worst_metrics_return_one(self) -> None: + analyzer = TrajectoryAnalyzer() + result = analyzer._compute_overall_score( + smoothness=0.0, + efficiency=0.0, + jitter=1.0, + hesitation_count=20, + correction_count=30, + ) + assert result == 1 + + +class TestTrajectoryAnalyzerIntegrationProperties: + """Property tests for TrajectoryAnalyzer.analyze end-to-end.""" + + @given( + n=st.integers(min_value=0, max_value=2), + dims=st.integers(min_value=1, max_value=6), + ) + @settings(max_examples=40) + def test_short_trajectory_returns_safe_defaults(self, n: int, dims: int) -> None: + positions = np.zeros((n, dims)) + timestamps = np.arange(n, dtype=np.float64) + analyzer = TrajectoryAnalyzer() + result = analyzer.analyze(positions, timestamps) + assert result.smoothness == 1.0 + assert result.efficiency == 1.0 + assert result.jitter == 0.0 + assert result.hesitation_count == 0 + assert result.correction_count == 0 + assert result.overall_score == 3 + assert result.flags == [] + + @given(data=_small_float_array) + @settings(max_examples=60, deadline=None) + def test_analyze_returns_valid_metric_types(self, data: tuple) -> None: + n, positions = data + timestamps = np.cumsum(np.full(n, 0.033)) + analyzer = TrajectoryAnalyzer() + result = analyzer.analyze(positions, timestamps) + assert isinstance(result.smoothness, float) + assert isinstance(result.efficiency, float) + assert isinstance(result.jitter, float) + assert isinstance(result.hesitation_count, int) + assert isinstance(result.correction_count, int) + assert result.overall_score in {1, 2, 3, 4, 5} + assert isinstance(result.flags, list) + + @given(data=_small_float_array) + @settings(max_examples=60) + def test_smoothness_and_efficiency_in_unit_interval(self, data: tuple) -> None: + n, positions = data + timestamps = np.cumsum(np.full(n, 0.033)) + analyzer = TrajectoryAnalyzer() + result = analyzer.analyze(positions, timestamps) + assert 0.0 <= result.smoothness <= 1.0 + assert 0.0 <= result.efficiency <= 1.0 + assert 0.0 <= result.jitter <= 1.0 + + @given(data=_small_float_array) + @settings(max_examples=60) + def test_counts_are_non_negative(self, data: tuple) -> None: + n, positions = data + timestamps = np.cumsum(np.full(n, 0.033)) + analyzer = TrajectoryAnalyzer() + result = analyzer.analyze(positions, timestamps) + assert result.hesitation_count >= 0 + assert result.correction_count >= 0 diff --git a/data-management/viewer/frontend/package-lock.json b/data-management/viewer/frontend/package-lock.json index 129f2afe..29d4f5e9 100644 --- a/data-management/viewer/frontend/package-lock.json +++ b/data-management/viewer/frontend/package-lock.json @@ -61,6 +61,7 @@ "eslint-plugin-react-hooks": "5.2.0", "eslint-plugin-react-refresh": "0.4.26", "eslint-plugin-simple-import-sort": "12.1.1", + "fast-check": "^4.6.0", "globals": "15.15.0", "happy-dom": "20.8.9", "lint-staged": "16.2.7", @@ -6515,6 +6516,29 @@ "node": ">=12.0.0" } }, + "node_modules/fast-check": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.6.0.tgz", + "integrity": "sha512-h7H6Dm0Fy+H4ciQYFxFjXnXkzR2kr9Fb22c0UBpHnm59K2zpr2t13aPTHlltFiNT6zuxp6HMPAVVvgur4BLdpA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -8929,6 +8953,23 @@ "node": ">=6" } }, + "node_modules/pure-rand": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.0.tgz", + "integrity": "sha512-IoM8YF/jY0hiugFo/wOWqfmarlE6J0wc6fDK1PhftMk7MGhVZl88sZimmqBBFomLOCSmcCCpsfj7wXASCpvK9A==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", diff --git a/data-management/viewer/frontend/package.json b/data-management/viewer/frontend/package.json index 4810c414..20a1936c 100644 --- a/data-management/viewer/frontend/package.json +++ b/data-management/viewer/frontend/package.json @@ -71,6 +71,7 @@ "eslint-plugin-react-hooks": "5.2.0", "eslint-plugin-react-refresh": "0.4.26", "eslint-plugin-simple-import-sort": "12.1.1", + "fast-check": "4.6.0", "globals": "15.15.0", "happy-dom": "20.8.9", "lint-staged": "16.2.7", diff --git a/data-management/viewer/frontend/src/lib/__tests__/api-client-fuzz.test.ts b/data-management/viewer/frontend/src/lib/__tests__/api-client-fuzz.test.ts new file mode 100644 index 00000000..bbf58b97 --- /dev/null +++ b/data-management/viewer/frontend/src/lib/__tests__/api-client-fuzz.test.ts @@ -0,0 +1,234 @@ +/** + * Fuzz harness for API client key-transformation utilities. + * + * Complements api-client.property.test.ts with adversarial inputs: + * full Unicode, control characters, deeply nested structures, and + * boundary values that exercise crash-resistance and invariant + * preservation under arbitrary data. + */ +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' + +import { snakeToCamel, transformKeys } from '../api-client' + +/** Build a JSON-like arbitrary at the given nesting depth. */ +function jsonLike(maxDepth: number): fc.Arbitrary { + const leaf = fc.oneof( + fc.integer(), + fc.double({ noNaN: true }), + fc.string(), + fc.boolean(), + fc.constant(null), + ) + let current: fc.Arbitrary = leaf + for (let d = 0; d < maxDepth; d++) { + const inner = current + current = fc.oneof( + { weight: 3, arbitrary: leaf }, + { weight: 1, arbitrary: fc.array(inner, { maxLength: 5 }) }, + { weight: 1, arbitrary: fc.dictionary(fc.string(), inner, { maxKeys: 5 }) }, + ) + } + return current +} + +/** Arbitrary producing strings from the full 16-bit code-point range. */ +const anyChars = fc + .array(fc.integer({ min: 0, max: 0xffff }), { minLength: 0, maxLength: 60 }) + .map((codes) => String.fromCharCode(...codes)) + +describe('snakeToCamel fuzz', () => { + it('never throws on arbitrary Unicode input', () => { + fc.assert( + fc.property(anyChars, (input) => { + expect(() => snakeToCamel(input)).not.toThrow() + }), + ) + }) + + it('never throws on 16-bit strings including control characters', () => { + fc.assert( + fc.property(anyChars, (input) => { + expect(() => snakeToCamel(input)).not.toThrow() + }), + ) + }) + + it('output length is at most input length', () => { + fc.assert( + fc.property(anyChars, (input) => { + expect(snakeToCamel(input).length).toBeLessThanOrEqual(input.length) + }), + ) + }) + + it('is idempotent on arbitrary Unicode strings', () => { + fc.assert( + fc.property(anyChars, (input) => { + const once = snakeToCamel(input) + expect(snakeToCamel(once)).toBe(once) + }), + ) + }) + + it('output never contains underscore-lowercase pattern on arbitrary input', () => { + fc.assert( + fc.property(anyChars, (input) => { + expect(snakeToCamel(input)).not.toMatch(/_[a-z]/) + }), + ) + }) + + it('returns empty string for empty input', () => { + expect(snakeToCamel('')).toBe('') + }) + + it('handles strings composed entirely of underscores', () => { + fc.assert( + fc.property( + fc.integer({ min: 1, max: 50 }).map((n) => '_'.repeat(n)), + (input) => { + const result = snakeToCamel(input) + expect(result).toBe(input) + expect(result).not.toMatch(/_[a-z]/) + }, + ), + ) + }) + + it('handles strings with embedded null bytes', () => { + fc.assert( + fc.property(fc.string(), (base) => { + const withNull = `${base}\0${base}` + expect(() => snakeToCamel(withNull)).not.toThrow() + }), + ) + }) +}) + +describe('transformKeys fuzz', () => { + it('never throws on arbitrary JSON-like input', () => { + fc.assert( + fc.property(jsonLike(4), (input) => { + expect(() => transformKeys(input)).not.toThrow() + }), + ) + }) + + it('preserves primitive values unchanged', () => { + fc.assert( + fc.property( + fc.oneof( + fc.integer(), + fc.double({ noNaN: true }), + fc.string(), + fc.boolean(), + fc.constant(null), + ), + (input) => { + expect(transformKeys(input)).toBe(input) + }, + ), + ) + }) + + it('undefined passes through as a primitive', () => { + expect(transformKeys(undefined)).toBeUndefined() + }) + + it('preserves array length on arbitrary arrays', () => { + fc.assert( + fc.property(fc.array(jsonLike(2), { maxLength: 20 }), (arr) => { + const result = transformKeys(arr) as unknown[] + expect(result).toHaveLength(arr.length) + }), + ) + }) + + it('output key count does not exceed input key count', () => { + fc.assert( + fc.property(fc.dictionary(fc.string(), jsonLike(1), { maxKeys: 10 }), (obj) => { + const result = transformKeys(obj) as Record + expect(Object.keys(result).length).toBeLessThanOrEqual(Object.keys(obj).length) + }), + ) + }) + + it('handles deeply nested structures without throwing', () => { + fc.assert( + fc.property(jsonLike(8), (input) => { + expect(() => transformKeys(input)).not.toThrow() + }), + { numRuns: 50 }, + ) + }) + + it('preserves leaf values through nested transformation', () => { + fc.assert( + fc.property(fc.integer(), fc.string(), fc.boolean(), (num, str, bool) => { + const input = { + num_value: num, + str_value: str, + nested_obj: { bool_value: bool }, + } + const result = transformKeys<{ + numValue: number + strValue: string + nestedObj: { boolValue: boolean } + }>(input) + expect(result.numValue).toBe(num) + expect(result.strValue).toBe(str) + expect(result.nestedObj.boolValue).toBe(bool) + }), + ) + }) + + it('arrays of objects maintain element order', () => { + fc.assert( + fc.property( + fc + .array(fc.integer(), { minLength: 1, maxLength: 10 }) + // cspell:ignore nums + .map((nums) => nums.map((n, i) => ({ item_index: i, item_value: n }))), + (arr) => { + const result = transformKeys(arr) as Array<{ itemIndex: number; itemValue: number }> + result.forEach((item, i) => { + expect(item.itemIndex).toBe(i) + expect(item.itemValue).toBe(arr[i].item_value) + }) + }, + ), + ) + }) + + it('handles mixed arrays of primitives and objects', () => { + fc.assert( + fc.property( + fc.array( + fc.oneof( + fc.integer(), + fc.string(), + fc.constant(null), + fc.record({ a_key: fc.integer() }), + ), + { maxLength: 10 }, + ), + (arr) => { + const result = transformKeys(arr) as unknown[] + expect(result).toHaveLength(arr.length) + }, + ), + ) + }) + + it('all output keys satisfy snake-to-camel invariant', () => { + fc.assert( + fc.property(fc.dictionary(fc.string(), fc.integer(), { maxKeys: 10 }), (obj) => { + const result = transformKeys(obj) as Record + for (const key of Object.keys(result)) { + expect(key).not.toMatch(/_[a-z]/) + } + }), + ) + }) +}) diff --git a/data-management/viewer/frontend/src/lib/__tests__/api-client.property.test.ts b/data-management/viewer/frontend/src/lib/__tests__/api-client.property.test.ts new file mode 100644 index 00000000..067a9349 --- /dev/null +++ b/data-management/viewer/frontend/src/lib/__tests__/api-client.property.test.ts @@ -0,0 +1,92 @@ +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' + +import { snakeToCamel, transformKeys } from '../api-client' + +const snakeChars = 'abcdefghijklmnopqrstuvwxyz_'.split('') +const alphaChars = 'abcdefghijklmnopqrstuvwxyz'.split('') + +const snakeString = fc + .array(fc.constantFrom(...snakeChars), { minLength: 1, maxLength: 50 }) + .map((chars) => chars.join('')) + +const alphaString = fc + .array(fc.constantFrom(...alphaChars), { minLength: 1, maxLength: 50 }) + .map((chars) => chars.join('')) + +describe('snakeToCamel', () => { + it('output never contains underscore followed by lowercase letter', () => { + fc.assert( + fc.property(snakeString, (input) => { + const result = snakeToCamel(input) + expect(result).not.toMatch(/_[a-z]/) + }), + ) + }) + + it('is idempotent on its own output', () => { + fc.assert( + fc.property(snakeString, (input) => { + const once = snakeToCamel(input) + const twice = snakeToCamel(once) + expect(twice).toBe(once) + }), + ) + }) + + it('preserves strings without underscores', () => { + fc.assert( + fc.property(alphaString, (input) => { + expect(snakeToCamel(input)).toBe(input) + }), + ) + }) +}) + +describe('transformKeys', () => { + it('preserves array length', () => { + fc.assert( + fc.property( + fc.array(fc.record({ some_key: fc.integer(), another_key: fc.string() }), { + maxLength: 20, + }), + (arr) => { + const result = transformKeys(arr) as unknown[] + expect(result).toHaveLength(arr.length) + }, + ), + ) + }) + + it('preserves primitive values through transformation', () => { + fc.assert( + fc.property(fc.integer(), (num) => { + expect(transformKeys(num)).toBe(num) + }), + ) + }) + + it('preserves null and string primitives', () => { + expect(transformKeys(null)).toBeNull() + fc.assert( + fc.property(fc.string(), (str) => { + expect(transformKeys(str)).toBe(str) + }), + ) + }) + + it('converts all snake_case keys in nested objects', () => { + fc.assert( + fc.property(fc.integer(), fc.integer(), (a, b) => { + const input = { outer_key: { inner_key: a }, simple_key: b } + const result = transformKeys(input) as Record + expect(result).toHaveProperty('outerKey') + expect(result).toHaveProperty('simpleKey') + const nested = result.outerKey as Record + expect(nested).toHaveProperty('innerKey') + expect(nested.innerKey).toBe(a) + expect(result.simpleKey).toBe(b) + }), + ) + }) +}) diff --git a/data-management/viewer/frontend/src/lib/__tests__/edit-store-frame-utils.property.test.ts b/data-management/viewer/frontend/src/lib/__tests__/edit-store-frame-utils.property.test.ts new file mode 100644 index 00000000..aac79acf --- /dev/null +++ b/data-management/viewer/frontend/src/lib/__tests__/edit-store-frame-utils.property.test.ts @@ -0,0 +1,110 @@ +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' + +import type { FrameInsertion } from '@/types/episode-edit' + +import { + getEffectiveFrameCount, + getEffectiveIndex, + getOriginalIndex, +} from '../../stores/edit-store-frame-utils' + +const frameInsertion: fc.Arbitrary = fc.record({ + afterFrameIndex: fc.nat({ max: 99 }), + interpolationFactor: fc.double({ min: 0, max: 1, noNaN: true }), +}) + +function buildInsertedFrames(entries: [number, FrameInsertion][]): Map { + return new Map(entries) +} + +function buildRemovedFrames(indices: number[]): Set { + return new Set(indices) +} + +const editScenario = fc + .record({ + originalCount: fc.integer({ min: 1, max: 100 }), + insertions: fc.array(fc.tuple(fc.nat({ max: 99 }), frameInsertion), { maxLength: 10 }), + removals: fc.array(fc.nat({ max: 99 }), { maxLength: 10 }), + }) + .map(({ originalCount, insertions, removals }) => ({ + originalCount, + insertedFrames: buildInsertedFrames(insertions), + removedFrames: buildRemovedFrames(removals.filter((r) => r < originalCount)), + })) + +describe('getEffectiveFrameCount', () => { + it('is always non-negative', () => { + fc.assert( + fc.property(editScenario, ({ originalCount, insertedFrames, removedFrames }) => { + const count = getEffectiveFrameCount(originalCount, insertedFrames, removedFrames) + expect(count).toBeGreaterThanOrEqual(0) + }), + ) + }) + + it('equals originalCount when no edits', () => { + fc.assert( + fc.property(fc.integer({ min: 1, max: 1000 }), (originalCount) => { + const count = getEffectiveFrameCount(originalCount, new Map(), new Set()) + expect(count).toBe(originalCount) + }), + ) + }) + + it('decreases by one per valid removal', () => { + fc.assert( + fc.property( + fc.integer({ min: 2, max: 100 }), + fc.integer({ min: 0, max: 98 }), + (originalCount, removeIdx) => { + fc.pre(removeIdx < originalCount) + const removed = new Set([removeIdx]) + const count = getEffectiveFrameCount(originalCount, new Map(), removed) + expect(count).toBe(originalCount - 1) + }, + ), + ) + }) +}) + +describe('getEffectiveIndex roundtrip', () => { + it('getOriginalIndex inverts getEffectiveIndex for non-inserted frames', () => { + fc.assert( + fc.property( + fc.integer({ min: 1, max: 50 }), + fc.integer({ min: 0, max: 49 }), + (originalCount, originalIndex) => { + fc.pre(originalIndex < originalCount) + const inserted = new Map() + const removed = new Set() + const effective = getEffectiveIndex(originalIndex, inserted, removed) + const recovered = getOriginalIndex(effective, inserted, removed) + expect(recovered).toBe(originalIndex) + }, + ), + ) + }) + + it('getOriginalIndex returns null for inserted frame positions', () => { + fc.assert( + fc.property( + fc.integer({ min: 3, max: 50 }), + fc.integer({ min: 0, max: 47 }), + (originalCount, afterIdx) => { + fc.pre(afterIdx < originalCount - 1) + const insertion: FrameInsertion = { + afterFrameIndex: afterIdx, + interpolationFactor: 0.5, + } + const inserted = new Map([[afterIdx, insertion]]) + const removed = new Set() + const insertedEffective = getEffectiveIndex(afterIdx, inserted, removed) + 1 + const recovered = getOriginalIndex(insertedEffective, inserted, removed) + expect(recovered).toBeNull() + }, + ), + ) + }) +}) diff --git a/data-management/viewer/frontend/src/lib/__tests__/playback-utils-fuzz.test.ts b/data-management/viewer/frontend/src/lib/__tests__/playback-utils-fuzz.test.ts new file mode 100644 index 00000000..3a8f92aa --- /dev/null +++ b/data-management/viewer/frontend/src/lib/__tests__/playback-utils-fuzz.test.ts @@ -0,0 +1,576 @@ +/** + * Fuzz harness for playback synchronization utilities. + * + * Complements playback-utils.property.test.ts with adversarial inputs: + * NaN, Infinity, negative values, extreme magnitudes, and boundary + * conditions that exercise crash-resistance under arbitrary data. + */ +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' + +import { + clampFrameToPlaybackRange, + computeEffectiveFps, + computePlaybackTarget, + computeSyncAction, + needsSeekBeforePlay, + type PlaybackRange, + resolvePlaybackRange, + resolvePlaybackTick, + shouldLoopActivePlaybackRange, + shouldRecoverPlaybackAfterDesync, + shouldRecoverStalledPlayback, + shouldRestartPlaybackAfterLoop, +} from '../playback-utils' + +// --- Adversarial arbitraries --- + +/** Arbitrary that includes NaN, ±Infinity, ±0, and extreme doubles. */ +const adversarialNumber = fc.oneof( + fc.constant(NaN), + fc.constant(Infinity), + fc.constant(-Infinity), + fc.constant(0), + fc.constant(-0), + fc.constant(Number.MAX_SAFE_INTEGER), + fc.constant(Number.MIN_SAFE_INTEGER), + fc.constant(Number.MAX_VALUE), + fc.constant(Number.MIN_VALUE), + fc.constant(-Number.MAX_VALUE), + fc.double(), + fc.integer(), +) + +/** Arbitrary producing any integer, including negatives and extreme values. */ +const anyInt = fc.oneof( + fc.integer(), + fc.constant(0), + fc.constant(-1), + fc.constant(Number.MAX_SAFE_INTEGER), + fc.constant(Number.MIN_SAFE_INTEGER), +) + +/** Arbitrary for playback ranges with adversarial endpoints. */ +const adversarialRange = fc.oneof( + fc.tuple(anyInt, anyInt).map(([a, b]): PlaybackRange => [a, b]), + fc.constant(null), +) + +/** Fps values including zero, negative, NaN, Infinity. */ +const adversarialFps = fc.oneof( + fc.constant(0), + fc.constant(-1), + fc.constant(NaN), + fc.constant(Infinity), + fc.constant(-Infinity), + fc.constant(0.001), + fc.constant(1e15), + fc.double({ min: 0.001, max: 1000, noNaN: true }), +) + +// --- Tests --- + +describe('resolvePlaybackRange fuzz', () => { + it('never throws on adversarial totalFrames and range', () => { + fc.assert( + fc.property(anyInt, adversarialRange, (totalFrames, range) => { + expect(() => resolvePlaybackRange(totalFrames, range)).not.toThrow() + }), + ) + }) + + it('always returns a two-element array', () => { + fc.assert( + fc.property(anyInt, adversarialRange, (totalFrames, range) => { + const result = resolvePlaybackRange(totalFrames, range) + expect(result).toHaveLength(2) + expect(typeof result[0]).toBe('number') + expect(typeof result[1]).toBe('number') + }), + ) + }) +}) + +describe('clampFrameToPlaybackRange fuzz', () => { + it('never throws on adversarial inputs', () => { + fc.assert( + fc.property(anyInt, anyInt, adversarialRange, (frame, totalFrames, range) => { + expect(() => clampFrameToPlaybackRange(frame, totalFrames, range)).not.toThrow() + }), + ) + }) + + it('always returns a number', () => { + fc.assert( + fc.property(anyInt, anyInt, adversarialRange, (frame, totalFrames, range) => { + expect(typeof clampFrameToPlaybackRange(frame, totalFrames, range)).toBe('number') + }), + ) + }) +}) + +describe('resolvePlaybackTick fuzz', () => { + it('never throws on adversarial inputs', () => { + fc.assert( + fc.property( + anyInt, + anyInt, + adversarialRange, + fc.boolean(), + (frame, totalFrames, range, autoLoop) => { + expect(() => resolvePlaybackTick(frame, totalFrames, range, autoLoop)).not.toThrow() + }, + ), + ) + }) + + it('always returns an object with frame and shouldStop', () => { + fc.assert( + fc.property( + anyInt, + anyInt, + adversarialRange, + fc.boolean(), + (frame, totalFrames, range, autoLoop) => { + const result = resolvePlaybackTick(frame, totalFrames, range, autoLoop) + expect(typeof result.frame).toBe('number') + expect(typeof result.shouldStop).toBe('boolean') + }, + ), + ) + }) +}) + +describe('shouldRestartPlaybackAfterLoop fuzz', () => { + it('never throws on adversarial inputs', () => { + fc.assert( + fc.property( + adversarialNumber, + adversarialNumber, + adversarialRange, + fc.boolean(), + (reportedFrame, resolvedFrame, range, autoLoop) => { + expect(() => + shouldRestartPlaybackAfterLoop(reportedFrame, resolvedFrame, range, autoLoop), + ).not.toThrow() + }, + ), + ) + }) + + it('always returns a boolean', () => { + fc.assert( + fc.property( + adversarialNumber, + adversarialNumber, + adversarialRange, + fc.boolean(), + (reportedFrame, resolvedFrame, range, autoLoop) => { + expect( + typeof shouldRestartPlaybackAfterLoop(reportedFrame, resolvedFrame, range, autoLoop), + ).toBe('boolean') + }, + ), + ) + }) + + it('returns false when autoLoop is false regardless of inputs', () => { + fc.assert( + fc.property( + adversarialNumber, + adversarialNumber, + adversarialRange, + (reportedFrame, resolvedFrame, range) => { + expect(shouldRestartPlaybackAfterLoop(reportedFrame, resolvedFrame, range, false)).toBe( + false, + ) + }, + ), + ) + }) + + it('returns false when range is null regardless of inputs', () => { + fc.assert( + fc.property( + adversarialNumber, + adversarialNumber, + fc.boolean(), + (reportedFrame, resolvedFrame, autoLoop) => { + expect(shouldRestartPlaybackAfterLoop(reportedFrame, resolvedFrame, null, autoLoop)).toBe( + false, + ) + }, + ), + ) + }) +}) + +describe('shouldLoopActivePlaybackRange fuzz', () => { + it('equals autoLoop for any range', () => { + fc.assert( + fc.property(adversarialRange, fc.boolean(), (range, autoLoop) => { + expect(shouldLoopActivePlaybackRange(range, autoLoop)).toBe(autoLoop) + }), + ) + }) +}) + +describe('shouldRecoverPlaybackAfterDesync fuzz', () => { + it('never throws on adversarial timing values', () => { + fc.assert( + fc.property( + fc.boolean(), + fc.boolean(), + adversarialNumber, + adversarialNumber, + (isPlaying, videoPaused, elapsed, cooldown) => { + expect(() => + shouldRecoverPlaybackAfterDesync(isPlaying, videoPaused, elapsed, cooldown), + ).not.toThrow() + }, + ), + ) + }) + + it('always returns a boolean', () => { + fc.assert( + fc.property( + fc.boolean(), + fc.boolean(), + adversarialNumber, + adversarialNumber, + (isPlaying, videoPaused, elapsed, cooldown) => { + expect( + typeof shouldRecoverPlaybackAfterDesync(isPlaying, videoPaused, elapsed, cooldown), + ).toBe('boolean') + }, + ), + ) + }) + + it('returns false when not playing', () => { + fc.assert( + fc.property( + fc.boolean(), + adversarialNumber, + adversarialNumber, + (videoPaused, elapsed, cooldown) => { + expect(shouldRecoverPlaybackAfterDesync(false, videoPaused, elapsed, cooldown)).toBe( + false, + ) + }, + ), + ) + }) + + it('returns false when video is not paused', () => { + fc.assert( + fc.property(adversarialNumber, adversarialNumber, (elapsed, cooldown) => { + expect(shouldRecoverPlaybackAfterDesync(true, false, elapsed, cooldown)).toBe(false) + }), + ) + }) +}) + +describe('shouldRecoverStalledPlayback fuzz', () => { + it('never throws on adversarial inputs', () => { + fc.assert( + fc.property( + fc.boolean(), + fc.boolean(), + adversarialNumber, + adversarialNumber, + adversarialNumber, + adversarialNumber, + (isPlaying, videoPaused, videoCurrentTime, lastAdvTime, elapsedMs, stallMs) => { + expect(() => + shouldRecoverStalledPlayback( + isPlaying, + videoPaused, + videoCurrentTime, + lastAdvTime, + elapsedMs, + stallMs, + ), + ).not.toThrow() + }, + ), + ) + }) + + it('returns false when not playing', () => { + fc.assert( + fc.property( + fc.boolean(), + adversarialNumber, + adversarialNumber, + adversarialNumber, + adversarialNumber, + (videoPaused, videoCurrentTime, lastAdvTime, elapsedMs, stallMs) => { + expect( + shouldRecoverStalledPlayback( + false, + videoPaused, + videoCurrentTime, + lastAdvTime, + elapsedMs, + stallMs, + ), + ).toBe(false) + }, + ), + ) + }) + + it('returns false when video is paused', () => { + fc.assert( + fc.property( + adversarialNumber, + adversarialNumber, + adversarialNumber, + adversarialNumber, + (videoCurrentTime, lastAdvTime, elapsedMs, stallMs) => { + expect( + shouldRecoverStalledPlayback( + true, + true, + videoCurrentTime, + lastAdvTime, + elapsedMs, + stallMs, + ), + ).toBe(false) + }, + ), + ) + }) + + it('returns false when current time differs from last advancing time', () => { + fc.assert( + fc.property( + fc.double({ min: 0, max: 1000, noNaN: true }), + fc.double({ min: 0, max: 1000, noNaN: true }), + adversarialNumber, + adversarialNumber, + (a, b, elapsedMs, stallMs) => { + fc.pre(a !== b) + expect(shouldRecoverStalledPlayback(true, false, a, b, elapsedMs, stallMs)).toBe(false) + }, + ), + ) + }) +}) + +describe('computeEffectiveFps fuzz', () => { + it('never throws on adversarial inputs', () => { + fc.assert( + fc.property( + adversarialNumber, + adversarialNumber, + adversarialNumber, + (totalFrames, duration, fps) => { + expect(() => computeEffectiveFps(totalFrames, duration, fps)).not.toThrow() + }, + ), + ) + }) + + it('always returns a number', () => { + fc.assert( + fc.property( + adversarialNumber, + adversarialNumber, + adversarialNumber, + (totalFrames, duration, fps) => { + expect(typeof computeEffectiveFps(totalFrames, duration, fps)).toBe('number') + }, + ), + ) + }) +}) + +describe('computeSyncAction fuzz', () => { + it('never throws on adversarial inputs', () => { + fc.assert( + fc.property( + fc.boolean(), + adversarialNumber, + anyInt, + anyInt, + fc.option(anyInt, { nil: null }), + adversarialFps, + adversarialNumber, + (isPlaying, speed, currentFrame, totalFrames, originalFrame, fps, videoTime) => { + expect(() => + computeSyncAction( + isPlaying, + speed, + currentFrame, + totalFrames, + originalFrame, + fps, + videoTime, + ), + ).not.toThrow() + }, + ), + ) + }) + + it('returns pause when not playing', () => { + fc.assert( + fc.property( + adversarialNumber, + anyInt, + anyInt, + fc.option(anyInt, { nil: null }), + adversarialFps, + adversarialNumber, + (speed, currentFrame, totalFrames, originalFrame, fps, videoTime) => { + const result = computeSyncAction( + false, + speed, + currentFrame, + totalFrames, + originalFrame, + fps, + videoTime, + ) + expect(result.kind).toBe('pause') + }, + ), + ) + }) + + it('action kind is always a valid variant', () => { + fc.assert( + fc.property( + fc.boolean(), + adversarialNumber, + anyInt, + anyInt, + fc.option(anyInt, { nil: null }), + adversarialFps, + adversarialNumber, + (isPlaying, speed, currentFrame, totalFrames, originalFrame, fps, videoTime) => { + const result = computeSyncAction( + isPlaying, + speed, + currentFrame, + totalFrames, + originalFrame, + fps, + videoTime, + ) + expect(['restart', 'seek-and-play', 'play', 'pause']).toContain(result.kind) + }, + ), + ) + }) + + it('playbackRate is set on non-pause actions', () => { + fc.assert( + fc.property( + adversarialNumber, + anyInt, + anyInt, + fc.option(anyInt, { nil: null }), + adversarialFps, + adversarialNumber, + (speed, currentFrame, totalFrames, originalFrame, fps, videoTime) => { + const result = computeSyncAction( + true, + speed, + currentFrame, + totalFrames, + originalFrame, + fps, + videoTime, + ) + if (result.kind !== 'pause') { + expect(typeof result.playbackRate).toBe('number') + } + }, + ), + ) + }) + + it('seek-and-play always includes a seekTo number', () => { + fc.assert( + fc.property( + adversarialNumber, + anyInt, + anyInt, + fc.option(anyInt, { nil: null }), + adversarialFps, + adversarialNumber, + (speed, currentFrame, totalFrames, originalFrame, fps, videoTime) => { + const result = computeSyncAction( + true, + speed, + currentFrame, + totalFrames, + originalFrame, + fps, + videoTime, + ) + if (result.kind === 'seek-and-play') { + expect(typeof result.seekTo).toBe('number') + } + }, + ), + ) + }) +}) + +describe('computePlaybackTarget fuzz', () => { + it('never throws on adversarial inputs', () => { + fc.assert( + fc.property( + anyInt, + anyInt, + fc.option(anyInt, { nil: null }), + adversarialFps, + (currentFrame, totalFrames, originalFrame, fps) => { + expect(() => + computePlaybackTarget(currentFrame, totalFrames, originalFrame, fps), + ).not.toThrow() + }, + ), + ) + }) + + it('always returns targetTime number and shouldRestart boolean', () => { + fc.assert( + fc.property( + anyInt, + anyInt, + fc.option(anyInt, { nil: null }), + adversarialFps, + (currentFrame, totalFrames, originalFrame, fps) => { + const result = computePlaybackTarget(currentFrame, totalFrames, originalFrame, fps) + expect(typeof result.targetTime).toBe('number') + expect(typeof result.shouldRestart).toBe('boolean') + }, + ), + ) + }) +}) + +describe('needsSeekBeforePlay fuzz', () => { + it('never throws on adversarial inputs', () => { + fc.assert( + fc.property(adversarialNumber, adversarialNumber, adversarialFps, (a, b, fps) => { + expect(() => needsSeekBeforePlay(a, b, fps)).not.toThrow() + }), + ) + }) + + it('always returns a boolean', () => { + fc.assert( + fc.property(adversarialNumber, adversarialNumber, adversarialFps, (a, b, fps) => { + expect(typeof needsSeekBeforePlay(a, b, fps)).toBe('boolean') + }), + ) + }) +}) diff --git a/data-management/viewer/frontend/src/lib/__tests__/playback-utils.property.test.ts b/data-management/viewer/frontend/src/lib/__tests__/playback-utils.property.test.ts new file mode 100644 index 00000000..88464d3f --- /dev/null +++ b/data-management/viewer/frontend/src/lib/__tests__/playback-utils.property.test.ts @@ -0,0 +1,198 @@ +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' + +import { + clampFrameToPlaybackRange, + computeEffectiveFps, + computePlaybackTarget, + needsSeekBeforePlay, + type PlaybackRange, + resolvePlaybackRange, + resolvePlaybackTick, +} from '../playback-utils' + +const positiveInt = fc.integer({ min: 1, max: 100_000 }) +const nonNegativeInt = fc.integer({ min: 0, max: 100_000 }) +const playbackRange = fc + .tuple(nonNegativeInt, nonNegativeInt) + .map(([a, b]): PlaybackRange => [a, b]) + +describe('resolvePlaybackRange', () => { + it('always produces start <= end', () => { + fc.assert( + fc.property(positiveInt, fc.option(playbackRange, { nil: null }), (totalFrames, range) => { + const [start, end] = resolvePlaybackRange(totalFrames, range) + expect(start).toBeLessThanOrEqual(end) + }), + ) + }) + + it('bounds output within [0, totalFrames-1]', () => { + fc.assert( + fc.property(positiveInt, fc.option(playbackRange, { nil: null }), (totalFrames, range) => { + const [start, end] = resolvePlaybackRange(totalFrames, range) + expect(start).toBeGreaterThanOrEqual(0) + expect(end).toBeLessThanOrEqual(Math.max(totalFrames - 1, 0)) + }), + ) + }) + + it('returns full range when range is null', () => { + fc.assert( + fc.property(positiveInt, (totalFrames) => { + const [start, end] = resolvePlaybackRange(totalFrames, null) + expect(start).toBe(0) + expect(end).toBe(totalFrames - 1) + }), + ) + }) +}) + +describe('clampFrameToPlaybackRange', () => { + it('result is always within the resolved range', () => { + fc.assert( + fc.property( + nonNegativeInt, + positiveInt, + fc.option(playbackRange, { nil: null }), + (frame, totalFrames, range) => { + const clamped = clampFrameToPlaybackRange(frame, totalFrames, range) + const [start, end] = resolvePlaybackRange(totalFrames, range) + expect(clamped).toBeGreaterThanOrEqual(start) + expect(clamped).toBeLessThanOrEqual(end) + }, + ), + ) + }) + + it('is idempotent', () => { + fc.assert( + fc.property( + nonNegativeInt, + positiveInt, + fc.option(playbackRange, { nil: null }), + (frame, totalFrames, range) => { + const once = clampFrameToPlaybackRange(frame, totalFrames, range) + const twice = clampFrameToPlaybackRange(once, totalFrames, range) + expect(twice).toBe(once) + }, + ), + ) + }) +}) + +describe('resolvePlaybackTick', () => { + it('frame is always within the resolved range', () => { + fc.assert( + fc.property( + nonNegativeInt, + positiveInt, + fc.option(playbackRange, { nil: null }), + fc.boolean(), + (frame, totalFrames, range, autoLoop) => { + const result = resolvePlaybackTick(frame, totalFrames, range, autoLoop) + const [start, end] = resolvePlaybackRange(totalFrames, range) + expect(result.frame).toBeGreaterThanOrEqual(start) + expect(result.frame).toBeLessThanOrEqual(end) + }, + ), + ) + }) + + it('never stops when frame is within range', () => { + fc.assert( + fc.property(positiveInt, fc.boolean(), (totalFrames, autoLoop) => { + const [start, end] = resolvePlaybackRange(totalFrames, null) + const mid = Math.floor((start + end) / 2) + const result = resolvePlaybackTick(mid, totalFrames, null, autoLoop) + expect(result.shouldStop).toBe(false) + }), + ) + }) +}) + +describe('computeEffectiveFps', () => { + it('returns positive fps for valid inputs', () => { + fc.assert( + fc.property( + positiveInt, + fc.double({ min: 0.01, max: 10_000, noNaN: true }), + fc.double({ min: 0.01, max: 1000, noNaN: true }), + (totalFrames, videoDuration, datasetFps) => { + const fps = computeEffectiveFps(totalFrames, videoDuration, datasetFps) + expect(fps).toBeGreaterThan(0) + }, + ), + ) + }) + + it('equals totalFrames / videoDuration when both are positive', () => { + fc.assert( + fc.property( + positiveInt, + fc.double({ min: 0.01, max: 10_000, noNaN: true }), + fc.double({ min: 0.01, max: 1000, noNaN: true }), + (totalFrames, videoDuration, datasetFps) => { + const fps = computeEffectiveFps(totalFrames, videoDuration, datasetFps) + expect(fps).toBeCloseTo(totalFrames / videoDuration, 5) + }, + ), + ) + }) +}) + +describe('computePlaybackTarget', () => { + it('targetTime is non-negative', () => { + fc.assert( + fc.property( + nonNegativeInt, + positiveInt, + fc.double({ min: 1, max: 1000, noNaN: true }), + (currentFrame, totalFrames, fps) => { + const result = computePlaybackTarget(currentFrame, totalFrames, null, fps) + expect(result.targetTime).toBeGreaterThanOrEqual(0) + }, + ), + ) + }) + + it('shouldRestart when at or past the range end', () => { + fc.assert( + fc.property( + positiveInt, + fc.double({ min: 1, max: 1000, noNaN: true }), + (totalFrames, fps) => { + const result = computePlaybackTarget(totalFrames - 1, totalFrames, null, fps) + expect(result.shouldRestart).toBe(true) + }, + ), + ) + }) +}) + +describe('needsSeekBeforePlay', () => { + it('returns false when times are equal', () => { + fc.assert( + fc.property( + fc.double({ min: 0, max: 10_000, noNaN: true }), + fc.double({ min: 1, max: 1000, noNaN: true }), + (time, fps) => { + expect(needsSeekBeforePlay(time, time, fps)).toBe(false) + }, + ), + ) + }) + + it('is symmetric in time difference direction', () => { + fc.assert( + fc.property( + fc.double({ min: 0, max: 10_000, noNaN: true }), + fc.double({ min: 0, max: 10_000, noNaN: true }), + fc.double({ min: 1, max: 1000, noNaN: true }), + (a, b, fps) => { + expect(needsSeekBeforePlay(a, b, fps)).toBe(needsSeekBeforePlay(b, a, fps)) + }, + ), + ) + }) +}) diff --git a/data-management/viewer/frontend/src/lib/__tests__/trajectory-graph-geometry.property.test.ts b/data-management/viewer/frontend/src/lib/__tests__/trajectory-graph-geometry.property.test.ts new file mode 100644 index 00000000..331f5683 --- /dev/null +++ b/data-management/viewer/frontend/src/lib/__tests__/trajectory-graph-geometry.property.test.ts @@ -0,0 +1,110 @@ +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' + +import type { TrajectoryPlotArea } from '../trajectory-graph-geometry' +import { resolveSelectionHighlightStyle, resolveSurfaceFrame } from '../trajectory-graph-geometry' + +const plotArea: fc.Arbitrary = fc.record({ + left: fc.double({ min: 0, max: 1000, noNaN: true }), + width: fc.double({ min: 1, max: 1000, noNaN: true }), +}) + +describe('resolveSurfaceFrame', () => { + it('output is always in [0, totalFrames-1] for valid inputs', () => { + fc.assert( + fc.property( + fc.double({ min: -500, max: 2000, noNaN: true }), + fc.integer({ min: 2, max: 10_000 }), + plotArea, + (surfaceX, totalFrames, area) => { + const frame = resolveSurfaceFrame(surfaceX, totalFrames, area) + expect(frame).toBeGreaterThanOrEqual(0) + expect(frame).toBeLessThanOrEqual(totalFrames - 1) + }, + ), + ) + }) + + it('returns 0 when plotArea is null', () => { + fc.assert( + fc.property( + fc.double({ min: -500, max: 2000, noNaN: true }), + fc.integer({ min: 1, max: 10_000 }), + (surfaceX, totalFrames) => { + expect(resolveSurfaceFrame(surfaceX, totalFrames, null)).toBe(0) + }, + ), + ) + }) + + it('is monotonically non-decreasing with surfaceX', () => { + fc.assert( + fc.property( + fc.double({ min: 0, max: 1000, noNaN: true }), + fc.double({ min: 0, max: 100, noNaN: true }), + fc.integer({ min: 2, max: 10_000 }), + plotArea, + (x1, delta, totalFrames, area) => { + const result1 = resolveSurfaceFrame(x1, totalFrames, area) + const result2 = resolveSurfaceFrame(x1 + delta, totalFrames, area) + expect(result2).toBeGreaterThanOrEqual(result1) + }, + ), + ) + }) +}) + +describe('resolveSelectionHighlightStyle', () => { + it('returns non-null with positive width for valid ranges', () => { + fc.assert( + fc.property( + fc.integer({ min: 0, max: 999 }), + fc.integer({ min: 0, max: 999 }), + fc.integer({ min: 2, max: 1000 }), + plotArea, + (a, b, totalFrames, area) => { + fc.pre(a !== b && a < totalFrames && b < totalFrames) + const result = resolveSelectionHighlightStyle([a, b], totalFrames, area) + if (result) { + expect(result.width).toBeGreaterThan(0) + } + }, + ), + ) + }) + + it('left is within plot area bounds', () => { + fc.assert( + fc.property( + fc.integer({ min: 0, max: 999 }), + fc.integer({ min: 0, max: 999 }), + fc.integer({ min: 2, max: 1000 }), + plotArea, + (a, b, totalFrames, area) => { + fc.pre(a !== b && a < totalFrames && b < totalFrames) + const result = resolveSelectionHighlightStyle([a, b], totalFrames, area) + if (result) { + expect(result.left).toBeGreaterThanOrEqual(area.left) + } + }, + ), + ) + }) + + it('produces identical results regardless of range order', () => { + fc.assert( + fc.property( + fc.integer({ min: 0, max: 999 }), + fc.integer({ min: 0, max: 999 }), + fc.integer({ min: 2, max: 1000 }), + plotArea, + (a, b, totalFrames, area) => { + fc.pre(a !== b && a < totalFrames && b < totalFrames) + const forward = resolveSelectionHighlightStyle([a, b], totalFrames, area) + const reversed = resolveSelectionHighlightStyle([b, a], totalFrames, area) + expect(forward).toEqual(reversed) + }, + ), + ) + }) +}) diff --git a/data-management/viewer/frontend/src/lib/api-client.ts b/data-management/viewer/frontend/src/lib/api-client.ts index 2958c1dd..978a0653 100644 --- a/data-management/viewer/frontend/src/lib/api-client.ts +++ b/data-management/viewer/frontend/src/lib/api-client.ts @@ -65,11 +65,11 @@ export function _resetCsrfToken(): void { /** * Convert snake_case keys to camelCase recursively. */ -function snakeToCamel(str: string): string { +export function snakeToCamel(str: string): string { return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()) } -function transformKeys(obj: unknown): T { +export function transformKeys(obj: unknown): T { if (Array.isArray(obj)) { return obj.map(transformKeys) as T } diff --git a/data-management/viewer/frontend/vitest.config.ts b/data-management/viewer/frontend/vitest.config.ts index ecf83194..55fee670 100644 --- a/data-management/viewer/frontend/vitest.config.ts +++ b/data-management/viewer/frontend/vitest.config.ts @@ -14,6 +14,10 @@ export default defineConfig({ globals: true, setupFiles: ['./src/test/setup.ts'], include: ['src/**/*.{test,spec}.{ts,tsx}'], + reporters: ['default', 'junit'], + outputFile: { + junit: '../../../logs/vitest-results.xml', + }, coverage: { provider: 'v8', reporter: ['text', 'lcov', 'cobertura', 'json-summary'], diff --git a/docs/contributing/README.md b/docs/contributing/README.md index f64e7bfb..52d77884 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -42,6 +42,7 @@ Contributions can include: | [Accessibility](accessibility.md) | Accessibility scope, documentation and CLI output guidelines | | [Updating External Components](component-updates.md) | Process for updating reused externally-maintained components | | [Documentation Maintenance](documentation-maintenance.md) | Update triggers, ownership, review criteria, freshness policy | +| [Fuzzing and Property-Based Testing](fuzzing.md) | Fuzz targets, property tests, Hypothesis and fast-check patterns | | [Roadmap](ROADMAP.md) | 12-month project roadmap, priorities, and success metrics | ### Quick Reference diff --git a/docs/contributing/fuzzing.md b/docs/contributing/fuzzing.md new file mode 100644 index 00000000..46b4e176 --- /dev/null +++ b/docs/contributing/fuzzing.md @@ -0,0 +1,210 @@ +--- +sidebar_position: 12 +title: Fuzzing and Property-Based Testing +description: Running fuzz targets and property-based tests for Python and TypeScript code +author: Microsoft Robotics-AI Team +ms.date: 2026-07-18 +ms.topic: how-to +keywords: + - fuzzing + - property-based testing + - atheris + - hypothesis + - fast-check + - security + - testing +--- + +This repository uses fuzz testing and property-based testing to find edge cases in input validation, data transformation, and serialization code. Python targets run under Atheris (coverage-guided fuzzing) and Hypothesis (property-based testing). TypeScript targets use fast-check for property-based testing. + +## Architecture + +| Layer | Framework | Scope | +|---|---|---| +| Coverage-guided fuzzing | [Atheris](https://github.com/google/atheris) | Python functions handling untrusted input | +| Python property tests | [Hypothesis](https://hypothesis.readthedocs.io/) | Deterministic pytest classes in the fuzz harness | +| TypeScript property tests | [fast-check](https://fast-check.dev/) | Pure utility functions in the dataviewer frontend | + +Python fuzz regression tests run in a dedicated CI workflow that uploads coverage under the `pytest-fuzz` Codecov flag. TypeScript property tests run through the existing vitest workflow and merge into the `vitest` flag. + +## Python Fuzz Harness + +The fuzz harness at `tests/fuzz_harness.py` operates in dual mode: + +| Mode | Trigger | Behavior | +|---|---|---| +| Pytest | `uv run pytest` | Deterministic test classes exercise targets with controlled inputs | +| Atheris | `python tests/fuzz_harness.py` | Coverage-guided fuzzing with randomized byte streams | + +### Running Pytest Mode + +```bash +uv sync --group dev +uv run pytest tests/fuzz_harness.py -v +``` + +All fuzz targets produce deterministic test classes prefixed with `Test*`. These run as part of the fuzz regression workflow and contribute to the `pytest-fuzz` Codecov flag. + +### Running Atheris Mode + +Atheris requires a separate install because it depends on native libFuzzer bindings: + +```bash +uv sync --group dev --group fuzz +uv run python tests/fuzz_harness.py +``` + +Atheris mode dispatches randomized bytes to all registered fuzz targets. Crash artifacts are written to `logs/fuzz-crashes/`. The harness creates this directory automatically. + +### Seed Corpus + +The harness auto-includes `tests/fuzz-corpus/` when the directory exists. Seed files give the fuzzer meaningful starting points so it reaches deep code paths faster than random byte generation alone. + +Each seed file is a raw binary blob whose first byte selects the target via `data[0] % 9`, and remaining bytes feed `FuzzedDataProvider`. + +#### Generating Seeds + +```bash +python3 tests/generate_fuzz_corpus.py +``` + +This creates 48 seed files covering all 9 targets with valid inputs, boundary values, and attack patterns (path traversal, null bytes, CRLF injection, NaN/Inf floats). + +#### Seed Organization + +| Prefix | Target | +|---|---| +| `t0_` | `fuzz_validate_blob_path` | +| `t1_` | `fuzz_get_validation_error` | +| `t2_` | `fuzz_extract_from_value` | +| `t3_` | `fuzz_extract_from_tracking_data` | +| `t4_` | `fuzz_sanitize_user_string` | +| `t5_` | `fuzz_sanitize_nested_value` | +| `t6_` | `fuzz_validate_safe_string` | +| `t7_` | `fuzz_dataset_id_to_blob_prefix` | +| `t8_` | `fuzz_datetime_encoder` | + +When adding a new fuzz target, add corresponding seeds in `generate_fuzz_corpus.py` and re-run the generator. + +### Current Targets + +| Target | Module | Function | +|---|---|---| +| Blob path validation | `data-management/tools/blob_path_validator.py` | `validate_blob_path`, `get_validation_error` | +| Metrics extraction | `training/utils/metrics.py` | `_extract_from_value`, `_extract_from_tracking_data` | +| Input sanitization | `data-management/viewer/backend/src/api/validation.py` | `sanitize_user_string`, `_sanitize_nested_value`, `validate_safe_string` | +| Storage paths | `data-management/viewer/backend/src/api/storage/paths.py` | `dataset_id_to_blob_prefix` | +| JSON serialization | `data-management/viewer/backend/src/api/storage/serializers.py` | `DateTimeEncoder` | + +### Adding a Fuzz Target + +1. Add a fuzz function following the `fuzz_*` naming convention: + +```python +def fuzz_my_function(data: bytes) -> None: + fdp = atheris.FuzzedDataProvider(data) + value = fdp.ConsumeUnicodeNoSurrogates(256) + with suppress(ValueError): + my_function(value) +``` + +1. Register it in the `_FUZZ_TARGETS` list at the bottom of the harness. + +1. Add a corresponding `Test*` class with deterministic edge-case inputs: + +```python +class TestMyFunction: + def test_empty_input(self) -> None: + assert my_function("") == expected + + def test_boundary_case(self) -> None: + assert my_function(boundary_value) == expected +``` + +1. Run the tests to confirm both modes work: + +```bash +uv run pytest tests/fuzz_harness.py -v +``` + +## TypeScript Property Tests + +Property-based tests for the dataviewer frontend use fast-check with Vitest. Test files follow the `*.property.test.ts` naming convention. + +### Running Property Tests + +```bash +cd data-management/viewer/frontend +npx vitest run --reporter=verbose +``` + +Property tests run as part of the standard vitest suite and contribute to the `vitest` Codecov flag. + +### Current Test Files + +| File | Module Under Test | +|---|---| +| `src/lib/__tests__/api-client.property.test.ts` | `snakeToCamel`, `transformKeys` | +| `src/lib/__tests__/api-client-fuzz.test.ts` | `snakeToCamel`, `transformKeys` (adversarial Unicode, deep nesting) | +| `src/lib/__tests__/playback-utils.property.test.ts` | Playback range resolution, frame clamping, FPS computation | +| `src/lib/__tests__/edit-store-frame-utils.property.test.ts` | Frame index conversion with insertions and removals | +| `src/lib/__tests__/trajectory-graph-geometry.property.test.ts` | Coordinate math for trajectory visualization | + +### Writing a Property Test + +Target pure functions with well-defined input/output contracts. Use arbitraries that match the function's domain: + +```typescript +import fc from 'fast-check' + +describe('myFunction', () => { + it('satisfies some invariant', () => { + fc.assert( + fc.property(fc.integer({ min: 0, max: 1000 }), (input) => { + const result = myFunction(input) + expect(result).toBeGreaterThanOrEqual(0) + }), + ) + }) +}) +``` + +Prefer these property patterns: + +| Pattern | Description | +|---|---| +| Invariant | Output always satisfies a constraint | +| Idempotence | Applying the function twice gives the same result | +| Roundtrip | Encode then decode returns the original value | +| Monotonicity | Larger input produces larger or equal output | +| Bounds | Output stays within a known range | + +## Hypothesis Configuration + +Global Hypothesis settings live in `pyproject.toml`: + +```toml +[tool.hypothesis] +max_examples = 50 +deadline = 500 +``` + +These settings apply to all Hypothesis-based tests. `max_examples` controls the number of random inputs per test case. `deadline` sets the per-example timeout in milliseconds. + +## Coverage Integration + +Fuzz and property test coverage merges into existing Codecov flags: + +| Test type | Codecov flag | Coverage file | +|---|---|---| +| Python fuzz harness | `pytest-fuzz` | `logs/coverage-fuzz.xml` | +| Dataviewer backend | `pytest-dataviewer` | `logs/coverage-dataviewer.xml` | +| TypeScript property tests | `vitest` | `coverage/cobertura-coverage.xml` | + +Per-flag patch coverage status is set to `informational: true` so fuzz coverage differences never block PRs. This follows the pattern used in [microsoft/hve-core](https://github.com/microsoft/hve-core). + +## Related Documentation + +- [Security Review](security-review.md) for security testing requirements +- [Prerequisites](prerequisites.md) for required tool versions +- [Deployment Validation](deployment-validation.md) for validation levels diff --git a/pyproject.toml b/pyproject.toml index e2473966..43c09bcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dev = [ "azureml-mlflow==1.62.0.post2", "azure-ai-ml==1.32.0", ] +fuzz = ["atheris>=3.0"] [tool.uv] package = false @@ -36,6 +37,7 @@ constraint-dependencies = ["pygments==2.20.0"] [tool.pytest.ini_options] testpaths = ["tests", "training/tests", "data-management/tools/tests"] pythonpath = [".", "data-management/tools"] +python_files = ["test_*.py", "fuzz_harness.py"] addopts = [ "-ra", "-m", @@ -45,6 +47,7 @@ addopts = [ "--cov=training", "--cov-report=term-missing", "--cov-report=xml", + "--junitxml=logs/pytest-results.xml", ] markers = [ "e2e: marks tests that submit real GPU training jobs (deselect: -m 'not e2e')", @@ -108,4 +111,4 @@ exclude_lines = [ ] [tool.coverage.xml] -output = "coverage.xml" +output = "logs/coverage.xml" diff --git a/scripts/tests/pester.config.ps1 b/scripts/tests/pester.config.ps1 index c2a75a7e..0448fe6c 100644 --- a/scripts/tests/pester.config.ps1 +++ b/scripts/tests/pester.config.ps1 @@ -49,7 +49,7 @@ $configuration.TestResult.TestSuiteName = 'Robotics-RefArch-PowerShell-Tests' if ($CodeCoverage.IsPresent) { $configuration.CodeCoverage.Enabled = $true $configuration.CodeCoverage.OutputFormat = 'JaCoCo' - $configuration.CodeCoverage.OutputPath = Join-Path $PSScriptRoot '../../logs/coverage.xml' + $configuration.CodeCoverage.OutputPath = Join-Path $PSScriptRoot '../../logs/coverage-pester.xml' # Resolve coverage paths explicitly - Join-Path with wildcards returns literal paths without file system expansion in Pester configuration $ciRoot = Split-Path $PSScriptRoot -Parent diff --git a/tests/fuzz-corpus/t0_empty b/tests/fuzz-corpus/t0_empty new file mode 100644 index 00000000..f76dd238 Binary files /dev/null and b/tests/fuzz-corpus/t0_empty differ diff --git a/tests/fuzz-corpus/t0_traversal b/tests/fuzz-corpus/t0_traversal new file mode 100644 index 00000000..ccf41b66 Binary files /dev/null and b/tests/fuzz-corpus/t0_traversal differ diff --git a/tests/fuzz-corpus/t0_uppercase b/tests/fuzz-corpus/t0_uppercase new file mode 100644 index 00000000..bd64fc17 Binary files /dev/null and b/tests/fuzz-corpus/t0_uppercase differ diff --git a/tests/fuzz-corpus/t0_valid_checkpoints b/tests/fuzz-corpus/t0_valid_checkpoints new file mode 100644 index 00000000..9ff05e27 Binary files /dev/null and b/tests/fuzz-corpus/t0_valid_checkpoints differ diff --git a/tests/fuzz-corpus/t0_valid_converted b/tests/fuzz-corpus/t0_valid_converted new file mode 100644 index 00000000..baa3a668 Binary files /dev/null and b/tests/fuzz-corpus/t0_valid_converted differ diff --git a/tests/fuzz-corpus/t0_valid_raw b/tests/fuzz-corpus/t0_valid_raw new file mode 100644 index 00000000..22ccb248 Binary files /dev/null and b/tests/fuzz-corpus/t0_valid_raw differ diff --git a/tests/fuzz-corpus/t0_valid_reports b/tests/fuzz-corpus/t0_valid_reports new file mode 100644 index 00000000..be42927a Binary files /dev/null and b/tests/fuzz-corpus/t0_valid_reports differ diff --git a/tests/fuzz-corpus/t1_spaces b/tests/fuzz-corpus/t1_spaces new file mode 100644 index 00000000..527e9bc5 --- /dev/null +++ b/tests/fuzz-corpus/t1_spaces @@ -0,0 +1 @@ +raw/robot 01/2026-03-05/ep.mcap \ No newline at end of file diff --git a/tests/fuzz-corpus/t1_special_chars b/tests/fuzz-corpus/t1_special_chars new file mode 100644 index 00000000..a88b0fa9 Binary files /dev/null and b/tests/fuzz-corpus/t1_special_chars differ diff --git a/tests/fuzz-corpus/t1_uppercase b/tests/fuzz-corpus/t1_uppercase new file mode 100644 index 00000000..6380525a --- /dev/null +++ b/tests/fuzz-corpus/t1_uppercase @@ -0,0 +1 @@ +RAW/ROBOT-01/2026/ep.mcap \ No newline at end of file diff --git a/tests/fuzz-corpus/t1_valid b/tests/fuzz-corpus/t1_valid new file mode 100644 index 00000000..2615ee44 --- /dev/null +++ b/tests/fuzz-corpus/t1_valid @@ -0,0 +1 @@ +raw/robot-01/2026-03-05/episode.mcap \ No newline at end of file diff --git a/tests/fuzz-corpus/t2_float b/tests/fuzz-corpus/t2_float new file mode 100644 index 00000000..a96b7327 --- /dev/null +++ b/tests/fuzz-corpus/t2_float @@ -0,0 +1 @@ +loss…ëQ¸ @ \ No newline at end of file diff --git a/tests/fuzz-corpus/t2_inf b/tests/fuzz-corpus/t2_inf new file mode 100644 index 00000000..62b83551 Binary files /dev/null and b/tests/fuzz-corpus/t2_inf differ diff --git a/tests/fuzz-corpus/t2_int b/tests/fuzz-corpus/t2_int new file mode 100644 index 00000000..ab53714f Binary files /dev/null and b/tests/fuzz-corpus/t2_int differ diff --git a/tests/fuzz-corpus/t2_nan b/tests/fuzz-corpus/t2_nan new file mode 100644 index 00000000..a106884e Binary files /dev/null and b/tests/fuzz-corpus/t2_nan differ diff --git a/tests/fuzz-corpus/t2_none b/tests/fuzz-corpus/t2_none new file mode 100644 index 00000000..f54c3089 --- /dev/null +++ b/tests/fuzz-corpus/t2_none @@ -0,0 +1 @@ +reward \ No newline at end of file diff --git a/tests/fuzz-corpus/t2_string b/tests/fuzz-corpus/t2_string new file mode 100644 index 00000000..379254cd --- /dev/null +++ b/tests/fuzz-corpus/t2_string @@ -0,0 +1 @@ +tag_not-a-number \ No newline at end of file diff --git a/tests/fuzz-corpus/t3_deep_nesting b/tests/fuzz-corpus/t3_deep_nesting new file mode 100644 index 00000000..1cf9034a Binary files /dev/null and b/tests/fuzz-corpus/t3_deep_nesting differ diff --git a/tests/fuzz-corpus/t3_empty b/tests/fuzz-corpus/t3_empty new file mode 100644 index 00000000..fc2b5693 --- /dev/null +++ b/tests/fuzz-corpus/t3_empty @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fuzz-corpus/t3_flat b/tests/fuzz-corpus/t3_flat new file mode 100644 index 00000000..b6245d17 Binary files /dev/null and b/tests/fuzz-corpus/t3_flat differ diff --git a/tests/fuzz-corpus/t3_nested b/tests/fuzz-corpus/t3_nested new file mode 100644 index 00000000..6e13f651 Binary files /dev/null and b/tests/fuzz-corpus/t3_nested differ diff --git a/tests/fuzz-corpus/t4_clean b/tests/fuzz-corpus/t4_clean new file mode 100644 index 00000000..67873e0e --- /dev/null +++ b/tests/fuzz-corpus/t4_clean @@ -0,0 +1 @@ +hello-world_123 \ No newline at end of file diff --git a/tests/fuzz-corpus/t4_cr b/tests/fuzz-corpus/t4_cr new file mode 100644 index 00000000..181e3592 --- /dev/null +++ b/tests/fuzz-corpus/t4_cr @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/tests/fuzz-corpus/t4_crlf b/tests/fuzz-corpus/t4_crlf new file mode 100644 index 00000000..e0695fd2 --- /dev/null +++ b/tests/fuzz-corpus/t4_crlf @@ -0,0 +1,2 @@ +hello +world \ No newline at end of file diff --git a/tests/fuzz-corpus/t4_empty b/tests/fuzz-corpus/t4_empty new file mode 100644 index 00000000..45a8ca02 --- /dev/null +++ b/tests/fuzz-corpus/t4_empty @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fuzz-corpus/t4_lf b/tests/fuzz-corpus/t4_lf new file mode 100644 index 00000000..e6b215bf --- /dev/null +++ b/tests/fuzz-corpus/t4_lf @@ -0,0 +1,2 @@ +hello +world \ No newline at end of file diff --git a/tests/fuzz-corpus/t4_long b/tests/fuzz-corpus/t4_long new file mode 100644 index 00000000..7a5b64e5 --- /dev/null +++ b/tests/fuzz-corpus/t4_long @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ No newline at end of file diff --git a/tests/fuzz-corpus/t4_null_byte b/tests/fuzz-corpus/t4_null_byte new file mode 100644 index 00000000..e699ea73 Binary files /dev/null and b/tests/fuzz-corpus/t4_null_byte differ diff --git a/tests/fuzz-corpus/t4_unicode b/tests/fuzz-corpus/t4_unicode new file mode 100644 index 00000000..a448f674 --- /dev/null +++ b/tests/fuzz-corpus/t4_unicode @@ -0,0 +1 @@ +café-über \ No newline at end of file diff --git a/tests/fuzz-corpus/t5_deep b/tests/fuzz-corpus/t5_deep new file mode 100644 index 00000000..21b249c6 Binary files /dev/null and b/tests/fuzz-corpus/t5_deep differ diff --git a/tests/fuzz-corpus/t5_nested_crlf b/tests/fuzz-corpus/t5_nested_crlf new file mode 100644 index 00000000..f9d0d1aa Binary files /dev/null and b/tests/fuzz-corpus/t5_nested_crlf differ diff --git a/tests/fuzz-corpus/t5_numeric b/tests/fuzz-corpus/t5_numeric new file mode 100644 index 00000000..f5d0ba7c Binary files /dev/null and b/tests/fuzz-corpus/t5_numeric differ diff --git a/tests/fuzz-corpus/t5_string b/tests/fuzz-corpus/t5_string new file mode 100644 index 00000000..3c850f14 --- /dev/null +++ b/tests/fuzz-corpus/t5_string @@ -0,0 +1,2 @@ +hello +world \ No newline at end of file diff --git a/tests/fuzz-corpus/t6_long b/tests/fuzz-corpus/t6_long new file mode 100644 index 00000000..53af6dbb --- /dev/null +++ b/tests/fuzz-corpus/t6_long @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ No newline at end of file diff --git a/tests/fuzz-corpus/t6_null_byte b/tests/fuzz-corpus/t6_null_byte new file mode 100644 index 00000000..bdfccbb4 Binary files /dev/null and b/tests/fuzz-corpus/t6_null_byte differ diff --git a/tests/fuzz-corpus/t6_special b/tests/fuzz-corpus/t6_special new file mode 100644 index 00000000..f47bff72 --- /dev/null +++ b/tests/fuzz-corpus/t6_special @@ -0,0 +1 @@ +!@#$%^&*() \ No newline at end of file diff --git a/tests/fuzz-corpus/t6_traversal b/tests/fuzz-corpus/t6_traversal new file mode 100644 index 00000000..08e8c723 --- /dev/null +++ b/tests/fuzz-corpus/t6_traversal @@ -0,0 +1 @@ +../etc/passwd \ No newline at end of file diff --git a/tests/fuzz-corpus/t6_valid_camera b/tests/fuzz-corpus/t6_valid_camera new file mode 100644 index 00000000..c39ccf4b --- /dev/null +++ b/tests/fuzz-corpus/t6_valid_camera @@ -0,0 +1 @@ +front_left.rgb \ No newline at end of file diff --git a/tests/fuzz-corpus/t6_valid_dataset_id b/tests/fuzz-corpus/t6_valid_dataset_id new file mode 100644 index 00000000..18c78006 --- /dev/null +++ b/tests/fuzz-corpus/t6_valid_dataset_id @@ -0,0 +1 @@ +robot-01.data_v2 \ No newline at end of file diff --git a/tests/fuzz-corpus/t7_double_dash b/tests/fuzz-corpus/t7_double_dash new file mode 100644 index 00000000..ca4df58e --- /dev/null +++ b/tests/fuzz-corpus/t7_double_dash @@ -0,0 +1 @@ +group--dataset \ No newline at end of file diff --git a/tests/fuzz-corpus/t7_empty b/tests/fuzz-corpus/t7_empty new file mode 100644 index 00000000..303e398c --- /dev/null +++ b/tests/fuzz-corpus/t7_empty @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fuzz-corpus/t7_multiple_sep b/tests/fuzz-corpus/t7_multiple_sep new file mode 100644 index 00000000..0fe39e72 --- /dev/null +++ b/tests/fuzz-corpus/t7_multiple_sep @@ -0,0 +1 @@ +a--b--c \ No newline at end of file diff --git a/tests/fuzz-corpus/t7_no_separator b/tests/fuzz-corpus/t7_no_separator new file mode 100644 index 00000000..9b4c612b --- /dev/null +++ b/tests/fuzz-corpus/t7_no_separator @@ -0,0 +1 @@ +simple-name \ No newline at end of file diff --git a/tests/fuzz-corpus/t7_triple_dash b/tests/fuzz-corpus/t7_triple_dash new file mode 100644 index 00000000..2808d3ed --- /dev/null +++ b/tests/fuzz-corpus/t7_triple_dash @@ -0,0 +1 @@ +a---b \ No newline at end of file diff --git a/tests/fuzz-corpus/t8_epoch b/tests/fuzz-corpus/t8_epoch new file mode 100644 index 00000000..e66a8626 Binary files /dev/null and b/tests/fuzz-corpus/t8_epoch differ diff --git a/tests/fuzz-corpus/t8_max_year b/tests/fuzz-corpus/t8_max_year new file mode 100644 index 00000000..b3a95408 Binary files /dev/null and b/tests/fuzz-corpus/t8_max_year differ diff --git a/tests/fuzz-corpus/t8_min_year b/tests/fuzz-corpus/t8_min_year new file mode 100644 index 00000000..e050ba67 Binary files /dev/null and b/tests/fuzz-corpus/t8_min_year differ diff --git a/tests/fuzz-corpus/t8_normal b/tests/fuzz-corpus/t8_normal new file mode 100644 index 00000000..34c3fa96 Binary files /dev/null and b/tests/fuzz-corpus/t8_normal differ diff --git a/tests/fuzz_harness.py b/tests/fuzz_harness.py new file mode 100644 index 00000000..89416196 --- /dev/null +++ b/tests/fuzz_harness.py @@ -0,0 +1,440 @@ +"""Polyglot fuzz harness — runs as pytest test AND Atheris coverage-guided fuzzer. + +Satisfies the OpenSSF Scorecard Fuzzing check (Phase 3) which detects +``import atheris`` in any .py file in the repository. + +When executed by pytest, deterministic test classes exercise the same +functions with controlled inputs. When executed directly with atheris +installed, ``fuzz_dispatch`` routes randomized bytes to all registered +targets via ``FuzzedDataProvider``. + +Targets: + - ``validate_blob_path`` / ``get_validation_error`` — blob path regex validation + - ``_extract_from_value`` / ``_extract_from_tracking_data`` — metrics type dispatch + - ``sanitize_user_string`` / ``_sanitize_nested_value`` — dataviewer input sanitization + - ``validate_safe_string`` — dataviewer string validation with pattern matching + - ``dataset_id_to_blob_prefix`` — dataset ID to storage path conversion + - ``DateTimeEncoder`` — JSON serialization of datetime objects +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from contextlib import suppress +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +try: + import atheris + + FUZZING = True +except ImportError: + FUZZING = False + +from blob_path_validator import get_validation_error, validate_blob_path + +_REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _load_module(name: str, relative_path: str) -> Any: + """Load a source module by file path, bypassing the package tree.""" + full_path = _REPO_ROOT / relative_path + spec = importlib.util.spec_from_file_location(name, full_path) + if spec is None or spec.loader is None: + msg = f"Unable to load module {name!r} from {full_path}" + raise RuntimeError(msg) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_metrics = _load_module("metrics_fuzz", "training/utils/metrics.py") +_extract_from_value = _metrics._extract_from_value +_extract_from_tracking_data = _metrics._extract_from_tracking_data + +_DATA_TYPES = ["raw", "converted", "reports", "checkpoints"] + +# Dataviewer backend modules loaded via importlib to avoid full FastAPI import chain. +# We import only the pure functions that don't require the FastAPI app context. +_validation = _load_module("validation_fuzz", "data-management/viewer/backend/src/api/validation.py") +_sanitize_user_string = _validation.sanitize_user_string +_sanitize_nested_value = _validation._sanitize_nested_value +_validate_safe_string = _validation.validate_safe_string +_SAFE_DATASET_ID_PATTERN = _validation.SAFE_DATASET_ID_PATTERN +_SAFE_CAMERA_NAME_PATTERN = _validation.SAFE_CAMERA_NAME_PATTERN + +_paths = _load_module("paths_fuzz", "data-management/viewer/backend/src/api/storage/paths.py") +_dataset_id_to_blob_prefix = _paths.dataset_id_to_blob_prefix + +_serializers = _load_module("serializers_fuzz", "data-management/viewer/backend/src/api/storage/serializers.py") +_DateTimeEncoder = _serializers.DateTimeEncoder + + +# ================================================================ +# Fuzz functions (Atheris mode only — never called during pytest) +# ================================================================ + + +def fuzz_validate_blob_path(data: bytes) -> None: + fdp = atheris.FuzzedDataProvider(data) + path = fdp.ConsumeUnicodeNoSurrogates(256) + idx = fdp.ConsumeIntInRange(0, len(_DATA_TYPES) - 1) + with suppress(ValueError): + validate_blob_path(path, _DATA_TYPES[idx]) + + +def fuzz_get_validation_error(data: bytes) -> None: + fdp = atheris.FuzzedDataProvider(data) + path = fdp.ConsumeUnicodeNoSurrogates(256) + idx = fdp.ConsumeIntInRange(0, len(_DATA_TYPES) - 1) + with suppress(ValueError): + get_validation_error(path, _DATA_TYPES[idx]) + + +def fuzz_extract_from_value(data: bytes) -> None: + fdp = atheris.FuzzedDataProvider(data) + name = fdp.ConsumeUnicodeNoSurrogates(64) + choice = fdp.ConsumeIntInRange(0, 4) + if choice == 0: + value: Any = fdp.ConsumeFloat() + elif choice == 1: + value = fdp.ConsumeInt(8) + elif choice == 2: + value = fdp.ConsumeUnicodeNoSurrogates(32) + elif choice == 3: + value = None + else: + value = [fdp.ConsumeFloat() for _ in range(fdp.ConsumeIntInRange(0, 5))] + metrics: dict[str, float] = {} + with suppress(ValueError, TypeError, OverflowError): + _extract_from_value(name, value, metrics) + + +def _build_fuzz_dict(fdp: Any, depth: int) -> dict[str, Any]: + d: dict[str, Any] = {} + num_keys = fdp.ConsumeIntInRange(0, 4) + for _ in range(num_keys): + key = fdp.ConsumeUnicodeNoSurrogates(16) + if depth > 0 and fdp.ConsumeBool(): + d[key] = _build_fuzz_dict(fdp, depth - 1) + else: + choice = fdp.ConsumeIntInRange(0, 2) + if choice == 0: + d[key] = fdp.ConsumeFloat() + elif choice == 1: + d[key] = fdp.ConsumeInt(8) + else: + d[key] = fdp.ConsumeUnicodeNoSurrogates(16) + return d + + +def fuzz_extract_from_tracking_data(data: bytes) -> None: + fdp = atheris.FuzzedDataProvider(data) + tracking = _build_fuzz_dict(fdp, depth=fdp.ConsumeIntInRange(0, 3)) + metrics: dict[str, float] = {} + with suppress(ValueError, TypeError, OverflowError): + _extract_from_tracking_data(tracking, metrics, "") + + +def fuzz_sanitize_user_string(data: bytes) -> None: + fdp = atheris.FuzzedDataProvider(data) + value = fdp.ConsumeUnicodeNoSurrogates(512) + _sanitize_user_string(value) + + +def _build_nested_fuzz_value(fdp: Any, depth: int) -> Any: + choice = fdp.ConsumeIntInRange(0, 6) + if choice == 0: + return fdp.ConsumeUnicodeNoSurrogates(64) + if choice == 1: + return fdp.ConsumeFloat() + if choice == 2: + return fdp.ConsumeInt(8) + if choice == 3 and depth > 0: + return [_build_nested_fuzz_value(fdp, depth - 1) for _ in range(fdp.ConsumeIntInRange(0, 3))] + if choice == 4 and depth > 0: + return tuple(_build_nested_fuzz_value(fdp, depth - 1) for _ in range(fdp.ConsumeIntInRange(0, 3))) + if choice == 5 and depth > 0: + return {fdp.ConsumeUnicodeNoSurrogates(16): _build_nested_fuzz_value(fdp, depth - 1)} + return fdp.ConsumeUnicodeNoSurrogates(16) + + +def fuzz_sanitize_nested_value(data: bytes) -> None: + fdp = atheris.FuzzedDataProvider(data) + value = _build_nested_fuzz_value(fdp, depth=fdp.ConsumeIntInRange(0, 3)) + _sanitize_nested_value(value) + + +def fuzz_validate_safe_string(data: bytes) -> None: + from fastapi import HTTPException + + fdp = atheris.FuzzedDataProvider(data) + value = fdp.ConsumeUnicodeNoSurrogates(256) + patterns = [_SAFE_DATASET_ID_PATTERN, _SAFE_CAMERA_NAME_PATTERN] + pattern = patterns[fdp.ConsumeIntInRange(0, len(patterns) - 1)] + with suppress(HTTPException, ValueError, TypeError): + _validate_safe_string(value, pattern=pattern, label="fuzz") + + +def fuzz_dataset_id_to_blob_prefix(data: bytes) -> None: + fdp = atheris.FuzzedDataProvider(data) + dataset_id = fdp.ConsumeUnicodeNoSurrogates(256) + _dataset_id_to_blob_prefix(dataset_id) + + +def fuzz_datetime_encoder(data: bytes) -> None: + fdp = atheris.FuzzedDataProvider(data) + year = fdp.ConsumeIntInRange(1, 9999) + month = fdp.ConsumeIntInRange(1, 12) + day = fdp.ConsumeIntInRange(1, 28) + hour = fdp.ConsumeIntInRange(0, 23) + minute = fdp.ConsumeIntInRange(0, 59) + second = fdp.ConsumeIntInRange(0, 59) + dt = datetime(year, month, day, hour, minute, second, tzinfo=UTC) + encoder = _DateTimeEncoder() + with suppress(TypeError, ValueError, OverflowError): + encoder.encode({"ts": dt}) + with suppress(TypeError, ValueError, OverflowError): + encoder.encode({"val": fdp.ConsumeUnicodeNoSurrogates(32)}) + + +FUZZ_TARGETS = [ + fuzz_validate_blob_path, + fuzz_get_validation_error, + fuzz_extract_from_value, + fuzz_extract_from_tracking_data, + fuzz_sanitize_user_string, + fuzz_sanitize_nested_value, + fuzz_validate_safe_string, + fuzz_dataset_id_to_blob_prefix, + fuzz_datetime_encoder, +] + + +def fuzz_dispatch(data: bytes) -> None: + if not data: + return + idx = data[0] % len(FUZZ_TARGETS) + FUZZ_TARGETS[idx](data[1:]) + + +# ================================================================ +# Pytest tests (deterministic mode) +# ================================================================ + +_BLOB_PATH_CASES = [ + ("raw/robot-01/2026-03-05/episode-001.mcap", "raw", True), + ("raw/ROBOT-01/2026-03-05/episode.mcap", "raw", False), + ("converted/pick-place/data/chunk-000.parquet", "converted", True), + ("converted/pick-place-v2/meta/info.json", "converted", True), + ("reports/eval-run/2026-01-15/summary.json", "reports", True), + ("checkpoints/policy-01/20260315_143022.pt", "checkpoints", True), + ("checkpoints/policy-01/20260315_143022_step_1000.onnx", "checkpoints", True), + ("", "raw", False), + ("../traversal/attack.mcap", "raw", False), +] + + +class TestFuzzValidateBlobPath: + def test_known_paths(self) -> None: + for path, data_type, expected in _BLOB_PATH_CASES: + assert validate_blob_path(path, data_type) is expected, f"Failed for {path!r}" + + def test_unknown_data_type_raises(self) -> None: + try: + validate_blob_path("any/path", "unknown") # type: ignore[arg-type] + except ValueError: + return + raise AssertionError("Expected ValueError") + + +class TestFuzzGetValidationError: + def test_valid_path_returns_none(self) -> None: + assert get_validation_error("raw/robot-01/2026-03-05/episode.mcap", "raw") is None + + def test_invalid_uppercase_path(self) -> None: + error = get_validation_error("INVALID/PATH", "raw") + assert error is not None + assert "uppercase" in error + + def test_invalid_spaces_path(self) -> None: + error = get_validation_error("raw/robot 01/2026-03-05/ep.mcap", "raw") + assert error is not None + assert "spaces" in error + + +class TestFuzzExtractFromValue: + def test_float_value(self) -> None: + metrics: dict[str, float] = {} + _extract_from_value("loss", 3.14, metrics) + assert metrics["loss"] == 3.14 + + def test_none_value(self) -> None: + metrics: dict[str, float] = {} + _extract_from_value("loss", None, metrics) + assert "loss" not in metrics + + def test_int_value(self) -> None: + metrics: dict[str, float] = {} + _extract_from_value("step", 42, metrics) + assert metrics["step"] == 42.0 + + def test_string_value_ignored(self) -> None: + metrics: dict[str, float] = {} + _extract_from_value("tag", "not-a-number", metrics) + assert "tag" not in metrics + + +class TestFuzzExtractFromTrackingData: + def test_flat_dict(self) -> None: + metrics: dict[str, float] = {} + _extract_from_tracking_data({"loss": 0.5, "reward": 1.0}, metrics, "") + assert metrics["loss"] == 0.5 + assert metrics["reward"] == 1.0 + + def test_nested_dict(self) -> None: + metrics: dict[str, float] = {} + _extract_from_tracking_data({"train": {"loss": 0.1}}, metrics, "") + assert metrics["train/loss"] == 0.1 + + def test_empty_dict(self) -> None: + metrics: dict[str, float] = {} + _extract_from_tracking_data({}, metrics, "") + assert len(metrics) == 0 + + +class TestFuzzSanitizeUserString: + def test_strips_carriage_return(self) -> None: + assert _sanitize_user_string("hello\rworld") == "helloworld" + + def test_strips_line_feed(self) -> None: + assert _sanitize_user_string("hello\nworld") == "helloworld" + + def test_strips_crlf(self) -> None: + assert _sanitize_user_string("a\r\nb") == "ab" + + def test_preserves_normal_string(self) -> None: + assert _sanitize_user_string("hello-world_123") == "hello-world_123" + + def test_empty_string(self) -> None: + assert _sanitize_user_string("") == "" + + def test_unicode_passthrough(self) -> None: + result = _sanitize_user_string("café-über") # cspell:ignore über + assert "\r" not in result + assert "\n" not in result + + +class TestFuzzSanitizeNestedValue: + def test_string_sanitized(self) -> None: + result = _sanitize_nested_value("hello\r\nworld") + assert result == "helloworld" + + def test_list_elements_sanitized(self) -> None: + result = _sanitize_nested_value(["a\rb", "c\nd"]) + assert result == ["ab", "cd"] + + def test_tuple_elements_sanitized(self) -> None: + result = _sanitize_nested_value(("x\ry",)) + assert result == ("xy",) + + def test_dict_values_sanitized(self) -> None: + result = _sanitize_nested_value({"key": "val\nue"}) + assert result == {"key": "value"} + + def test_non_string_passthrough(self) -> None: + assert _sanitize_nested_value(42) == 42 + assert _sanitize_nested_value(3.14) == 3.14 + assert _sanitize_nested_value(None) is None + + +class TestFuzzValidateSafeString: + def test_valid_dataset_id(self) -> None: + _validate_safe_string("robot-01.data_v2", pattern=_SAFE_DATASET_ID_PATTERN, label="test") + + def test_null_byte_rejected(self) -> None: + from fastapi import HTTPException + + try: + _validate_safe_string("robot\x00evil", pattern=_SAFE_DATASET_ID_PATTERN, label="test") + except HTTPException as exc: + assert exc.status_code == 400 + return + raise AssertionError("Expected HTTPException for null byte") + + def test_traversal_rejected(self) -> None: + from fastapi import HTTPException + + try: + _validate_safe_string("../etc/passwd", pattern=_SAFE_DATASET_ID_PATTERN, label="test") + except HTTPException as exc: + assert exc.status_code == 400 + return + raise AssertionError("Expected HTTPException for path traversal") + + def test_valid_camera_name(self) -> None: + _validate_safe_string("front_left.rgb", pattern=_SAFE_CAMERA_NAME_PATTERN, label="test") + + def test_pattern_mismatch_rejected(self) -> None: + from fastapi import HTTPException + + try: + _validate_safe_string("!@#$%^&*()", pattern=_SAFE_DATASET_ID_PATTERN, label="test") + except HTTPException as exc: + assert exc.status_code == 400 + return + raise AssertionError("Expected HTTPException for invalid pattern") + + +class TestFuzzDatasetIdToBlobPrefix: + def test_double_dash_replaced(self) -> None: + assert _dataset_id_to_blob_prefix("group--dataset") == "group/dataset" + + def test_multiple_separators(self) -> None: + assert _dataset_id_to_blob_prefix("a--b--c") == "a/b/c" + + def test_no_separator(self) -> None: + assert _dataset_id_to_blob_prefix("simple-name") == "simple-name" + + def test_empty_string(self) -> None: + assert _dataset_id_to_blob_prefix("") == "" + + +class TestFuzzDateTimeEncoder: + def test_datetime_serialized(self) -> None: + dt = datetime(2026, 3, 15, 14, 30, 22, tzinfo=UTC) + result = json.loads(json.dumps({"ts": dt}, cls=_DateTimeEncoder)) + assert result["ts"] == "2026-03-15T14:30:22+00:00" + + def test_non_datetime_passthrough(self) -> None: + result = json.loads(json.dumps({"val": "hello", "num": 42}, cls=_DateTimeEncoder)) + assert result["val"] == "hello" + assert result["num"] == 42 + + def test_non_serializable_raises(self) -> None: + try: + json.dumps({"bad": set()}, cls=_DateTimeEncoder) + except TypeError: + return + raise AssertionError("Expected TypeError for non-serializable set") + + +# ================================================================ +# Atheris entry point +# ================================================================ + +if __name__ == "__main__" and FUZZING: + _crash_dir = _REPO_ROOT / "logs" / "fuzz-crashes" + _crash_dir.mkdir(parents=True, exist_ok=True) + + _corpus_dir = Path(__file__).parent / "fuzz-corpus" + _argv = [*sys.argv, f"-artifact_prefix={_crash_dir}/"] + if _corpus_dir.is_dir() and str(_corpus_dir) not in sys.argv: + _argv.append(str(_corpus_dir)) + + atheris.instrument_all() + atheris.Setup(_argv, fuzz_dispatch) + atheris.Fuzz() diff --git a/tests/generate_fuzz_corpus.py b/tests/generate_fuzz_corpus.py new file mode 100644 index 00000000..d80bc214 --- /dev/null +++ b/tests/generate_fuzz_corpus.py @@ -0,0 +1,126 @@ +"""Generate initial seed corpus for the fuzz harness. + +Creates binary seed files in ``tests/fuzz-corpus/`` that give Atheris +meaningful starting points for mutation-based fuzzing. Each seed embeds +a routing byte (``data[0] % 9``) that selects the target, followed by +byte sequences representative of that target's input domain. + +Run this script once to bootstrap the corpus, then pass the directory to +the harness:: + + python tests/generate_fuzz_corpus.py + python tests/fuzz_harness.py tests/fuzz-corpus/ +""" + +from __future__ import annotations + +import struct +from pathlib import Path + +_CORPUS_DIR = Path(__file__).parent / "fuzz-corpus" + +# Routing bytes: data[0] % 9 selects the target index in FUZZ_TARGETS. +_T0 = b"\x00" # fuzz_validate_blob_path +_T1 = b"\x01" # fuzz_get_validation_error +_T2 = b"\x02" # fuzz_extract_from_value +_T3 = b"\x03" # fuzz_extract_from_tracking_data +_T4 = b"\x04" # fuzz_sanitize_user_string +_T5 = b"\x05" # fuzz_sanitize_nested_value +_T6 = b"\x06" # fuzz_validate_safe_string +_T7 = b"\x07" # fuzz_dataset_id_to_blob_prefix +_T8 = b"\x08" # fuzz_datetime_encoder + + +def _double(val: float) -> bytes: + return struct.pack(" None: + _CORPUS_DIR.mkdir(parents=True, exist_ok=True) + for name, content in _SEEDS: + (_CORPUS_DIR / name).write_bytes(content) + print(f"Generated {len(_SEEDS)} seed files in {_CORPUS_DIR}") + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock index 2c924d72..91b0ffa2 100644 --- a/uv.lock +++ b/uv.lock @@ -77,6 +77,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, ] +[[package]] +name = "atheris" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/58/5965955898e16bee17c8379eae12194993bf641c4629016991248b862069/atheris-3.0.0.tar.gz", hash = "sha256:1f0929c7bc3040f3fe4102e557718734190cf2d7718bbb8e3ce6d3eb56ef5bb3", size = 373239, upload-time = "2025-11-24T23:54:02.15Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/15/cf109e2e8696a54c8c4bc3ef79a79bec32361eceb64eaa36690a682e83a9/atheris-3.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8a5c8a781467c187da40fd29139784193e2647058831f837f675d0bb8cbd8746", size = 34805555, upload-time = "2025-11-24T23:53:53.477Z" }, + { url = "https://files.pythonhosted.org/packages/85/8c/e9960b996e70e5f6a523670431166b2b238de52fef094955515dcf854da1/atheris-3.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:510e502c57b6dc615fb174066407af620d4c7f73cf08a782c86e7761bf12c4eb", size = 34907016, upload-time = "2025-11-24T23:53:56.535Z" }, + { url = "https://files.pythonhosted.org/packages/db/48/df670f75f458cc7c1752a01a394fd59c830b08172dd59cf29d73f31050f9/atheris-3.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a402cdca8a650d1371050b1f9552eb4cdc488d2db64950d603c4560318365eac", size = 34858525, upload-time = "2025-11-24T23:53:59.925Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -1803,6 +1814,9 @@ dev = [ { name = "ruff" }, { name = "tqdm" }, ] +fuzz = [ + { name = "atheris" }, +] [package.metadata] @@ -1822,6 +1836,7 @@ dev = [ { name = "ruff", specifier = "==0.15.9" }, { name = "tqdm", specifier = "==4.67.3" }, ] +fuzz = [{ name = "atheris", specifier = ">=3.0" }] [[package]] name = "pillow"