Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
the 0.62.1 Intel resolution because the newer release has no macOS Intel wheel.
- Union overlapping matching chord-estimate intervals before duration-weighted recall so acceptance scores cannot double-count annotated time or exceed 100%.
- Reject malformed accuracy-report provenance, including non-hex SHA-256 text and non-finite metric values, before acceptance evidence is consumed.
- Bound accuracy fixture byte size, channel count, sample rate, and decoded duration before checksum staging or PCM allocation.
- Score the C major acceptance case from checksummed on-disk WAV bytes instead of the pre-write in-memory triad.
- Name tonight's first playable range on the ready rehearsal map and tell the player to check that span on their instrument before the section.
- Display the analyzed song tempo (BPM) as a badge in the rehearsal workspace.
Expand Down
9 changes: 6 additions & 3 deletions docs/doctoring/real-audio-accuracy-acceptance.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet?
repository product `VERSION`.
- Mitigations: no network, no shell, checksum fail-closed before C-major
decode and before tempo scoring, deterministic file-decoder downmix for
ordinary multichannel WAV input, non-empty finite floating-point
ordinary mono/stereo and bounded multichannel WAV input, a 100 MiB fixture
byte cap before checksum reads, an eight-channel cap, a 192 kHz sample-rate
cap, and a 15-minute decoded-duration cap, non-empty finite floating-point
one-dimensional mono PCM admission at the direct C-major evaluator boundary,
finite positive non-Boolean decoded sample-rate evidence, overlap-safe chord
duration, finite non-Boolean annotation/estimate timing, strictly increasing
Expand All @@ -142,8 +144,9 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet?
rejection, exact non-empty product-version provenance, bounded fixture
durations, and no copyrighted commercial recordings. Fixture paths are pytest
temp files; reports store SHA-256 and labels, not waveform bytes.
- Test points: deterministic digest, C major recall after file decode, ordinary
stereo WAV downmix at the file-decoder boundary, direct decoded-PCM
- Test points: deterministic digest, over-sized fixture rejection before digest
allocation, C major recall after file decode, ordinary stereo WAV downmix at
the file-decoder boundary, excessive channel/rate/duration rejection, direct decoded-PCM
empty/non-floating/non-finite/non-mono rejection and invalid sample-rate
rejection, overlapping matching intervals do not double-count annotation
duration, non-finite and Boolean chord annotation/estimate timing rejection,
Expand Down
3 changes: 3 additions & 0 deletions docs/plans/2026-08-16-real-audio-accuracy-acceptance.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ Trusted: in-repo generators, metric definitions, and registered floors.

- No network and no shell interpolation.
- Checksum mismatch raises before C-major decode and before tempo scoring.
- Fixture bytes are capped before hashing; WAV headers are bounded by channel,
sample-rate, and duration limits before decoded PCM allocation.
- Manifest parsing fails closed on missing or mistyped fields.
- Fixtures are short, synthetic, and license-clean.

Expand All @@ -37,6 +39,7 @@ Trusted: in-repo generators, metric definitions, and registered floors.
- Silence on disk fails even when a C major array exists in memory
- 120 BPM Acc1 after file decode
- Checksum mismatch through both file evaluators
- Oversized, excessive-channel, excessive-rate, and excessive-duration input rejection
- Malformed report rejection
- Silence must not pass as C major

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,13 @@
C_MAJOR_LABEL,
DEFAULT_CLICK_BPM,
DEFAULT_SAMPLE_RATE,
MAX_ACCURACY_CHANNELS,
MAX_ACCURACY_DURATION_SECONDS,
MAX_ACCURACY_FILE_BYTES,
MAX_ACCURACY_SAMPLE_RATE,
assert_fixture_checksum,
read_pcm_wav,
read_verified_fixture_bytes,
render_c_major_triad,
render_click_track,
write_pcm_wav,
Expand All @@ -29,6 +34,10 @@
"C_MAJOR_LABEL",
"DEFAULT_CLICK_BPM",
"DEFAULT_SAMPLE_RATE",
"MAX_ACCURACY_CHANNELS",
"MAX_ACCURACY_DURATION_SECONDS",
"MAX_ACCURACY_FILE_BYTES",
"MAX_ACCURACY_SAMPLE_RATE",
"assert_fixture_checksum",
"build_case_report",
"duration_weighted_chord_recall",
Expand All @@ -37,6 +46,7 @@
"evaluate_click_tempo_file",
"parse_case_report",
"read_pcm_wav",
"read_verified_fixture_bytes",
"read_product_version",
"render_c_major_triad",
"render_click_track",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
C_MAJOR_LABEL,
DEFAULT_CLICK_BPM,
DEFAULT_SAMPLE_RATE,
_validate_wav_header,
read_pcm_wav,
read_verified_fixture_bytes,
)
Expand All @@ -33,6 +34,7 @@ def _verified_fixture_path(audio_path: Path, expected_sha256: str) -> Iterator[P
with TemporaryDirectory(prefix="bandscope-accuracy-") as temp_dir:
verified_path = Path(temp_dir) / "verified.wav"
verified_path.write_bytes(payload)
_validate_wav_header(verified_path)
yield verified_path


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
C4_HZ = 261.63
E4_HZ = 329.63
G4_HZ = 392.00
MAX_ACCURACY_FILE_BYTES = 100 * 1024 * 1024
MAX_ACCURACY_DURATION_SECONDS = 15 * 60
MAX_ACCURACY_CHANNELS = 8
MAX_ACCURACY_SAMPLE_RATE = 192_000
_CLICK_FREQUENCY_HZ = 1_000.0
_CLICK_DURATION_SECONDS = 0.01
_CLICK_DECAY = 80.0
Expand Down Expand Up @@ -147,6 +151,28 @@ def write_pcm_wav(path: Path, audio: NDArray[np.floating], sample_rate: int) ->
return hashlib.sha256(path.read_bytes()).hexdigest()


def _validate_wav_header(path: Path) -> None:
"""Validate WAV resource metadata before allocating decoded PCM."""
try:
info = sf.info(path)
except Exception as error:
raise ValueError("WAV header could not be inspected") from error

if info.channels > MAX_ACCURACY_CHANNELS:
raise ValueError(
f"WAV has too many channels: {info.channels} (max {MAX_ACCURACY_CHANNELS})"
)
if info.samplerate <= 0 or info.samplerate > MAX_ACCURACY_SAMPLE_RATE:
raise ValueError(f"WAV sample rate is outside the supported range: {info.samplerate}")
if not np.isfinite(info.duration):
raise ValueError("WAV duration must be finite")
if info.duration > MAX_ACCURACY_DURATION_SECONDS:
Comment thread
seonghobae marked this conversation as resolved.
raise ValueError(
f"WAV is too long for accuracy analysis: {info.duration:g} seconds "
f"(max {MAX_ACCURACY_DURATION_SECONDS} seconds)"
)


def read_pcm_wav(path: Path) -> tuple[NDArray[np.float32], int]:
"""Decode a WAV file to mono float32 PCM.

Expand All @@ -157,8 +183,10 @@ def read_pcm_wav(path: Path) -> tuple[NDArray[np.float32], int]:
A tuple of mono samples and the file sample rate.

Raises:
ValueError: If the file has no samples after decode.
ValueError: If the header exceeds the acceptance resource limits or the
file has no samples after decode.
"""
_validate_wav_header(path)
audio, sample_rate = sf.read(path, dtype="float32", always_2d=False)
samples = np.asarray(audio, dtype=np.float32)
if samples.ndim > 1:
Expand All @@ -179,9 +207,26 @@ def read_verified_fixture_bytes(path: Path, expected_sha256: str) -> bytes:
The exact bytes whose SHA-256 matched ``expected_sha256``.

Raises:
ValueError: If the snapshot digest does not match.
ValueError: If the file exceeds the byte limit, cannot be read, or the
snapshot digest does not match.
"""
payload = path.read_bytes()
try:
file_size = path.stat().st_size
except OSError as error:
raise ValueError("Accuracy fixture could not be inspected") from error
if file_size > MAX_ACCURACY_FILE_BYTES:
raise ValueError(
f"Accuracy fixture is too large: {file_size} bytes "
f"(max {MAX_ACCURACY_FILE_BYTES} bytes)"
)

try:
with path.open("rb") as fileobj:
payload = fileobj.read(MAX_ACCURACY_FILE_BYTES + 1)
except OSError as error:
raise ValueError("Accuracy fixture could not be read") from error
if len(payload) > MAX_ACCURACY_FILE_BYTES:
raise ValueError(f"Accuracy fixture is too large (max {MAX_ACCURACY_FILE_BYTES} bytes)")
actual = hashlib.sha256(payload).hexdigest()
if actual != expected_sha256:
raise ValueError("Accuracy fixture checksum mismatch")
Expand Down
117 changes: 117 additions & 0 deletions services/analysis-engine/tests/test_accuracy_acceptance.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,21 @@
from __future__ import annotations

from pathlib import Path
from types import SimpleNamespace

import numpy as np
import pytest
import soundfile as sf

import bandscope_analysis.accuracy.fixtures as fixture_helpers
from bandscope_analysis.accuracy import (
C_MAJOR_LABEL,
DEFAULT_CLICK_BPM,
DEFAULT_SAMPLE_RATE,
MAX_ACCURACY_CHANNELS,
MAX_ACCURACY_DURATION_SECONDS,
MAX_ACCURACY_FILE_BYTES,
MAX_ACCURACY_SAMPLE_RATE,
build_case_report,
duration_weighted_chord_recall,
evaluate_c_major_file,
Expand All @@ -25,6 +31,7 @@
parse_case_report,
read_pcm_wav,
read_product_version,
read_verified_fixture_bytes,
render_c_major_triad,
render_click_track,
tempo_acc1,
Expand Down Expand Up @@ -132,6 +139,116 @@ def test_read_pcm_wav_rejects_empty_file(tmp_path: Path) -> None:
read_pcm_wav(path)


def test_read_pcm_wav_rejects_resource_excesses(tmp_path: Path) -> None:
"""WAV decode must reject excessive channels, rates, and duration first."""
too_many_channels = tmp_path / "too-many-channels.wav"
sf.write(
too_many_channels,
np.zeros((8, MAX_ACCURACY_CHANNELS + 1), dtype=np.float32),
DEFAULT_SAMPLE_RATE,
)
with pytest.raises(ValueError, match="too many channels"):
read_pcm_wav(too_many_channels)

too_fast = tmp_path / "too-fast.wav"
sf.write(too_fast, np.zeros(8, dtype=np.float32), MAX_ACCURACY_SAMPLE_RATE + 1)
with pytest.raises(ValueError, match="sample rate"):
read_pcm_wav(too_fast)

too_long = tmp_path / "too-long.wav"
sf.write(
too_long,
np.zeros(MAX_ACCURACY_DURATION_SECONDS + 1, dtype=np.float32),
1,
)
with pytest.raises(ValueError, match="too long"):
read_pcm_wav(too_long)


def test_click_tempo_file_applies_wav_resource_limits(tmp_path: Path) -> None:
"""Tempo acceptance must use the same header guard as chord acceptance."""
path = tmp_path / "too-many-channels-tempo.wav"
audio = np.zeros((8, MAX_ACCURACY_CHANNELS + 1), dtype=np.float32)
digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE)

with pytest.raises(ValueError, match="too many channels"):
evaluate_click_tempo_file(path, digest)


def test_read_pcm_wav_rejects_uninspectable_or_non_finite_header(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Header inspection must fail closed before an unbounded decode attempt."""
path = tmp_path / "header.wav"
path.write_bytes(b"not-a-wav")

def raise_info(_path: Path) -> object:
raise RuntimeError("invalid header")

monkeypatch.setattr(sf, "info", raise_info)
with pytest.raises(ValueError, match="header could not be inspected"):
read_pcm_wav(path)

monkeypatch.setattr(
sf,
"info",
lambda _path: SimpleNamespace(
channels=1,
samplerate=DEFAULT_SAMPLE_RATE,
duration=float("nan"),
),
)
with pytest.raises(ValueError, match="duration must be finite"):
read_pcm_wav(path)


def test_read_verified_fixture_rejects_oversized_file(tmp_path: Path) -> None:
"""Digest verification must not read an over-sized fixture into memory."""
path = tmp_path / "oversized.wav"
with path.open("wb") as fileobj:
fileobj.truncate(MAX_ACCURACY_FILE_BYTES + 1)

with pytest.raises(ValueError, match="too large"):
read_verified_fixture_bytes(path, "0" * 64)


def test_read_verified_fixture_normalizes_file_inspection_errors(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Fixture reads must expose stable errors when stat or open fails."""
path = tmp_path / "fixture.wav"
path.write_bytes(b"fixture")

def raise_stat(_path: Path) -> object:
raise OSError("stat denied")

monkeypatch.setattr(Path, "stat", raise_stat)
with pytest.raises(ValueError, match="could not be inspected"):
read_verified_fixture_bytes(path, "0" * 64)

monkeypatch.undo()

def raise_open(_path: Path, *_args: object, **_kwargs: object) -> object:
raise OSError("open denied")

monkeypatch.setattr(Path, "open", raise_open)
with pytest.raises(ValueError, match="could not be read"):
read_verified_fixture_bytes(path, "0" * 64)


def test_read_verified_fixture_rejects_growth_after_stat(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A file that grows after stat must not be hashed as truncated evidence."""
path = tmp_path / "grown.wav"
path.write_bytes(b"12345")
monkeypatch.setattr(fixture_helpers, "MAX_ACCURACY_FILE_BYTES", 4)
monkeypatch.setattr(Path, "stat", lambda _path: SimpleNamespace(st_size=4))

with pytest.raises(ValueError, match="too large"):
read_verified_fixture_bytes(path, "0" * 64)


def test_pipeline_surfaces_c_on_active_lead_vocal() -> None:
"""Unmocked assembly must put measured C on lead vocal when that stem is active."""
audio = render_c_major_triad(duration_seconds=3.0)
Expand Down