From 1b9ed317a651f96054574b2467af2f9b3c6675a4 Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 29 Jul 2026 21:06:54 +0800 Subject: [PATCH 1/7] feat: add offline embedding benchmark runner --- applications/embedding_benchmark.py | 195 +++++ .../2026-07-29-embedding-benchmark.json | 8 + eval/README.md | 36 + eval/__init__.py | 1 + eval/embedding-benchmark/input.schema.json | 47 ++ eval/embedding-benchmark/report.schema.json | 142 ++++ eval/embedding_benchmark.py | 723 ++++++++++++++++++ pyproject.toml | 3 +- tests/unit/test_embedding_benchmark_cli.py | 143 ++++ .../test_embedding_benchmark_determinism.py | 42 + .../unit/test_embedding_benchmark_identity.py | 59 ++ .../unit/test_embedding_benchmark_metrics.py | 139 ++++ ...test_embedding_benchmark_report_privacy.py | 35 + .../test_embedding_benchmark_report_schema.py | 144 ++++ ...t_embedding_benchmark_runtime_untouched.py | 25 + .../test_embedding_provider_validation.py | 33 + 16 files changed, 1774 insertions(+), 1 deletion(-) create mode 100644 applications/embedding_benchmark.py create mode 100644 docs/evaluation/2026-07-29-embedding-benchmark.json create mode 100644 eval/__init__.py create mode 100644 eval/embedding-benchmark/input.schema.json create mode 100644 eval/embedding-benchmark/report.schema.json create mode 100644 eval/embedding_benchmark.py create mode 100644 tests/unit/test_embedding_benchmark_cli.py create mode 100644 tests/unit/test_embedding_benchmark_determinism.py create mode 100644 tests/unit/test_embedding_benchmark_identity.py create mode 100644 tests/unit/test_embedding_benchmark_metrics.py create mode 100644 tests/unit/test_embedding_benchmark_report_privacy.py create mode 100644 tests/unit/test_embedding_benchmark_report_schema.py create mode 100644 tests/unit/test_embedding_benchmark_runtime_untouched.py create mode 100644 tests/unit/test_embedding_provider_validation.py diff --git a/applications/embedding_benchmark.py b/applications/embedding_benchmark.py new file mode 100644 index 00000000..cf9c90c2 --- /dev/null +++ b/applications/embedding_benchmark.py @@ -0,0 +1,195 @@ +"""Tracked offline CLI for the embedding benchmark; never production-composed.""" + +from __future__ import annotations + +import argparse +import importlib +import json +import sys +from collections.abc import Callable, Sequence +from pathlib import Path +from time import perf_counter +from typing import Protocol, cast + +from eval.embedding_benchmark import ( + BenchmarkCase, + BenchmarkDataset, + BenchmarkDocument, + BenchmarkEmbeddingProvider, + BenchmarkUnavailable, + RetrievalJudge, + run_benchmark, + validate_report_document, +) + +REPORT_SCHEMA_PATH = Path("eval/embedding-benchmark/report.schema.json") + + +class LocalProviderFactory(Protocol): + def __call__(self, model_dir: Path) -> BenchmarkEmbeddingProvider: ... + + +def build_local_provider(role: str, model_dir: Path) -> BenchmarkEmbeddingProvider: + """Load an optional benchmark-only provider plugin from the model directory. + + The plugin is deliberately outside the locked production environment. A local + model directory supplies ``context_engine_provider.py`` with ``create_provider``; + the returned provider still must expose the exact pinned identity in the report. + """ + + plugin = model_dir / "context_engine_provider.py" + if not plugin.is_file(): + raise BenchmarkUnavailable( + f"{role} local provider is unavailable; model directory must contain " + "context_engine_provider.py" + ) + spec = importlib.util.spec_from_file_location( + f"context_engine_embedding_benchmark_{role}", plugin + ) + if spec is None or spec.loader is None: + raise BenchmarkUnavailable(f"{role} local provider is unavailable") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + factory = getattr(module, "create_provider", None) + if not callable(factory): + raise BenchmarkUnavailable(f"{role} local provider is unavailable") + return cast(LocalProviderFactory, factory)(model_dir) + + +def load_retrieval_judge(specification: str) -> RetrievalJudge: + """Load #129's judge through its injected adapter factory.""" + + try: + module_name, separator, factory_name = specification.partition(":") + if not separator or not module_name or not factory_name: + raise ValueError + factory = getattr(importlib.import_module(module_name), factory_name) + if not callable(factory): + raise TypeError + return cast(Callable[[], RetrievalJudge], factory)() + except (ImportError, AttributeError, TypeError, ValueError): + raise BenchmarkUnavailable("retrieval judge is unavailable") from None + + +def load_dataset(path: Path) -> BenchmarkDataset: + """Load the strict drop-in input format used by the pending durable corpus.""" + + try: + root = _closed( + json.loads(path.read_text(encoding="utf-8")), + frozenset({"cases", "documents", "schemaVersion"}), + ) + if root["schemaVersion"] != "context-engine-embedding-benchmark-input-v1": + raise BenchmarkUnavailable("benchmark dataset is unavailable") + raw_documents = root["documents"] + raw_cases = root["cases"] + if type(raw_documents) is not list or type(raw_cases) is not list: + raise BenchmarkUnavailable("benchmark dataset is unavailable") + documents = tuple( + BenchmarkDocument( + document_ref=_text(document["documentRef"]), + text=_text(document["text"]), + ) + for value in raw_documents + for document in [_closed(value, frozenset({"documentRef", "text"}))] + ) + cases = tuple( + BenchmarkCase( + case_ref=_text(case["caseRef"]), + query=_text(case["query"]), + expected_document_refs=_refs(case["expectedDocumentRefs"]), + slice_name=_text(case["slice"]), + ) + for value in raw_cases + for case in [ + _closed( + value, + frozenset( + {"caseRef", "expectedDocumentRefs", "query", "slice"} + ), + ) + ] + ) + return BenchmarkDataset(documents=documents, cases=cases) + except BenchmarkUnavailable: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError): + raise BenchmarkUnavailable("benchmark dataset is unavailable") from None + + +def _closed(value: object, fields: frozenset[str]) -> dict[str, object]: + if type(value) is not dict or frozenset(value) != fields: + raise BenchmarkUnavailable("benchmark dataset is unavailable") + if any(type(key) is not str for key in value): + raise BenchmarkUnavailable("benchmark dataset is unavailable") + return value + + +def _text(value: object) -> str: + if type(value) is not str or not value or value != value.strip(): + raise BenchmarkUnavailable("benchmark dataset is unavailable") + return value + + +def _refs(value: object) -> tuple[str, ...]: + if type(value) is not list: + raise BenchmarkUnavailable("benchmark dataset is unavailable") + return tuple(_text(item) for item in value) + + +def _require_ignored_output(path: Path, *, workspace_root: Path) -> None: + state_root = (workspace_root / ".context-engine").resolve() + if not path.resolve().is_relative_to(state_root): + raise BenchmarkUnavailable( + "benchmark output must be under the ignored .context-engine directory" + ) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="context-engine-embedding-benchmark", + description="Run the offline 384-dimensional embedding comparison.", + ) + commands = parser.add_subparsers(dest="command", required=True) + run = commands.add_parser("run", help="run both pinned local models") + run.add_argument("--dataset", type=Path, required=True) + run.add_argument("--primary-model-dir", type=Path, required=True) + run.add_argument("--baseline-model-dir", type=Path, required=True) + run.add_argument("--judge", required=True) + run.add_argument("--output", type=Path, required=True) + run.add_argument("--top-k", type=int, default=10) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = _parser() + arguments = parser.parse_args(argv) + try: + output = cast(Path, arguments.output) + _require_ignored_output(output, workspace_root=Path.cwd()) + report = run_benchmark( + dataset=load_dataset(cast(Path, arguments.dataset)), + primary=build_local_provider( + "primary", cast(Path, arguments.primary_model_dir) + ), + baseline=build_local_provider( + "baseline", cast(Path, arguments.baseline_model_dir) + ), + judge=load_retrieval_judge(cast(str, arguments.judge)), + top_k=cast(int, arguments.top_k), + clock=perf_counter, + ) + validate_report_document(report, schema_path=REPORT_SCHEMA_PATH) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return 0 + except BenchmarkUnavailable as failure: + print(str(failure), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/evaluation/2026-07-29-embedding-benchmark.json b/docs/evaluation/2026-07-29-embedding-benchmark.json new file mode 100644 index 00000000..05a2c5a8 --- /dev/null +++ b/docs/evaluation/2026-07-29-embedding-benchmark.json @@ -0,0 +1,8 @@ +{ + "models": { + "baseline": "pending_corpus", + "primary": "pending_corpus" + }, + "result": "pending_corpus", + "schemaVersion": "context-engine-embedding-benchmark-frozen-result-v1" +} diff --git a/eval/README.md b/eval/README.md index 114aaafc..f498394a 100644 --- a/eval/README.md +++ b/eval/README.md @@ -42,3 +42,39 @@ authorized Package, and case pass rate. Package Evidence has no relevance-rank contract, so this version does not report recall@k. Quality is `measured`; Reliability and Budget remain `not-evaluated`. This report does not modify or feed the release-gate report, and there is no CI threshold. + +## Offline embedding comparison + +`context-engine-embedding-benchmark` is an offline evaluation CLI. It compares +the exact 384-dimensional primary and baseline model identities required by +issue #128 and never composes an `EmbeddingProvider` into Supply or Runtime. +The main environment deliberately contains no model framework: each local, +pinned model directory supplies a benchmark-only `context_engine_provider.py` +plugin and its fully resolved identity (repository id, immutable revision, +artifact digest, dimension, normalization, pooling, prompt prefixes, reduction, +precision, and batching). + +The input follows `embedding-benchmark/input.schema.json`; the output follows +`embedding-benchmark/report.schema.json` and must be written below the ignored +`.context-engine/` directory. Retrieval scores are not implemented by this +runner. `--judge module:factory` injects the retrieval judge owned by issue #129, +which is solely responsible for case hit, macro/micro Evidence recall, and slice +breakdowns. + +The maintainer corpus is private and pending delivery to a durable, +maintainer-controlled location outside disposable Git worktrees. Once available, +an operator may run: + +```bash +uv run context-engine-embedding-benchmark run \ + --dataset "$DURABLE_GOLDEN_ROOT/embedding-benchmark-v1.json" \ + --primary-model-dir "$PINNED_MODEL_ROOT/qwen3-embedding-0.6b" \ + --baseline-model-dir "$PINNED_MODEL_ROOT/multilingual-e5-small" \ + --judge eval.retrieval_judge:create_judge \ + --output .context-engine/eval/embedding-benchmark-v1.json +``` + +The tracked frozen result contains metrics only. It currently records +`pending_corpus`; it must never contain queries, note titles, paths, excerpts, +or model weights. A losing primary is a valid benchmark outcome and is frozen as +`lose`, not treated as a runner error. diff --git a/eval/__init__.py b/eval/__init__.py new file mode 100644 index 00000000..6e29dd93 --- /dev/null +++ b/eval/__init__.py @@ -0,0 +1 @@ +"""Offline evaluation tools and deterministic judge contracts.""" diff --git a/eval/embedding-benchmark/input.schema.json b/eval/embedding-benchmark/input.schema.json new file mode 100644 index 00000000..346ec5c8 --- /dev/null +++ b/eval/embedding-benchmark/input.schema.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://context-engine.invalid/eval/embedding-benchmark/input.schema.json", + "title": "ContextEngine offline embedding benchmark input v1", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "documents", "cases"], + "properties": { + "schemaVersion": {"const": "context-engine-embedding-benchmark-input-v1"}, + "documents": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["documentRef", "text"], + "properties": { + "documentRef": {"$ref": "#/$defs/nonblank"}, + "text": {"$ref": "#/$defs/nonblank"} + } + } + }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["caseRef", "query", "expectedDocumentRefs", "slice"], + "properties": { + "caseRef": {"type": "string", "pattern": "^[a-z][a-z0-9_-]{0,127}$"}, + "query": {"$ref": "#/$defs/nonblank"}, + "expectedDocumentRefs": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/nonblank"} + }, + "slice": {"$ref": "#/$defs/nonblank"} + } + } + } + }, + "$defs": { + "nonblank": {"type": "string", "minLength": 1, "pattern": ".*\\S.*"} + } +} diff --git a/eval/embedding-benchmark/report.schema.json b/eval/embedding-benchmark/report.schema.json new file mode 100644 index 00000000..91b35c45 --- /dev/null +++ b/eval/embedding-benchmark/report.schema.json @@ -0,0 +1,142 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://context-engine.invalid/eval/embedding-benchmark/report.schema.json", + "title": "ContextEngine offline embedding benchmark report v1", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "run", "models", "standingTwinBaseline", "comparison"], + "properties": { + "schemaVersion": {"const": "context-engine-embedding-benchmark-report-v1"}, + "run": {"$ref": "#/$defs/run"}, + "models": { + "type": "object", + "additionalProperties": false, + "required": ["primary", "baseline"], + "properties": { + "primary": {"$ref": "#/$defs/modelReport"}, + "baseline": {"$ref": "#/$defs/modelReport"} + } + }, + "standingTwinBaseline": { + "type": "object", + "additionalProperties": false, + "required": ["caseHitValue"], + "properties": {"caseHitValue": {"type": "number", "minimum": 0, "maximum": 1}} + }, + "comparison": { + "type": "object", + "additionalProperties": false, + "required": ["primaryAgainstModelBaseline", "primaryAgainstStandingTwinBaseline"], + "properties": { + "primaryAgainstModelBaseline": {"enum": ["win", "tie", "lose"]}, + "primaryAgainstStandingTwinBaseline": {"enum": ["win", "tie", "lose"]} + } + } + }, + "$defs": { + "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "run": { + "type": "object", + "additionalProperties": false, + "required": ["datasetDigest", "runIdentity", "topK"], + "properties": { + "datasetDigest": {"$ref": "#/$defs/digest"}, + "runIdentity": {"$ref": "#/$defs/digest"}, + "topK": {"type": "integer", "minimum": 1} + } + }, + "identity": { + "type": "object", + "additionalProperties": false, + "required": ["modelId", "revision", "artifactDigest", "dimension", "normalization", "pooling", "queryPrefix", "documentPrefix", "reduction", "precision", "batchSize"], + "properties": { + "modelId": {"type": "string", "minLength": 1}, + "revision": {"type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}, + "artifactDigest": {"$ref": "#/$defs/digest"}, + "dimension": {"const": 384}, + "normalization": {"enum": ["l2", "none"]}, + "pooling": {"type": "string", "minLength": 1}, + "queryPrefix": {"type": "string"}, + "documentPrefix": {"type": "string"}, + "reduction": {"enum": ["none_native_384", "matryoshka_truncate_384"]}, + "precision": {"type": "string", "minLength": 1}, + "batchSize": {"type": "integer", "minimum": 1} + } + }, + "caseHit": { + "type": "object", + "additionalProperties": false, + "required": ["hits", "totalCases", "value"], + "properties": { + "hits": {"type": "integer", "minimum": 0}, + "totalCases": {"type": "integer", "minimum": 1}, + "value": {"type": "number", "minimum": 0, "maximum": 1} + } + }, + "recall": { + "type": "object", + "additionalProperties": false, + "required": ["macro", "micro"], + "properties": { + "macro": { + "type": "object", + "additionalProperties": false, + "required": ["value"], + "properties": {"value": {"type": "number", "minimum": 0, "maximum": 1}} + }, + "micro": { + "type": "object", + "additionalProperties": false, + "required": ["hits", "totalExpected", "value"], + "properties": { + "hits": {"type": "integer", "minimum": 0}, + "totalExpected": {"type": "integer", "minimum": 1}, + "value": {"type": "number", "minimum": 0, "maximum": 1} + } + } + } + }, + "metrics": { + "type": "object", + "additionalProperties": false, + "required": ["caseHit", "evidenceRecall", "perSlice"], + "properties": { + "caseHit": {"$ref": "#/$defs/caseHit"}, + "evidenceRecall": {"$ref": "#/$defs/recall"}, + "perSlice": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["caseHit", "evidenceRecall"], + "properties": { + "caseHit": {"$ref": "#/$defs/caseHit"}, + "evidenceRecall": {"$ref": "#/$defs/recall"} + } + } + } + } + }, + "timing": { + "type": "object", + "additionalProperties": false, + "required": ["documentCount", "perDocumentEmbedMilliseconds", "wallClockMilliseconds"], + "properties": { + "documentCount": {"type": "integer", "minimum": 1}, + "perDocumentEmbedMilliseconds": {"type": "number", "minimum": 0}, + "wallClockMilliseconds": {"type": "number", "minimum": 0} + } + }, + "modelReport": { + "type": "object", + "additionalProperties": false, + "required": ["identity", "metrics", "timing"], + "properties": { + "identity": {"$ref": "#/$defs/identity"}, + "metrics": {"$ref": "#/$defs/metrics"}, + "timing": {"$ref": "#/$defs/timing"} + } + } + } +} diff --git a/eval/embedding_benchmark.py b/eval/embedding_benchmark.py new file mode 100644 index 00000000..ea685efb --- /dev/null +++ b/eval/embedding_benchmark.py @@ -0,0 +1,723 @@ +"""Offline-only embedding benchmark contracts and report validation.""" + +from __future__ import annotations + +import json +import math +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from hashlib import sha256 +from pathlib import Path +from typing import Final, Protocol, cast + +EMBEDDING_DIMENSION: Final = 384 +REPORT_SCHEMA_VERSION: Final = "context-engine-embedding-benchmark-report-v1" +_PINNED_REVISION = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") +_SHA256_DIGEST = re.compile(r"^[0-9a-f]{64}$") +_NORMALIZATIONS: Final = frozenset({"l2", "none"}) +_REDUCTIONS: Final = frozenset({"none_native_384", "matryoshka_truncate_384"}) + + +class BenchmarkUnavailable(RuntimeError): + """The benchmark input, identity, provider, judge, or output is unavailable.""" + + +@dataclass(frozen=True, slots=True) +class ModelIdentity: + """Complete, reproducible identity for one offline embedding provider.""" + + model_id: str + revision: str + artifact_digest: str + dimension: int + normalization: str + pooling: str + query_prefix: str + document_prefix: str + reduction: str + precision: str + batch_size: int + + def __post_init__(self) -> None: + if ( + type(self.model_id) is not str + or not self.model_id + or self.model_id != self.model_id.strip() + or not _PINNED_REVISION.fullmatch(self.revision) + or not _SHA256_DIGEST.fullmatch(self.artifact_digest) + or type(self.dimension) is not int + or self.dimension != EMBEDDING_DIMENSION + or self.normalization not in _NORMALIZATIONS + or type(self.pooling) is not str + or not self.pooling + or self.pooling != self.pooling.strip() + or type(self.query_prefix) is not str + or type(self.document_prefix) is not str + or self.reduction not in _REDUCTIONS + or type(self.precision) is not str + or not self.precision + or self.precision != self.precision.strip() + or type(self.batch_size) is not int + or not 1 <= self.batch_size <= 1024 + ): + raise BenchmarkUnavailable("model identity is unresolved") + + def public_document(self) -> dict[str, object]: + return { + "artifactDigest": self.artifact_digest, + "batchSize": self.batch_size, + "dimension": self.dimension, + "documentPrefix": self.document_prefix, + "modelId": self.model_id, + "normalization": self.normalization, + "pooling": self.pooling, + "precision": self.precision, + "queryPrefix": self.query_prefix, + "reduction": self.reduction, + "revision": self.revision, + } + + +@dataclass(frozen=True, slots=True) +class BenchmarkDocument: + """One opaque benchmark document reference and its offline content.""" + + document_ref: str + text: str = field(repr=False) + + def __post_init__(self) -> None: + _require_nonblank("benchmark document_ref", self.document_ref) + _require_nonblank("benchmark document text", self.text) + + +@dataclass(frozen=True, slots=True) +class BenchmarkCase: + """One retrieval query and its exact expected document references.""" + + case_ref: str + query: str = field(repr=False) + expected_document_refs: tuple[str, ...] + slice_name: str + + def __post_init__(self) -> None: + _require_nonblank("benchmark case_ref", self.case_ref) + _require_nonblank("benchmark query", self.query) + _require_nonblank("benchmark slice", self.slice_name) + if ( + type(self.expected_document_refs) is not tuple + or not self.expected_document_refs + or any( + type(value) is not str or not value or value != value.strip() + for value in self.expected_document_refs + ) + or len(set(self.expected_document_refs)) + != len(self.expected_document_refs) + ): + raise BenchmarkUnavailable("benchmark expected Evidence is unavailable") + + +@dataclass(frozen=True, slots=True) +class BenchmarkDataset: + """Strict offline input that can be loaded from the pending durable corpus.""" + + documents: tuple[BenchmarkDocument, ...] = field(repr=False) + cases: tuple[BenchmarkCase, ...] = field(repr=False) + digest: str = field(init=False) + + def __post_init__(self) -> None: + if ( + type(self.documents) is not tuple + or not self.documents + or any(type(value) is not BenchmarkDocument for value in self.documents) + or type(self.cases) is not tuple + or not self.cases + or any(type(value) is not BenchmarkCase for value in self.cases) + ): + raise BenchmarkUnavailable("benchmark dataset is unavailable") + document_refs = tuple(value.document_ref for value in self.documents) + case_refs = tuple(value.case_ref for value in self.cases) + if ( + len(set(document_refs)) != len(document_refs) + or len(set(case_refs)) != len(case_refs) + or any( + expected not in set(document_refs) + for case in self.cases + for expected in case.expected_document_refs + ) + ): + raise BenchmarkUnavailable("benchmark dataset is unavailable") + document = { + "cases": [ + { + "caseRef": case.case_ref, + "expectedDocumentRefs": list(case.expected_document_refs), + "query": case.query, + "slice": case.slice_name, + } + for case in self.cases + ], + "documents": [ + {"documentRef": value.document_ref, "text": value.text} + for value in self.documents + ], + "schemaVersion": "context-engine-embedding-benchmark-input-v1", + } + object.__setattr__(self, "digest", _digest(document)) + + +class BenchmarkEmbeddingProvider(Protocol): + """Benchmark-only provider; no production composition accepts this seam.""" + + @property + def identity(self) -> ModelIdentity: ... + + def embed_queries( + self, values: tuple[str, ...] + ) -> tuple[tuple[float, ...], ...]: ... + + def embed_documents( + self, values: tuple[str, ...] + ) -> tuple[tuple[float, ...], ...]: ... + + +@dataclass(frozen=True, slots=True) +class RetrievalJudgeCase: + """Adapter document passed to the #129 retrieval judge.""" + + case_ref: str + expected_evidence: tuple[str, ...] + retrieved_evidence: tuple[str, ...] + slice_name: str + + +@dataclass(frozen=True, slots=True) +class CaseHitMetric: + hits: int + total_cases: int + value: float + + +@dataclass(frozen=True, slots=True) +class MacroRecallMetric: + value: float + + +@dataclass(frozen=True, slots=True) +class MicroRecallMetric: + hits: int + total_expected: int + value: float + + +@dataclass(frozen=True, slots=True) +class EvidenceRecallMetric: + macro: MacroRecallMetric + micro: MicroRecallMetric + + +@dataclass(frozen=True, slots=True) +class SliceMetrics: + case_hit: CaseHitMetric + evidence_recall: EvidenceRecallMetric + + +@dataclass(frozen=True, slots=True) +class RetrievalMetrics: + case_hit: CaseHitMetric + evidence_recall: EvidenceRecallMetric + per_slice: Mapping[str, SliceMetrics] + + +class RetrievalJudge(Protocol): + """Injected adapter to #129; this issue intentionally implements no metrics.""" + + def evaluate_retrieval( + self, cases: tuple[RetrievalJudgeCase, ...] + ) -> RetrievalMetrics: ... + + +def run_benchmark( + *, + dataset: BenchmarkDataset, + primary: BenchmarkEmbeddingProvider, + baseline: BenchmarkEmbeddingProvider, + judge: RetrievalJudge, + top_k: int, + clock: Callable[[], float], +) -> dict[str, object]: + """Compare two pinned providers and delegate every metric to #129's judge.""" + + if type(dataset) is not BenchmarkDataset or type(top_k) is not int or top_k < 1: + raise BenchmarkUnavailable("benchmark configuration is unavailable") + if primary.identity.model_id != "Qwen/Qwen3-Embedding-0.6B": + raise BenchmarkUnavailable("primary benchmark model is unavailable") + if primary.identity.reduction != "matryoshka_truncate_384": + raise BenchmarkUnavailable("primary benchmark reduction is unavailable") + if baseline.identity.model_id != "intfloat/multilingual-e5-small": + raise BenchmarkUnavailable("baseline benchmark model is unavailable") + if baseline.identity.reduction != "none_native_384": + raise BenchmarkUnavailable("baseline benchmark reduction is unavailable") + primary_report = _run_model(dataset, primary, judge, top_k, clock) + baseline_report = _run_model(dataset, baseline, judge, top_k, clock) + primary_hit = _metric_case_hit(primary_report) + baseline_hit = _metric_case_hit(baseline_report) + run_document = { + "datasetDigest": dataset.digest, + "topK": top_k, + } + run_identity = _digest( + { + **run_document, + "baselineIdentity": baseline.identity.public_document(), + "primaryIdentity": primary.identity.public_document(), + } + ) + return { + "comparison": { + "primaryAgainstModelBaseline": _comparison(primary_hit, baseline_hit), + "primaryAgainstStandingTwinBaseline": _comparison(primary_hit, 0.038), + }, + "models": {"baseline": baseline_report, "primary": primary_report}, + "run": {**run_document, "runIdentity": run_identity}, + "schemaVersion": REPORT_SCHEMA_VERSION, + "standingTwinBaseline": {"caseHitValue": 0.038}, + } + + +def _run_model( + dataset: BenchmarkDataset, + provider: BenchmarkEmbeddingProvider, + judge: RetrievalJudge, + top_k: int, + clock: Callable[[], float], +) -> dict[str, object]: + query_values = tuple( + provider.identity.query_prefix + case.query for case in dataset.cases + ) + document_values = tuple( + provider.identity.document_prefix + document.text + for document in dataset.documents + ) + started = clock() + query_vectors = _validated_vectors( + query_values, + provider.embed_queries(query_values), + ) + document_started = clock() + document_vectors = _validated_vectors( + document_values, provider.embed_documents(document_values) + ) + document_finished = clock() + document_refs = tuple(value.document_ref for value in dataset.documents) + judged_cases = tuple( + RetrievalJudgeCase( + case_ref=case.case_ref, + expected_evidence=case.expected_document_refs, + retrieved_evidence=_retrieve( + query_vectors[index], document_vectors, document_refs, top_k + ), + slice_name=case.slice_name, + ) + for index, case in enumerate(dataset.cases) + ) + metrics = judge.evaluate_retrieval(judged_cases) + finished = clock() + return { + "identity": provider.identity.public_document(), + "metrics": _metrics_document(metrics), + "timing": { + "documentCount": len(dataset.documents), + "perDocumentEmbedMilliseconds": max( + 0.0, + (document_finished - document_started) + * 1000 + / len(dataset.documents), + ), + "wallClockMilliseconds": max(0.0, (finished - started) * 1000), + }, + } + + +def _validated_vectors( + inputs: tuple[str, ...], + vectors: object, +) -> tuple[tuple[float, ...], ...]: + if type(vectors) is not tuple or len(vectors) != len(inputs): + raise BenchmarkUnavailable("embedding provider is unavailable") + validated: list[tuple[float, ...]] = [] + for vector in vectors: + if ( + type(vector) is not tuple + or len(vector) != EMBEDDING_DIMENSION + or any(type(value) not in {int, float} for value in vector) + ): + raise BenchmarkUnavailable("embedding provider is unavailable") + converted = tuple(float(value) for value in vector) + if not all(math.isfinite(value) for value in converted) or not any(converted): + raise BenchmarkUnavailable("embedding provider is unavailable") + validated.append(converted) + return tuple(validated) + + +def _retrieve( + query: tuple[float, ...], + documents: tuple[tuple[float, ...], ...], + document_refs: tuple[str, ...], + top_k: int, +) -> tuple[str, ...]: + ranked = sorted( + zip(document_refs, documents, strict=True), + key=lambda item: ( + -_cosine_similarity(query, item[1]), + item[0], + ), + ) + return tuple(ref for ref, _vector in ranked[:top_k]) + + +def _cosine_similarity( + left: tuple[float, ...], + right: tuple[float, ...], +) -> float: + dot_product = sum( + left_value * right_value + for left_value, right_value in zip(left, right, strict=True) + ) + left_norm = math.sqrt(sum(value * value for value in left)) + right_norm = math.sqrt(sum(value * value for value in right)) + return dot_product / (left_norm * right_norm) + + +def _metrics_document(metrics: RetrievalMetrics) -> dict[str, object]: + return { + "caseHit": { + "hits": metrics.case_hit.hits, + "totalCases": metrics.case_hit.total_cases, + "value": metrics.case_hit.value, + }, + "evidenceRecall": _recall_document(metrics.evidence_recall), + "perSlice": { + name: { + "caseHit": { + "hits": value.case_hit.hits, + "totalCases": value.case_hit.total_cases, + "value": value.case_hit.value, + }, + "evidenceRecall": _recall_document(value.evidence_recall), + } + for name, value in sorted(metrics.per_slice.items()) + }, + } + + +def _recall_document(value: EvidenceRecallMetric) -> dict[str, object]: + return { + "macro": {"value": value.macro.value}, + "micro": { + "hits": value.micro.hits, + "totalExpected": value.micro.total_expected, + "value": value.micro.value, + }, + } + + +def _metric_case_hit(model_report: dict[str, object]) -> float: + metrics = _object(model_report["metrics"]) + case_hit = _object(metrics["caseHit"]) + value = case_hit["value"] + if type(value) not in {int, float}: + raise BenchmarkUnavailable("retrieval judge is unavailable") + return float(cast(int | float, value)) + + +def _comparison(left: float, right: float) -> str: + if left > right: + return "win" + if left < right: + return "lose" + return "tie" + + +def _digest(document: object) -> str: + return sha256( + json.dumps( + document, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + + +def _require_nonblank(name: str, value: object) -> str: + if type(value) is not str or not value or value != value.strip(): + raise BenchmarkUnavailable(f"{name} is unavailable") + return value + + +def validate_report_document( + document: object, + *, + schema_path: Path, +) -> None: + """Validate the closed v1 report without adding a production dependency.""" + + try: + schema = json.loads(schema_path.read_text(encoding="utf-8")) + _validate_json_schema(document, _object(schema), _object(schema)) + _validate_report(document, schema) + except BenchmarkUnavailable: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError): + raise BenchmarkUnavailable("benchmark report schema is unavailable") from None + + +def _validate_report(document: object, schema: object) -> None: + del schema + report = _closed_object( + document, + frozenset( + { + "comparison", + "models", + "run", + "schemaVersion", + "standingTwinBaseline", + } + ), + ) + if report["schemaVersion"] != REPORT_SCHEMA_VERSION: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + _closed_object( + report["comparison"], + frozenset( + {"primaryAgainstModelBaseline", "primaryAgainstStandingTwinBaseline"} + ), + ) + run = _closed_object( + report["run"], frozenset({"datasetDigest", "runIdentity", "topK"}) + ) + _require_digest(run["datasetDigest"]) + _require_digest(run["runIdentity"]) + if type(run["topK"]) is not int or run["topK"] < 1: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + standing = _closed_object( + report["standingTwinBaseline"], frozenset({"caseHitValue"}) + ) + _require_ratio(standing["caseHitValue"]) + models = _closed_object(report["models"], frozenset({"baseline", "primary"})) + _validate_model_report(models["primary"]) + _validate_model_report(models["baseline"]) + + +def _validate_json_schema( + value: object, + schema: dict[str, object], + root: dict[str, object], +) -> None: + """Apply the small Draft 2020-12 vocabulary used by the tracked schema.""" + + reference = schema.get("$ref") + if reference is not None: + if type(reference) is not str or not reference.startswith("#/"): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + resolved: object = root + for component in reference[2:].split("/"): + resolved = _object(resolved).get(component) + _validate_json_schema(value, _object(resolved), root) + return + + if "const" in schema and value != schema["const"]: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + enum = schema.get("enum") + if enum is not None and (type(enum) is not list or value not in enum): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + + expected_type = schema.get("type") + if expected_type == "object": + document = _object(value) + required = schema.get("required", []) + if type(required) is not list or any( + type(item) is not str for item in required + ): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + if not set(cast(list[str], required)).issubset(document): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + minimum_properties = schema.get("minProperties", 0) + if type(minimum_properties) is not int or len(document) < minimum_properties: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + properties = _object(schema.get("properties", {})) + additional = schema.get("additionalProperties", True) + for key, item in document.items(): + child_schema = properties.get(key) + if child_schema is not None: + _validate_json_schema(item, _object(child_schema), root) + elif additional is False: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + elif type(additional) is dict: + _validate_json_schema(item, _object(additional), root) + elif additional is not True: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + elif expected_type == "string": + if type(value) is not str: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + minimum_length = schema.get("minLength", 0) + pattern = schema.get("pattern") + if type(minimum_length) is not int or len(value) < minimum_length: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + if pattern is not None and ( + type(pattern) is not str or re.fullmatch(pattern, value) is None + ): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + elif expected_type == "integer": + if type(value) is not int: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + _validate_numeric_bounds(value, schema) + elif expected_type == "number": + if type(value) not in {int, float}: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + _validate_numeric_bounds(cast(int | float, value), schema) + elif expected_type is not None: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + + +def _validate_numeric_bounds(value: int | float, schema: dict[str, object]) -> None: + minimum = schema.get("minimum") + maximum = schema.get("maximum") + if minimum is not None and ( + type(minimum) not in {int, float} or value < cast(int | float, minimum) + ): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + if maximum is not None and ( + type(maximum) not in {int, float} or value > cast(int | float, maximum) + ): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + + +def _validate_model_report(value: object) -> None: + report = _closed_object(value, frozenset({"identity", "metrics", "timing"})) + identity = _closed_object( + report["identity"], + frozenset( + { + "artifactDigest", + "batchSize", + "dimension", + "documentPrefix", + "modelId", + "normalization", + "pooling", + "precision", + "queryPrefix", + "reduction", + "revision", + } + ), + ) + ModelIdentity( + model_id=cast(str, identity["modelId"]), + revision=cast(str, identity["revision"]), + artifact_digest=cast(str, identity["artifactDigest"]), + dimension=cast(int, identity["dimension"]), + normalization=cast(str, identity["normalization"]), + pooling=cast(str, identity["pooling"]), + query_prefix=cast(str, identity["queryPrefix"]), + document_prefix=cast(str, identity["documentPrefix"]), + reduction=cast(str, identity["reduction"]), + precision=cast(str, identity["precision"]), + batch_size=cast(int, identity["batchSize"]), + ) + _validate_metrics(report["metrics"]) + timing = _closed_object( + report["timing"], + frozenset( + {"documentCount", "perDocumentEmbedMilliseconds", "wallClockMilliseconds"} + ), + ) + if type(timing["documentCount"]) is not int or timing["documentCount"] < 1: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + _require_nonnegative_number(timing["perDocumentEmbedMilliseconds"]) + _require_nonnegative_number(timing["wallClockMilliseconds"]) + + +def _validate_metrics(value: object) -> None: + metrics = _closed_object( + value, + frozenset({"caseHit", "evidenceRecall", "perSlice"}), + ) + _validate_case_hit(metrics["caseHit"]) + _validate_recall(metrics["evidenceRecall"]) + per_slice = _object(metrics["perSlice"]) + if not per_slice: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + for slice_metrics in per_slice.values(): + item = _closed_object( + slice_metrics, + frozenset({"caseHit", "evidenceRecall"}), + ) + _validate_case_hit(item["caseHit"]) + _validate_recall(item["evidenceRecall"]) + + +def _validate_case_hit(value: object) -> None: + metric = _closed_object(value, frozenset({"hits", "totalCases", "value"})) + if ( + type(metric["hits"]) is not int + or type(metric["totalCases"]) is not int + or not 0 <= metric["hits"] <= metric["totalCases"] + or metric["totalCases"] < 1 + ): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + _require_ratio(metric["value"]) + + +def _validate_recall(value: object) -> None: + recall = _closed_object(value, frozenset({"macro", "micro"})) + macro = _closed_object(recall["macro"], frozenset({"value"})) + _require_ratio(macro["value"]) + micro = _closed_object( + recall["micro"], frozenset({"hits", "totalExpected", "value"}) + ) + if ( + type(micro["hits"]) is not int + or type(micro["totalExpected"]) is not int + or not 0 <= micro["hits"] <= micro["totalExpected"] + or micro["totalExpected"] < 1 + ): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + _require_ratio(micro["value"]) + + +def _object(value: object) -> dict[str, object]: + if type(value) is not dict or any(type(key) is not str for key in value): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + return value + + +def _closed_object(value: object, fields: frozenset[str]) -> dict[str, object]: + document = _object(value) + if frozenset(document) != fields: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + return document + + +def _require_digest(value: object) -> None: + if type(value) is not str or not _SHA256_DIGEST.fullmatch(value): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + + +def _require_ratio(value: object) -> None: + if type(value) not in {int, float}: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + number = float(cast(int | float, value)) + if not math.isfinite(number) or not 0.0 <= number <= 1.0: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + + +def _require_nonnegative_number(value: object) -> None: + if type(value) not in {int, float}: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + number = float(cast(int | float, value)) + if not math.isfinite(number) or number < 0.0: + raise BenchmarkUnavailable("benchmark report schema is unavailable") diff --git a/pyproject.toml b/pyproject.toml index e85bdc6d..6d66a984 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,10 +36,11 @@ context-engine-api = "applications.api:main" context-engine-control = "applications.control:main" context-engine-dogfood-seed = "applications.dogfood:main" context-engine-dogfood-eval = "applications.dogfood_evaluation:main" +context-engine-embedding-benchmark = "applications.embedding_benchmark:main" context-engine-worker = "applications.worker:main" [tool.hatch.build.targets.wheel] -packages = ["engine", "adapters", "applications", "migrations"] +packages = ["engine", "adapters", "applications", "eval", "migrations"] [tool.mypy] python_version = "3.13" diff --git a/tests/unit/test_embedding_benchmark_cli.py b/tests/unit/test_embedding_benchmark_cli.py new file mode 100644 index 00000000..3aedb673 --- /dev/null +++ b/tests/unit/test_embedding_benchmark_cli.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, cast + +import pytest + +from applications import embedding_benchmark as cli +from eval.embedding_benchmark import BenchmarkUnavailable +from tests.unit.test_embedding_benchmark_metrics import ( + RecordingJudge, + SyntheticProvider, + _identity, +) + + +def _write_dataset(path: Path) -> None: + path.write_text( + json.dumps( + { + "cases": [ + { + "caseRef": "case-alpha", + "expectedDocumentRefs": ["doc-alpha"], + "query": "alpha", + "slice": "single_doc", + } + ], + "documents": [ + {"documentRef": "doc-alpha", "text": "alpha"}, + {"documentRef": "doc-beta", "text": "beta"}, + ], + "schemaVersion": "context-engine-embedding-benchmark-input-v1", + } + ), + encoding="utf-8", + ) + + +def test_cli_run_writes_a_schema_valid_report_for_a_synthetic_set( + tmp_path: Path, + monkeypatch: Any, +) -> None: + dataset_path = tmp_path / "dataset.json" + output_path = Path(".context-engine") / "test" / "embedding-benchmark.json" + _write_dataset(dataset_path) + + def provider(role: str, _model_dir: Path) -> SyntheticProvider: + model_id = ( + "Qwen/Qwen3-Embedding-0.6B" + if role == "primary" + else "intfloat/multilingual-e5-small" + ) + return SyntheticProvider(_identity(model_id)) + + monkeypatch.setattr(cli, "build_local_provider", provider) + monkeypatch.setattr(cli, "load_retrieval_judge", lambda _spec: RecordingJudge()) + + result = cli.main( + [ + "run", + "--dataset", + str(dataset_path), + "--primary-model-dir", + str(tmp_path / "primary"), + "--baseline-model-dir", + str(tmp_path / "baseline"), + "--judge", + "eval.retrieval_judge:create_judge", + "--output", + str(output_path), + "--top-k", + "1", + ] + ) + + report = json.loads(output_path.read_text(encoding="utf-8")) + assert result == 0 + assert report["models"]["primary"]["identity"]["modelId"] == ( + "Qwen/Qwen3-Embedding-0.6B" + ) + assert report["comparison"]["primaryAgainstModelBaseline"] == "tie" + output_path.unlink() + + +def test_cli_refuses_report_output_outside_the_ignored_state_directory( + tmp_path: Path, +) -> None: + dataset_path = tmp_path / "dataset.json" + _write_dataset(dataset_path) + + result = cli.main( + [ + "run", + "--dataset", + str(dataset_path), + "--primary-model-dir", + str(tmp_path / "primary"), + "--baseline-model-dir", + str(tmp_path / "baseline"), + "--judge", + "eval.retrieval_judge:create_judge", + "--output", + str(Path("tests") / "tracked.json"), + ] + ) + + assert result == 2 + assert not (Path("tests") / "tracked.json").exists() + + +def test_cli_refuses_a_missing_retrieval_judge() -> None: + with pytest.raises(BenchmarkUnavailable, match="judge is unavailable"): + cli.load_retrieval_judge("eval.retrieval_judge:create_judge") + + +def test_output_cannot_escape_the_workspace_state_directory(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + + with pytest.raises(BenchmarkUnavailable, match="must be under"): + cli._require_ignored_output( + workspace / ".context-engine" / ".." / "tracked.json", + workspace_root=workspace, + ) + + +def test_local_plugin_provider_is_loaded_only_from_the_requested_model_directory( + tmp_path: Path, +) -> None: + model_dir = tmp_path / "model" + model_dir.mkdir() + plugin = model_dir / "context_engine_provider.py" + plugin.write_text( + "def create_provider(model_dir):\n" + " return {'requested': str(model_dir)}\n", + encoding="utf-8", + ) + + provider = cli.build_local_provider("primary", model_dir) + + assert cast(Any, provider) == {"requested": str(model_dir)} diff --git a/tests/unit/test_embedding_benchmark_determinism.py b/tests/unit/test_embedding_benchmark_determinism.py new file mode 100644 index 00000000..c0650772 --- /dev/null +++ b/tests/unit/test_embedding_benchmark_determinism.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import Any, cast + +from eval.embedding_benchmark import run_benchmark +from tests.unit.test_embedding_benchmark_metrics import ( + RecordingJudge, + SyntheticProvider, + _dataset, + _identity, +) + + +def test_same_dataset_and_model_identity_produce_identical_retrieval_metrics() -> None: + primary = SyntheticProvider(_identity("Qwen/Qwen3-Embedding-0.6B")) + baseline = SyntheticProvider(_identity("intfloat/multilingual-e5-small")) + + first = run_benchmark( + dataset=_dataset(), + primary=primary, + baseline=baseline, + judge=RecordingJudge(), + top_k=2, + clock=iter((1.0, 1.001, 1.002, 1.003, 2.0, 2.001, 2.002, 2.003)).__next__, + ) + second = run_benchmark( + dataset=_dataset(), + primary=primary, + baseline=baseline, + judge=RecordingJudge(), + top_k=2, + clock=iter((5.0, 5.5, 6.0, 6.75, 8.0, 8.5, 9.0, 9.75)).__next__, + ) + + first_models = cast(dict[str, Any], first["models"]) + second_models = cast(dict[str, Any], second["models"]) + assert first_models["primary"]["metrics"] == second_models["primary"]["metrics"] + assert first_models["baseline"]["metrics"] == second_models["baseline"][ + "metrics" + ] + assert first["run"] == second["run"] + assert first_models["primary"]["timing"] != second_models["primary"]["timing"] diff --git a/tests/unit/test_embedding_benchmark_identity.py b/tests/unit/test_embedding_benchmark_identity.py new file mode 100644 index 00000000..444f4eff --- /dev/null +++ b/tests/unit/test_embedding_benchmark_identity.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from dataclasses import replace +from typing import Any, cast + +import pytest + +from eval.embedding_benchmark import ( + EMBEDDING_DIMENSION, + BenchmarkUnavailable, + ModelIdentity, +) + + +def _resolved_identity() -> ModelIdentity: + return ModelIdentity( + model_id="Qwen/Qwen3-Embedding-0.6B", + revision="a" * 40, + artifact_digest="b" * 64, + dimension=EMBEDDING_DIMENSION, + normalization="l2", + pooling="last_token", + query_prefix="Instruct: retrieve relevant passages\nQuery:", + document_prefix="", + reduction="matryoshka_truncate_384", + precision="float32", + batch_size=8, + ) + + +@pytest.mark.parametrize( + ("field_name", "unresolved"), + ( + ("model_id", ""), + ("revision", "main"), + ("artifact_digest", "pending"), + ("dimension", 0), + ("normalization", "unknown"), + ("pooling", ""), + ("query_prefix", None), + ("document_prefix", None), + ("reduction", "unknown"), + ("precision", ""), + ("batch_size", 0), + ), +) +def test_benchmark_refuses_every_unresolved_model_identity_field( + field_name: str, + unresolved: object, +) -> None: + with pytest.raises(BenchmarkUnavailable, match="model identity is unresolved"): + replace(_resolved_identity(), **cast(Any, {field_name: unresolved})) + + +def test_empty_document_prefix_is_an_explicit_resolved_identity() -> None: + identity = _resolved_identity() + + assert identity.document_prefix == "" + assert identity.dimension == 384 diff --git a/tests/unit/test_embedding_benchmark_metrics.py b/tests/unit/test_embedding_benchmark_metrics.py new file mode 100644 index 00000000..93a5f93e --- /dev/null +++ b/tests/unit/test_embedding_benchmark_metrics.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, cast + +from eval.embedding_benchmark import ( + EMBEDDING_DIMENSION, + BenchmarkCase, + BenchmarkDataset, + BenchmarkDocument, + CaseHitMetric, + EvidenceRecallMetric, + MacroRecallMetric, + MicroRecallMetric, + ModelIdentity, + RetrievalJudgeCase, + RetrievalMetrics, + SliceMetrics, + run_benchmark, +) + + +def _identity(model_id: str) -> ModelIdentity: + return ModelIdentity( + model_id=model_id, + revision="a" * 40, + artifact_digest=("b" if model_id.startswith("Qwen") else "c") * 64, + dimension=EMBEDDING_DIMENSION, + normalization="l2", + pooling="mean", + query_prefix="query: ", + document_prefix="passage: ", + reduction=( + "matryoshka_truncate_384" + if model_id.startswith("Qwen") + else "none_native_384" + ), + precision="float32", + batch_size=8, + ) + + +@dataclass +class SyntheticProvider: + identity: ModelIdentity + + def embed_queries(self, values: tuple[str, ...]) -> tuple[tuple[float, ...], ...]: + return tuple(_vector(value) for value in values) + + def embed_documents( + self, values: tuple[str, ...] + ) -> tuple[tuple[float, ...], ...]: + return tuple(_vector(value) for value in values) + + +def _vector(value: str) -> tuple[float, ...]: + first = 1.0 if "alpha" in value else 0.0 + second = 1.0 if "beta" in value else 0.0 + third = 1.0 if first == second == 0.0 else 0.0 + return (first, second, third, *(0.0 for _ in range(EMBEDDING_DIMENSION - 3))) + + +class RecordingJudge: + def __init__(self) -> None: + self.calls: list[object] = [] + + def evaluate_retrieval( + self, cases: tuple[RetrievalJudgeCase, ...] + ) -> RetrievalMetrics: + self.calls.append(cases) + # Hand-checked oracle: recalls 1/2 and 1/1 => macro .75, micro 2/3. + return RetrievalMetrics( + case_hit=CaseHitMetric(hits=2, total_cases=2, value=1.0), + evidence_recall=EvidenceRecallMetric( + macro=MacroRecallMetric(value=0.75), + micro=MicroRecallMetric(hits=2, total_expected=3, value=2 / 3), + ), + per_slice={ + "single_doc": SliceMetrics( + case_hit=CaseHitMetric(hits=2, total_cases=2, value=1.0), + evidence_recall=EvidenceRecallMetric( + macro=MacroRecallMetric(value=0.75), + micro=MicroRecallMetric( + hits=2, + total_expected=3, + value=2 / 3, + ), + ), + ) + }, + ) + + +def _dataset() -> BenchmarkDataset: + return BenchmarkDataset( + documents=( + BenchmarkDocument(document_ref="doc-alpha", text="alpha"), + BenchmarkDocument(document_ref="doc-beta", text="beta"), + BenchmarkDocument(document_ref="doc-neither", text="neither"), + ), + cases=( + BenchmarkCase( + case_ref="case-alpha", + query="alpha", + expected_document_refs=("doc-alpha", "doc-neither"), + slice_name="single_doc", + ), + BenchmarkCase( + case_ref="case-beta", + query="beta", + expected_document_refs=("doc-beta",), + slice_name="single_doc", + ), + ), + ) + + +def test_runner_delegates_metrics_to_the_injected_129_retrieval_judge() -> None: + judge = RecordingJudge() + + report = run_benchmark( + dataset=_dataset(), + primary=SyntheticProvider(_identity("Qwen/Qwen3-Embedding-0.6B")), + baseline=SyntheticProvider(_identity("intfloat/multilingual-e5-small")), + judge=judge, + top_k=2, + clock=lambda: 1.0, + ) + + assert len(judge.calls) == 2 + first_cases = judge.calls[0] + assert isinstance(first_cases, tuple) + assert first_cases[0].expected_evidence == ("doc-alpha", "doc-neither") + assert first_cases[0].retrieved_evidence == ("doc-alpha", "doc-beta") + models = cast(dict[str, Any], report["models"]) + assert models["primary"]["metrics"]["evidenceRecall"] == { + "macro": {"value": 0.75}, + "micro": {"hits": 2, "totalExpected": 3, "value": 2 / 3}, + } diff --git a/tests/unit/test_embedding_benchmark_report_privacy.py b/tests/unit/test_embedding_benchmark_report_privacy.py new file mode 100644 index 00000000..6def5f95 --- /dev/null +++ b/tests/unit/test_embedding_benchmark_report_privacy.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path + +REPORT_PATH = Path("docs/evaluation/2026-07-29-embedding-benchmark.json") +FORBIDDEN_KEYS = frozenset({"excerpt", "path", "query", "text", "title"}) +PERSONAL_PATH = re.compile(r"(?:/Users/|[A-Za-z]:\\|\.md(?:\b|$))") + + +def _keys(value: object) -> set[str]: + if isinstance(value, dict): + return {str(key).lower() for key in value} | { + nested for item in value.values() for nested in _keys(item) + } + if isinstance(value, list): + return {nested for item in value for nested in _keys(item)} + return set() + + +def test_tracked_frozen_report_cannot_carry_personal_content() -> None: + report_text = REPORT_PATH.read_text(encoding="utf-8") + report = json.loads(report_text) + + assert not FORBIDDEN_KEYS.intersection(_keys(report)) + assert PERSONAL_PATH.search(report_text) is None + assert report == { + "models": { + "baseline": "pending_corpus", + "primary": "pending_corpus", + }, + "result": "pending_corpus", + "schemaVersion": "context-engine-embedding-benchmark-frozen-result-v1", + } diff --git a/tests/unit/test_embedding_benchmark_report_schema.py b/tests/unit/test_embedding_benchmark_report_schema.py new file mode 100644 index 00000000..94f88ea6 --- /dev/null +++ b/tests/unit/test_embedding_benchmark_report_schema.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path +from typing import Any + +import pytest + +from eval.embedding_benchmark import BenchmarkUnavailable, validate_report_document + +REPORT_SCHEMA = Path("eval/embedding-benchmark/report.schema.json") + + +def _identity(model_id: str, *, pooling: str) -> dict[str, object]: + return { + "artifactDigest": "b" * 64, + "batchSize": 8, + "dimension": 384, + "documentPrefix": "", + "modelId": model_id, + "normalization": "l2", + "pooling": pooling, + "precision": "float32", + "queryPrefix": "query: ", + "reduction": "none_native_384", + "revision": "a" * 40, + } + + +def _metrics() -> dict[str, object]: + return { + "caseHit": {"hits": 2, "totalCases": 2, "value": 1.0}, + "evidenceRecall": { + "macro": {"value": 0.75}, + "micro": {"hits": 3, "totalExpected": 4, "value": 0.75}, + }, + "perSlice": { + "single_doc": { + "caseHit": {"hits": 2, "totalCases": 2, "value": 1.0}, + "evidenceRecall": { + "macro": {"value": 0.75}, + "micro": { + "hits": 3, + "totalExpected": 4, + "value": 0.75, + }, + }, + } + }, + } + + +def _report() -> dict[str, object]: + model = { + "identity": _identity("Qwen/Qwen3-Embedding-0.6B", pooling="last_token"), + "metrics": _metrics(), + "timing": { + "documentCount": 4, + "perDocumentEmbedMilliseconds": 1.25, + "wallClockMilliseconds": 8.0, + }, + } + baseline = deepcopy(model) + baseline["identity"] = _identity( + "intfloat/multilingual-e5-small", pooling="mean" + ) + return { + "comparison": { + "primaryAgainstModelBaseline": "tie", + "primaryAgainstStandingTwinBaseline": "win", + }, + "models": {"baseline": baseline, "primary": model}, + "run": { + "datasetDigest": "c" * 64, + "runIdentity": "d" * 64, + "topK": 10, + }, + "schemaVersion": "context-engine-embedding-benchmark-report-v1", + "standingTwinBaseline": {"caseHitValue": 0.038}, + } + + +def test_report_validates_against_the_tracked_closed_schema() -> None: + schema = json.loads(REPORT_SCHEMA.read_text(encoding="utf-8")) + + validate_report_document(_report(), schema_path=REPORT_SCHEMA) + + assert schema["additionalProperties"] is False + assert schema["$id"].endswith("/embedding-benchmark/report.schema.json") + + +def test_validator_uses_the_tracked_schema_as_its_authority(tmp_path: Path) -> None: + schema = json.loads(REPORT_SCHEMA.read_text(encoding="utf-8")) + schema["properties"]["schemaVersion"]["const"] = "future-version" + changed_schema = tmp_path / "report.schema.json" + changed_schema.write_text(json.dumps(schema), encoding="utf-8") + + with pytest.raises(BenchmarkUnavailable, match="report schema"): + validate_report_document(_report(), schema_path=changed_schema) + + +@pytest.mark.parametrize( + "mutate", + ( + lambda report: report["comparison"].update( + {"primaryAgainstModelBaseline": "preferred"} + ), + lambda report: report.update({"queries": []}), + lambda report: report["models"]["primary"]["identity"].update( + {"nickname": "qwen"} + ), + ), +) +def test_report_refuses_values_or_fields_outside_the_tracked_schema( + mutate: Any, +) -> None: + report: dict[str, Any] = _report() + mutate(report) + + with pytest.raises(BenchmarkUnavailable, match="report schema"): + validate_report_document(report, schema_path=REPORT_SCHEMA) + + +@pytest.mark.parametrize( + "remove", + ( + ("models", "primary", "identity", "revision"), + ("models", "primary", "metrics", "caseHit"), + ("models", "baseline", "metrics", "evidenceRecall", "micro"), + ("models", "baseline", "timing", "perDocumentEmbedMilliseconds"), + ), +) +def test_report_refuses_a_missing_identity_metric_or_cost_field( + remove: tuple[str, ...], +) -> None: + report: dict[str, Any] = _report() + parent: dict[str, Any] = report + for key in remove[:-1]: + parent = parent[key] + del parent[remove[-1]] + + with pytest.raises(BenchmarkUnavailable, match="report schema"): + validate_report_document(report, schema_path=REPORT_SCHEMA) diff --git a/tests/unit/test_embedding_benchmark_runtime_untouched.py b/tests/unit/test_embedding_benchmark_runtime_untouched.py new file mode 100644 index 00000000..33573cbc --- /dev/null +++ b/tests/unit/test_embedding_benchmark_runtime_untouched.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +from engine.supply.embeddings import CONTEXT_FRAGMENT_EMBEDDING_DIMENSION + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] + + +def test_offline_benchmark_does_not_modify_or_compose_production_runtime() -> None: + changed_runtime = subprocess.run( + ("git", "diff", "--name-only", "origin/main", "--", "engine/runtime"), + cwd=REPOSITORY_ROOT, + check=True, + capture_output=True, + text=True, + ).stdout + + assert changed_runtime == "" + assert CONTEXT_FRAGMENT_EMBEDDING_DIMENSION == 384 + assert not any( + "eval.embedding_benchmark" in path.read_text(encoding="utf-8") + for path in (REPOSITORY_ROOT / "engine" / "runtime").glob("*.py") + ) diff --git a/tests/unit/test_embedding_provider_validation.py b/tests/unit/test_embedding_provider_validation.py new file mode 100644 index 00000000..722b57db --- /dev/null +++ b/tests/unit/test_embedding_provider_validation.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import math + +import pytest + +from engine.supply.embeddings import ( + EmbeddingProfile, + EmbeddingProviderUnavailable, + validate_embedding_batch, +) + + +@pytest.mark.parametrize( + ("inputs", "vectors"), + ( + (("first",), ((1.0,),)), + (("first",), ((1.0, math.nan),)), + (("first",), ((0.0, 0.0),)), + (("first", "second"), ((1.0, 1.0),)), + ), +) +def test_invalid_provider_vectors_fail_closed_as_provider_unavailable( + inputs: tuple[str, ...], + vectors: tuple[tuple[float, ...], ...], +) -> None: + profile = EmbeddingProfile(2) + + with pytest.raises( + EmbeddingProviderUnavailable, + match="Embedding provider is unavailable", + ): + validate_embedding_batch(inputs, vectors, profile) From c2b1237e4b0280fdaeb2dd0ed5fdfbef4f7c5235 Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 29 Jul 2026 21:12:13 +0800 Subject: [PATCH 2/7] fix: harden benchmark contracts --- eval/embedding_benchmark.py | 62 +++++++++++++++++++ .../test_embedding_benchmark_determinism.py | 7 ++- .../unit/test_embedding_benchmark_metrics.py | 54 +++++++++++++++- 3 files changed, 120 insertions(+), 3 deletions(-) diff --git a/eval/embedding_benchmark.py b/eval/embedding_benchmark.py index ea685efb..6b0f8f8b 100644 --- a/eval/embedding_benchmark.py +++ b/eval/embedding_benchmark.py @@ -197,11 +197,24 @@ class CaseHitMetric: total_cases: int value: float + def __post_init__(self) -> None: + if ( + type(self.hits) is not int + or type(self.total_cases) is not int + or not 0 <= self.hits <= self.total_cases + or self.total_cases < 1 + ): + raise BenchmarkUnavailable("retrieval judge is unavailable") + _require_judge_ratio(self.value) + @dataclass(frozen=True, slots=True) class MacroRecallMetric: value: float + def __post_init__(self) -> None: + _require_judge_ratio(self.value) + @dataclass(frozen=True, slots=True) class MicroRecallMetric: @@ -209,18 +222,42 @@ class MicroRecallMetric: total_expected: int value: float + def __post_init__(self) -> None: + if ( + type(self.hits) is not int + or type(self.total_expected) is not int + or not 0 <= self.hits <= self.total_expected + or self.total_expected < 1 + ): + raise BenchmarkUnavailable("retrieval judge is unavailable") + _require_judge_ratio(self.value) + @dataclass(frozen=True, slots=True) class EvidenceRecallMetric: macro: MacroRecallMetric micro: MicroRecallMetric + def __post_init__(self) -> None: + if ( + type(self.macro) is not MacroRecallMetric + or type(self.micro) is not MicroRecallMetric + ): + raise BenchmarkUnavailable("retrieval judge is unavailable") + @dataclass(frozen=True, slots=True) class SliceMetrics: case_hit: CaseHitMetric evidence_recall: EvidenceRecallMetric + def __post_init__(self) -> None: + if ( + type(self.case_hit) is not CaseHitMetric + or type(self.evidence_recall) is not EvidenceRecallMetric + ): + raise BenchmarkUnavailable("retrieval judge is unavailable") + @dataclass(frozen=True, slots=True) class RetrievalMetrics: @@ -228,6 +265,21 @@ class RetrievalMetrics: evidence_recall: EvidenceRecallMetric per_slice: Mapping[str, SliceMetrics] + def __post_init__(self) -> None: + if ( + type(self.case_hit) is not CaseHitMetric + or type(self.evidence_recall) is not EvidenceRecallMetric + or not isinstance(self.per_slice, Mapping) + or not self.per_slice + or any( + type(name) is not str + or not name + or type(value) is not SliceMetrics + for name, value in self.per_slice.items() + ) + ): + raise BenchmarkUnavailable("retrieval judge is unavailable") + class RetrievalJudge(Protocol): """Injected adapter to #129; this issue intentionally implements no metrics.""" @@ -322,6 +374,8 @@ def _run_model( for index, case in enumerate(dataset.cases) ) metrics = judge.evaluate_retrieval(judged_cases) + if type(metrics) is not RetrievalMetrics: + raise BenchmarkUnavailable("retrieval judge is unavailable") finished = clock() return { "identity": provider.identity.public_document(), @@ -456,6 +510,14 @@ def _require_nonblank(name: str, value: object) -> str: return value +def _require_judge_ratio(value: object) -> None: + if type(value) not in {int, float}: + raise BenchmarkUnavailable("retrieval judge is unavailable") + number = float(cast(int | float, value)) + if not math.isfinite(number) or not 0.0 <= number <= 1.0: + raise BenchmarkUnavailable("retrieval judge is unavailable") + + def validate_report_document( document: object, *, diff --git a/tests/unit/test_embedding_benchmark_determinism.py b/tests/unit/test_embedding_benchmark_determinism.py index c0650772..727af6c0 100644 --- a/tests/unit/test_embedding_benchmark_determinism.py +++ b/tests/unit/test_embedding_benchmark_determinism.py @@ -15,19 +15,21 @@ def test_same_dataset_and_model_identity_produce_identical_retrieval_metrics() - primary = SyntheticProvider(_identity("Qwen/Qwen3-Embedding-0.6B")) baseline = SyntheticProvider(_identity("intfloat/multilingual-e5-small")) + first_judge = RecordingJudge() first = run_benchmark( dataset=_dataset(), primary=primary, baseline=baseline, - judge=RecordingJudge(), + judge=first_judge, top_k=2, clock=iter((1.0, 1.001, 1.002, 1.003, 2.0, 2.001, 2.002, 2.003)).__next__, ) + second_judge = RecordingJudge() second = run_benchmark( dataset=_dataset(), primary=primary, baseline=baseline, - judge=RecordingJudge(), + judge=second_judge, top_k=2, clock=iter((5.0, 5.5, 6.0, 6.75, 8.0, 8.5, 9.0, 9.75)).__next__, ) @@ -39,4 +41,5 @@ def test_same_dataset_and_model_identity_produce_identical_retrieval_metrics() - "metrics" ] assert first["run"] == second["run"] + assert first_judge.calls == second_judge.calls assert first_models["primary"]["timing"] != second_models["primary"]["timing"] diff --git a/tests/unit/test_embedding_benchmark_metrics.py b/tests/unit/test_embedding_benchmark_metrics.py index 93a5f93e..fcda03fd 100644 --- a/tests/unit/test_embedding_benchmark_metrics.py +++ b/tests/unit/test_embedding_benchmark_metrics.py @@ -1,13 +1,16 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any, cast +import pytest + from eval.embedding_benchmark import ( EMBEDDING_DIMENSION, BenchmarkCase, BenchmarkDataset, BenchmarkDocument, + BenchmarkUnavailable, CaseHitMetric, EvidenceRecallMetric, MacroRecallMetric, @@ -137,3 +140,52 @@ def test_runner_delegates_metrics_to_the_injected_129_retrieval_judge() -> None: "macro": {"value": 0.75}, "micro": {"hits": 2, "totalExpected": 3, "value": 2 / 3}, } + + +def test_primary_loss_is_recorded_as_a_valid_result() -> None: + class LosingJudge(RecordingJudge): + def evaluate_retrieval( + self, cases: tuple[RetrievalJudgeCase, ...] + ) -> RetrievalMetrics: + result = super().evaluate_retrieval(cases) + if len(self.calls) == 1: + return RetrievalMetrics( + case_hit=CaseHitMetric(hits=0, total_cases=2, value=0.0), + evidence_recall=result.evidence_recall, + per_slice=result.per_slice, + ) + return result + + report = run_benchmark( + dataset=_dataset(), + primary=SyntheticProvider(_identity("Qwen/Qwen3-Embedding-0.6B")), + baseline=SyntheticProvider(_identity("intfloat/multilingual-e5-small")), + judge=LosingJudge(), + top_k=2, + clock=lambda: 1.0, + ) + + assert cast(dict[str, Any], report["comparison"])[ + "primaryAgainstModelBaseline" + ] == "lose" + + +def test_runner_requires_the_declared_384_dimension_reduction_per_model() -> None: + primary = SyntheticProvider( + replace( + _identity("Qwen/Qwen3-Embedding-0.6B"), + reduction="none_native_384", + ) + ) + + with pytest.raises(BenchmarkUnavailable, match="reduction is unavailable"): + run_benchmark( + dataset=_dataset(), + primary=primary, + baseline=SyntheticProvider( + _identity("intfloat/multilingual-e5-small") + ), + judge=RecordingJudge(), + top_k=2, + clock=lambda: 1.0, + ) From 8c8f9c8414e65259a29452d4f0d9f60c7785c5ff Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 29 Jul 2026 21:32:24 +0800 Subject: [PATCH 3/7] fix: close benchmark fidelity gaps --- applications/embedding_benchmark.py | 264 ++++++- eval/README.md | 29 +- eval/embedding-benchmark/input.schema.json | 11 +- eval/embedding-benchmark/report.schema.json | 17 +- eval/embedding_benchmark.py | 182 ++++- pyproject.toml | 6 + tests/unit/test_embedding_benchmark_cli.py | 228 +++++- .../unit/test_embedding_benchmark_metrics.py | 51 ++ .../test_embedding_benchmark_report_schema.py | 5 + uv.lock | 683 ++++++++++++++++++ 10 files changed, 1354 insertions(+), 122 deletions(-) diff --git a/applications/embedding_benchmark.py b/applications/embedding_benchmark.py index cf9c90c2..8622dd4e 100644 --- a/applications/embedding_benchmark.py +++ b/applications/embedding_benchmark.py @@ -7,67 +7,220 @@ import json import sys from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import StrEnum +from hashlib import sha256 from pathlib import Path from time import perf_counter -from typing import Protocol, cast +from typing import Any, cast + +import rfc8785 from eval.embedding_benchmark import ( + EMBEDDING_DIMENSION, BenchmarkCase, BenchmarkDataset, BenchmarkDocument, BenchmarkEmbeddingProvider, BenchmarkUnavailable, + ModelIdentity, RetrievalJudge, run_benchmark, + validate_json_schema_document, validate_report_document, ) -REPORT_SCHEMA_PATH = Path("eval/embedding-benchmark/report.schema.json") +EVAL_ROOT = Path(__file__).resolve().parents[1] / "eval" / "embedding-benchmark" +INPUT_SCHEMA_PATH = EVAL_ROOT / "input.schema.json" +REPORT_SCHEMA_PATH = EVAL_ROOT / "report.schema.json" +LOCK_PROFILE = "sha256-rfc8785-v1" +BENCHMARK_EXTRA = "context-engine[benchmark]" + + +class SupportedBackend(StrEnum): + """Closed benchmark backend set; arbitrary import paths are not accepted.""" + + SENTENCE_TRANSFORMERS = "sentence-transformers" + +@dataclass(frozen=True, slots=True) +class _ExpectedModel: + model_id: str + revision: str + pooling: str + query_prefix: str + document_prefix: str + reduction: str -class LocalProviderFactory(Protocol): - def __call__(self, model_dir: Path) -> BenchmarkEmbeddingProvider: ... +_EXPECTED_MODELS = { + "primary": _ExpectedModel( + model_id="Qwen/Qwen3-Embedding-0.6B", + revision="97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3", + pooling="last_token", + query_prefix=( + "Instruct: Given a web search query, retrieve relevant passages that " + "answer the query\nQuery:" + ), + document_prefix="", + reduction="matryoshka_truncate_384", + ), + "baseline": _ExpectedModel( + model_id="intfloat/multilingual-e5-small", + revision="614241f622f53c4eeff9890bdc4f31cfecc418b3", + pooling="mean", + query_prefix="query: ", + document_prefix="passage: ", + reduction="none_native_384", + ), +} + + +class SentenceTransformersProvider: + """Local benchmark backend available only through the optional extra.""" + + def __init__(self, *, identity: ModelIdentity, model_dir: Path) -> None: + self.identity = identity + backend = _load_sentence_transformers() + try: + self._model = backend.SentenceTransformer( + str(model_dir), + local_files_only=True, + trust_remote_code=False, + ) + except Exception: + raise BenchmarkUnavailable("local benchmark model is unavailable") from None -def build_local_provider(role: str, model_dir: Path) -> BenchmarkEmbeddingProvider: - """Load an optional benchmark-only provider plugin from the model directory. + def embed_queries( + self, values: tuple[str, ...] + ) -> tuple[tuple[float, ...], ...]: + return self._embed(values) - The plugin is deliberately outside the locked production environment. A local - model directory supplies ``context_engine_provider.py`` with ``create_provider``; - the returned provider still must expose the exact pinned identity in the report. - """ + def embed_documents( + self, values: tuple[str, ...] + ) -> tuple[tuple[float, ...], ...]: + return self._embed(values) - plugin = model_dir / "context_engine_provider.py" - if not plugin.is_file(): + def _embed(self, values: tuple[str, ...]) -> tuple[tuple[float, ...], ...]: + try: + vectors = self._model.encode( + list(values), + batch_size=self.identity.batch_size, + convert_to_numpy=True, + normalize_embeddings=self.identity.normalization == "l2", + precision=self.identity.precision, + show_progress_bar=False, + ) + return tuple( + tuple(float(value) for value in vector[:EMBEDDING_DIMENSION]) + for vector in vectors + ) + except Exception: + raise BenchmarkUnavailable("local benchmark model is unavailable") from None + + +def _load_sentence_transformers() -> Any: + try: + return importlib.import_module("sentence_transformers") + except ModuleNotFoundError: raise BenchmarkUnavailable( - f"{role} local provider is unavailable; model directory must contain " - "context_engine_provider.py" + f"benchmark backend is unavailable; install the {BENCHMARK_EXTRA} extra" + ) from None + + +def build_local_provider( + backend: SupportedBackend, + role: str, + model_dir: Path, +) -> BenchmarkEmbeddingProvider: + """Construct one of the two known models through a closed backend enum.""" + + if type(backend) is not SupportedBackend or role not in _EXPECTED_MODELS: + raise BenchmarkUnavailable("benchmark backend is unavailable") + expected = _EXPECTED_MODELS[role] + identity = _load_model_identity(model_dir, expected) + if backend is SupportedBackend.SENTENCE_TRANSFORMERS: + return SentenceTransformersProvider(identity=identity, model_dir=model_dir) + raise BenchmarkUnavailable("benchmark backend is unavailable") + + +def _load_model_identity(model_dir: Path, expected: _ExpectedModel) -> ModelIdentity: + manifest_path = model_dir / "benchmark-model-identity.json" + try: + manifest = _closed( + json.loads(manifest_path.read_text(encoding="utf-8")), + frozenset( + { + "artifactDigest", + "batchSize", + "dimension", + "documentPrefix", + "modelId", + "normalization", + "pooling", + "precision", + "queryPrefix", + "reduction", + "revision", + } + ), ) - spec = importlib.util.spec_from_file_location( - f"context_engine_embedding_benchmark_{role}", plugin - ) - if spec is None or spec.loader is None: - raise BenchmarkUnavailable(f"{role} local provider is unavailable") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - factory = getattr(module, "create_provider", None) - if not callable(factory): - raise BenchmarkUnavailable(f"{role} local provider is unavailable") - return cast(LocalProviderFactory, factory)(model_dir) + identity = ModelIdentity( + model_id=cast(str, manifest["modelId"]), + revision=cast(str, manifest["revision"]), + artifact_digest=cast(str, manifest["artifactDigest"]), + dimension=cast(int, manifest["dimension"]), + normalization=cast(str, manifest["normalization"]), + pooling=cast(str, manifest["pooling"]), + query_prefix=cast(str, manifest["queryPrefix"]), + document_prefix=cast(str, manifest["documentPrefix"]), + reduction=cast(str, manifest["reduction"]), + precision=cast(str, manifest["precision"]), + batch_size=cast(int, manifest["batchSize"]), + ) + if ( + identity.model_id != expected.model_id + or identity.revision != expected.revision + or identity.pooling != expected.pooling + or identity.query_prefix != expected.query_prefix + or identity.document_prefix != expected.document_prefix + or identity.reduction != expected.reduction + or identity.artifact_digest + != _model_artifact_digest(model_dir, excluded=manifest_path) + ): + raise BenchmarkUnavailable("model identity is unresolved") + return identity + except BenchmarkUnavailable: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError): + raise BenchmarkUnavailable("model identity is unresolved") from None + +def _model_artifact_digest(model_dir: Path, *, excluded: Path) -> str: + digest = sha256() + files = tuple(sorted(path for path in model_dir.rglob("*") if path.is_file())) + if not files or any(path.is_symlink() for path in files): + raise BenchmarkUnavailable("model identity is unresolved") + for path in files: + if path == excluded: + continue + relative = path.relative_to(model_dir).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() -def load_retrieval_judge(specification: str) -> RetrievalJudge: - """Load #129's judge through its injected adapter factory.""" +def load_retrieval_judge() -> RetrievalJudge: + """Load only the fixed #129 judge factory; caller-authored imports are denied.""" try: - module_name, separator, factory_name = specification.partition(":") - if not separator or not module_name or not factory_name: - raise ValueError - factory = getattr(importlib.import_module(module_name), factory_name) + factory = importlib.import_module("eval.retrieval_judge").create_judge if not callable(factory): raise TypeError return cast(Callable[[], RetrievalJudge], factory)() - except (ImportError, AttributeError, TypeError, ValueError): + except (ImportError, AttributeError, TypeError): raise BenchmarkUnavailable("retrieval judge is unavailable") from None @@ -75,12 +228,25 @@ def load_dataset(path: Path) -> BenchmarkDataset: """Load the strict drop-in input format used by the pending durable corpus.""" try: + raw_root = json.loads(path.read_text(encoding="utf-8")) + schema = json.loads(INPUT_SCHEMA_PATH.read_text(encoding="utf-8")) + try: + validate_json_schema_document(raw_root, schema) + except BenchmarkUnavailable: + raise BenchmarkUnavailable( + "benchmark input schema is unavailable" + ) from None root = _closed( - json.loads(path.read_text(encoding="utf-8")), - frozenset({"cases", "documents", "schemaVersion"}), + raw_root, + frozenset({"cases", "documents", "lock", "schemaVersion"}), ) - if root["schemaVersion"] != "context-engine-embedding-benchmark-input-v1": - raise BenchmarkUnavailable("benchmark dataset is unavailable") + lock = _closed(root["lock"], frozenset({"contentDigest", "profile"})) + unlocked = {key: value for key, value in root.items() if key != "lock"} + if ( + lock["profile"] != LOCK_PROFILE + or lock["contentDigest"] != dataset_content_digest(unlocked) + ): + raise BenchmarkUnavailable("benchmark dataset lock is unavailable") raw_documents = root["documents"] raw_cases = root["cases"] if type(raw_documents) is not list or type(raw_cases) is not list: @@ -117,6 +283,15 @@ def load_dataset(path: Path) -> BenchmarkDataset: raise BenchmarkUnavailable("benchmark dataset is unavailable") from None +def dataset_content_digest(document: object) -> str: + """Return the mechanical lock digest over the complete unlocked pilot.""" + + try: + return sha256(rfc8785.dumps(cast(Any, document))).hexdigest() + except (TypeError, ValueError, OverflowError): + raise BenchmarkUnavailable("benchmark dataset lock is unavailable") from None + + def _closed(value: object, fields: frozenset[str]) -> dict[str, object]: if type(value) is not dict or frozenset(value) != fields: raise BenchmarkUnavailable("benchmark dataset is unavailable") @@ -153,9 +328,14 @@ def _parser() -> argparse.ArgumentParser: commands = parser.add_subparsers(dest="command", required=True) run = commands.add_parser("run", help="run both pinned local models") run.add_argument("--dataset", type=Path, required=True) + run.add_argument( + "--backend", + type=SupportedBackend, + choices=tuple(SupportedBackend), + default=SupportedBackend.SENTENCE_TRANSFORMERS, + ) run.add_argument("--primary-model-dir", type=Path, required=True) run.add_argument("--baseline-model-dir", type=Path, required=True) - run.add_argument("--judge", required=True) run.add_argument("--output", type=Path, required=True) run.add_argument("--top-k", type=int, default=10) return parser @@ -170,12 +350,16 @@ def main(argv: Sequence[str] | None = None) -> int: report = run_benchmark( dataset=load_dataset(cast(Path, arguments.dataset)), primary=build_local_provider( - "primary", cast(Path, arguments.primary_model_dir) + cast(SupportedBackend, arguments.backend), + "primary", + cast(Path, arguments.primary_model_dir), ), baseline=build_local_provider( - "baseline", cast(Path, arguments.baseline_model_dir) + cast(SupportedBackend, arguments.backend), + "baseline", + cast(Path, arguments.baseline_model_dir), ), - judge=load_retrieval_judge(cast(str, arguments.judge)), + judge=load_retrieval_judge(), top_k=cast(int, arguments.top_k), clock=perf_counter, ) diff --git a/eval/README.md b/eval/README.md index f498394a..0a295beb 100644 --- a/eval/README.md +++ b/eval/README.md @@ -48,17 +48,20 @@ feed the release-gate report, and there is no CI threshold. `context-engine-embedding-benchmark` is an offline evaluation CLI. It compares the exact 384-dimensional primary and baseline model identities required by issue #128 and never composes an `EmbeddingProvider` into Supply or Runtime. -The main environment deliberately contains no model framework: each local, -pinned model directory supplies a benchmark-only `context_engine_provider.py` -plugin and its fully resolved identity (repository id, immutable revision, -artifact digest, dimension, normalization, pooling, prompt prefixes, reduction, -precision, and batching). +The main environment deliberately contains no model framework. The closed +`sentence-transformers` backend is installed explicitly with +`uv sync --extra benchmark`; each local pinned model directory supplies +`benchmark-model-identity.json` with its fully resolved repository id, +immutable revision, artifact digest, dimension, normalization, pooling, prompt +prefixes, reduction, precision, and batching. The runner verifies the digest +over the local model artifacts before it embeds anything. The input follows `embedding-benchmark/input.schema.json`; the output follows `embedding-benchmark/report.schema.json` and must be written below the ignored -`.context-engine/` directory. Retrieval scores are not implemented by this -runner. `--judge module:factory` injects the retrieval judge owned by issue #129, -which is solely responsible for case hit, macro/micro Evidence recall, and slice +`.context-engine/` directory. The input includes an RFC 8785/SHA-256 content +lock and any post-lock edit is refused. Retrieval scores are not implemented by +this runner: it imports the one fixed judge factory owned by issue #129, which +is solely responsible for case hit, macro/micro Evidence recall, and slice breakdowns. The maintainer corpus is private and pending delivery to a durable, @@ -68,13 +71,17 @@ an operator may run: ```bash uv run context-engine-embedding-benchmark run \ --dataset "$DURABLE_GOLDEN_ROOT/embedding-benchmark-v1.json" \ + --backend sentence-transformers \ --primary-model-dir "$PINNED_MODEL_ROOT/qwen3-embedding-0.6b" \ --baseline-model-dir "$PINNED_MODEL_ROOT/multilingual-e5-small" \ - --judge eval.retrieval_judge:create_judge \ --output .context-engine/eval/embedding-benchmark-v1.json ``` The tracked frozen result contains metrics only. It currently records `pending_corpus`; it must never contain queries, note titles, paths, excerpts, -or model weights. A losing primary is a valid benchmark outcome and is frozen as -`lose`, not treated as a runner error. +or model weights. The model verdict uses Pareto dominance across case hit, +macro Evidence recall, and micro Evidence recall: one model wins only when it is +no worse on all three and strictly better on at least one. Exact equality is a +tie; mixed wins are `inconclusive`. Per-slice results remain diagnostic and no +weighted composite or tiebreak manufactures a winner. A losing or inconclusive +primary is a valid benchmark outcome, not a runner error. diff --git a/eval/embedding-benchmark/input.schema.json b/eval/embedding-benchmark/input.schema.json index 346ec5c8..61b882b6 100644 --- a/eval/embedding-benchmark/input.schema.json +++ b/eval/embedding-benchmark/input.schema.json @@ -4,9 +4,18 @@ "title": "ContextEngine offline embedding benchmark input v1", "type": "object", "additionalProperties": false, - "required": ["schemaVersion", "documents", "cases"], + "required": ["schemaVersion", "lock", "documents", "cases"], "properties": { "schemaVersion": {"const": "context-engine-embedding-benchmark-input-v1"}, + "lock": { + "type": "object", + "additionalProperties": false, + "required": ["profile", "contentDigest"], + "properties": { + "profile": {"const": "sha256-rfc8785-v1"}, + "contentDigest": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } + }, "documents": { "type": "array", "minItems": 1, diff --git a/eval/embedding-benchmark/report.schema.json b/eval/embedding-benchmark/report.schema.json index 91b35c45..d1013c1f 100644 --- a/eval/embedding-benchmark/report.schema.json +++ b/eval/embedding-benchmark/report.schema.json @@ -24,12 +24,23 @@ "properties": {"caseHitValue": {"type": "number", "minimum": 0, "maximum": 1}} }, "comparison": { + "description": "Frozen Pareto verdict across case hit, macro Evidence recall, and micro Evidence recall. One model wins iff no worse on all three and strictly better on at least one; exact equality ties; mixed wins are inconclusive. Per-slice metrics do not enter the verdict and weighted blending is prohibited.", "type": "object", "additionalProperties": false, - "required": ["primaryAgainstModelBaseline", "primaryAgainstStandingTwinBaseline"], + "required": ["primaryAgainstModelBaseline", "primaryAgainstStandingTwinBaseline", "metricDeltas"], "properties": { - "primaryAgainstModelBaseline": {"enum": ["win", "tie", "lose"]}, - "primaryAgainstStandingTwinBaseline": {"enum": ["win", "tie", "lose"]} + "primaryAgainstModelBaseline": {"enum": ["win", "tie", "lose", "inconclusive"]}, + "primaryAgainstStandingTwinBaseline": {"enum": ["win", "tie", "lose"]}, + "metricDeltas": { + "type": "object", + "additionalProperties": false, + "required": ["caseHit", "macroEvidenceRecall", "microEvidenceRecall"], + "properties": { + "caseHit": {"type": "number"}, + "macroEvidenceRecall": {"type": "number"}, + "microEvidenceRecall": {"type": "number"} + } + } } } }, diff --git a/eval/embedding_benchmark.py b/eval/embedding_benchmark.py index 6b0f8f8b..b7ce015d 100644 --- a/eval/embedding_benchmark.py +++ b/eval/embedding_benchmark.py @@ -9,7 +9,9 @@ from dataclasses import dataclass, field from hashlib import sha256 from pathlib import Path -from typing import Final, Protocol, cast +from typing import Any, Final, Protocol, cast + +import rfc8785 EMBEDDING_DIMENSION: Final = 384 REPORT_SCHEMA_VERSION: Final = "context-engine-embedding-benchmark-report-v1" @@ -123,6 +125,7 @@ class BenchmarkDataset: documents: tuple[BenchmarkDocument, ...] = field(repr=False) cases: tuple[BenchmarkCase, ...] = field(repr=False) + locked: bool = True digest: str = field(init=False) def __post_init__(self) -> None: @@ -133,6 +136,8 @@ def __post_init__(self) -> None: or type(self.cases) is not tuple or not self.cases or any(type(value) is not BenchmarkCase for value in self.cases) + or type(self.locked) is not bool + or not self.locked ): raise BenchmarkUnavailable("benchmark dataset is unavailable") document_refs = tuple(value.document_ref for value in self.documents) @@ -281,6 +286,25 @@ def __post_init__(self) -> None: raise BenchmarkUnavailable("retrieval judge is unavailable") +@dataclass(frozen=True, slots=True) +class ModelComparison: + """Typed Pareto verdict and primary-minus-baseline metric deltas.""" + + outcome: str + deltas: Mapping[str, float] + + def __post_init__(self) -> None: + if self.outcome not in {"win", "tie", "lose", "inconclusive"}: + raise BenchmarkUnavailable("model comparison is unavailable") + if frozenset(self.deltas) != frozenset( + {"caseHit", "macroEvidenceRecall", "microEvidenceRecall"} + ) or any(not math.isfinite(value) for value in self.deltas.values()): + raise BenchmarkUnavailable("model comparison is unavailable") + + def public_document(self) -> dict[str, object]: + return {"deltas": dict(self.deltas), "outcome": self.outcome} + + class RetrievalJudge(Protocol): """Injected adapter to #129; this issue intentionally implements no metrics.""" @@ -312,8 +336,11 @@ def run_benchmark( raise BenchmarkUnavailable("baseline benchmark reduction is unavailable") primary_report = _run_model(dataset, primary, judge, top_k, clock) baseline_report = _run_model(dataset, baseline, judge, top_k, clock) - primary_hit = _metric_case_hit(primary_report) - baseline_hit = _metric_case_hit(baseline_report) + model_comparison = compare_model_metrics( + primary_report.metrics, + baseline_report.metrics, + ) + primary_hit = primary_report.metrics.case_hit.value run_document = { "datasetDigest": dataset.digest, "topK": top_k, @@ -327,23 +354,36 @@ def run_benchmark( ) return { "comparison": { - "primaryAgainstModelBaseline": _comparison(primary_hit, baseline_hit), + "metricDeltas": dict(model_comparison.deltas), + "primaryAgainstModelBaseline": model_comparison.outcome, "primaryAgainstStandingTwinBaseline": _comparison(primary_hit, 0.038), }, - "models": {"baseline": baseline_report, "primary": primary_report}, + "models": { + "baseline": baseline_report.public_document(), + "primary": primary_report.public_document(), + }, "run": {**run_document, "runIdentity": run_identity}, "schemaVersion": REPORT_SCHEMA_VERSION, "standingTwinBaseline": {"caseHitValue": 0.038}, } +@dataclass(frozen=True, slots=True) +class _ModelRun: + document: Mapping[str, object] + metrics: RetrievalMetrics + + def public_document(self) -> dict[str, object]: + return dict(self.document) + + def _run_model( dataset: BenchmarkDataset, provider: BenchmarkEmbeddingProvider, judge: RetrievalJudge, top_k: int, clock: Callable[[], float], -) -> dict[str, object]: +) -> _ModelRun: query_values = tuple( provider.identity.query_prefix + case.query for case in dataset.cases ) @@ -377,20 +417,23 @@ def _run_model( if type(metrics) is not RetrievalMetrics: raise BenchmarkUnavailable("retrieval judge is unavailable") finished = clock() - return { - "identity": provider.identity.public_document(), - "metrics": _metrics_document(metrics), - "timing": { - "documentCount": len(dataset.documents), - "perDocumentEmbedMilliseconds": max( - 0.0, - (document_finished - document_started) - * 1000 - / len(dataset.documents), - ), - "wallClockMilliseconds": max(0.0, (finished - started) * 1000), + return _ModelRun( + document={ + "identity": provider.identity.public_document(), + "metrics": _metrics_document(metrics), + "timing": { + "documentCount": len(dataset.documents), + "perDocumentEmbedMilliseconds": max( + 0.0, + (document_finished - document_started) + * 1000 + / len(dataset.documents), + ), + "wallClockMilliseconds": max(0.0, (finished - started) * 1000), + }, }, - } + metrics=metrics, + ) def _validated_vectors( @@ -476,13 +519,44 @@ def _recall_document(value: EvidenceRecallMetric) -> dict[str, object]: } -def _metric_case_hit(model_report: dict[str, object]) -> float: - metrics = _object(model_report["metrics"]) - case_hit = _object(metrics["caseHit"]) - value = case_hit["value"] - if type(value) not in {int, float}: - raise BenchmarkUnavailable("retrieval judge is unavailable") - return float(cast(int | float, value)) +def compare_model_metrics( + primary: RetrievalMetrics, + baseline: RetrievalMetrics, +) -> ModelComparison: + """Apply the frozen no-blending Pareto rule across three retrieval metrics.""" + + primary_values = ( + primary.case_hit.value, + primary.evidence_recall.macro.value, + primary.evidence_recall.micro.value, + ) + baseline_values = ( + baseline.case_hit.value, + baseline.evidence_recall.macro.value, + baseline.evidence_recall.micro.value, + ) + if primary_values == baseline_values: + outcome = "tie" + elif all( + left >= right + for left, right in zip(primary_values, baseline_values, strict=True) + ): + outcome = "win" + elif all( + left <= right + for left, right in zip(primary_values, baseline_values, strict=True) + ): + outcome = "lose" + else: + outcome = "inconclusive" + return ModelComparison( + outcome=outcome, + deltas={ + "caseHit": primary_values[0] - baseline_values[0], + "macroEvidenceRecall": primary_values[1] - baseline_values[1], + "microEvidenceRecall": primary_values[2] - baseline_values[2], + }, + ) def _comparison(left: float, right: float) -> str: @@ -527,7 +601,7 @@ def validate_report_document( try: schema = json.loads(schema_path.read_text(encoding="utf-8")) - _validate_json_schema(document, _object(schema), _object(schema)) + validate_json_schema_document(document, schema) _validate_report(document, schema) except BenchmarkUnavailable: raise @@ -551,12 +625,24 @@ def _validate_report(document: object, schema: object) -> None: ) if report["schemaVersion"] != REPORT_SCHEMA_VERSION: raise BenchmarkUnavailable("benchmark report schema is unavailable") - _closed_object( + comparison = _closed_object( report["comparison"], frozenset( - {"primaryAgainstModelBaseline", "primaryAgainstStandingTwinBaseline"} + { + "metricDeltas", + "primaryAgainstModelBaseline", + "primaryAgainstStandingTwinBaseline", + } ), ) + deltas = _closed_object( + comparison["metricDeltas"], + frozenset( + {"caseHit", "macroEvidenceRecall", "microEvidenceRecall"} + ), + ) + for value in deltas.values(): + _require_finite_number(value) run = _closed_object( report["run"], frozenset({"datasetDigest", "runIdentity", "topK"}) ) @@ -573,6 +659,13 @@ def _validate_report(document: object, schema: object) -> None: _validate_model_report(models["baseline"]) +def validate_json_schema_document(value: object, schema: object) -> None: + """Validate documents against the tracked schema vocabulary used by eval.""" + + root = _object(schema) + _validate_json_schema(value, root, root) + + def _validate_json_schema( value: object, schema: dict[str, object], @@ -640,11 +733,28 @@ def _validate_json_schema( if type(value) not in {int, float}: raise BenchmarkUnavailable("benchmark report schema is unavailable") _validate_numeric_bounds(cast(int | float, value), schema) + elif expected_type == "array": + if type(value) is not list: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + minimum_items = schema.get("minItems", 0) + if type(minimum_items) is not int or len(value) < minimum_items: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + if schema.get("uniqueItems") is True and len( + {_canonical_json(item) for item in value} + ) != len(value): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + items = schema.get("items") + if items is not None: + item_schema = _object(items) + for item in value: + _validate_json_schema(item, item_schema, root) elif expected_type is not None: raise BenchmarkUnavailable("benchmark report schema is unavailable") def _validate_numeric_bounds(value: int | float, schema: dict[str, object]) -> None: + if not math.isfinite(float(value)): + raise BenchmarkUnavailable("benchmark report schema is unavailable") minimum = schema.get("minimum") maximum = schema.get("maximum") if minimum is not None and ( @@ -657,6 +767,13 @@ def _validate_numeric_bounds(value: int | float, schema: dict[str, object]) -> N raise BenchmarkUnavailable("benchmark report schema is unavailable") +def _canonical_json(value: object) -> bytes: + try: + return rfc8785.dumps(cast(Any, value)) + except (TypeError, ValueError, OverflowError): + raise BenchmarkUnavailable("benchmark report schema is unavailable") from None + + def _validate_model_report(value: object) -> None: report = _closed_object(value, frozenset({"identity", "metrics", "timing"})) identity = _closed_object( @@ -783,3 +900,10 @@ def _require_nonnegative_number(value: object) -> None: number = float(cast(int | float, value)) if not math.isfinite(number) or number < 0.0: raise BenchmarkUnavailable("benchmark report schema is unavailable") + + +def _require_finite_number(value: object) -> None: + if type(value) not in {int, float} or not math.isfinite( + float(cast(int | float, value)) + ): + raise BenchmarkUnavailable("benchmark report schema is unavailable") diff --git a/pyproject.toml b/pyproject.toml index 6d66a984..3af03e0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,11 @@ context-engine-dogfood-eval = "applications.dogfood_evaluation:main" context-engine-embedding-benchmark = "applications.embedding_benchmark:main" context-engine-worker = "applications.worker:main" +[project.optional-dependencies] +benchmark = [ + "sentence-transformers>=5,<6", +] + [tool.hatch.build.targets.wheel] packages = ["engine", "adapters", "applications", "eval", "migrations"] @@ -49,6 +54,7 @@ files = [ "engine", "adapters", "applications", + "eval", "migrations", "scripts/provision_database_roles.py", "scripts/wait_for_database.py", diff --git a/tests/unit/test_embedding_benchmark_cli.py b/tests/unit/test_embedding_benchmark_cli.py index 3aedb673..f0c94699 100644 --- a/tests/unit/test_embedding_benchmark_cli.py +++ b/tests/unit/test_embedding_benchmark_cli.py @@ -1,13 +1,23 @@ from __future__ import annotations +import importlib import json from pathlib import Path -from typing import Any, cast +from typing import Any import pytest from applications import embedding_benchmark as cli -from eval.embedding_benchmark import BenchmarkUnavailable +from eval.embedding_benchmark import ( + BenchmarkUnavailable, + CaseHitMetric, + EvidenceRecallMetric, + MacroRecallMetric, + MicroRecallMetric, + RetrievalJudgeCase, + RetrievalMetrics, + SliceMetrics, +) from tests.unit.test_embedding_benchmark_metrics import ( RecordingJudge, SyntheticProvider, @@ -15,27 +25,110 @@ ) +class BaselineWinningJudge: + def __init__(self) -> None: + self.calls = 0 + + def evaluate_retrieval( + self, cases: tuple[RetrievalJudgeCase, ...] + ) -> RetrievalMetrics: + self.calls += 1 + value = 0.0 if self.calls == 1 else 1.0 + return RetrievalMetrics( + case_hit=CaseHitMetric(hits=int(value), total_cases=1, value=value), + evidence_recall=EvidenceRecallMetric( + macro=MacroRecallMetric(value=value), + micro=MicroRecallMetric( + hits=int(value), total_expected=1, value=value + ), + ), + per_slice={ + "single_doc": SliceMetrics( + case_hit=CaseHitMetric( + hits=int(value), total_cases=1, value=value + ), + evidence_recall=EvidenceRecallMetric( + macro=MacroRecallMetric(value=value), + micro=MicroRecallMetric( + hits=int(value), total_expected=1, value=value + ), + ), + ) + }, + ) + + def _write_dataset(path: Path) -> None: - path.write_text( - json.dumps( + document = { + "cases": [ { - "cases": [ - { - "caseRef": "case-alpha", - "expectedDocumentRefs": ["doc-alpha"], - "query": "alpha", - "slice": "single_doc", - } - ], - "documents": [ - {"documentRef": "doc-alpha", "text": "alpha"}, - {"documentRef": "doc-beta", "text": "beta"}, - ], - "schemaVersion": "context-engine-embedding-benchmark-input-v1", + "caseRef": "case-alpha", + "expectedDocumentRefs": ["doc-alpha"], + "query": "alpha", + "slice": "single_doc", } + ], + "documents": [ + {"documentRef": "doc-alpha", "text": "alpha"}, + {"documentRef": "doc-beta", "text": "beta"}, + ], + "schemaVersion": "context-engine-embedding-benchmark-input-v1", + } + content_digest = cli.dataset_content_digest(document) + document["lock"] = { + "contentDigest": content_digest, + "profile": "sha256-rfc8785-v1", + } + path.write_text(json.dumps(document), encoding="utf-8") + + +def test_loader_refuses_a_pilot_changed_after_locking(tmp_path: Path) -> None: + path = tmp_path / "dataset.json" + _write_dataset(path) + document = json.loads(path.read_text(encoding="utf-8")) + document["cases"][0]["query"] = "changed after locking" + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(BenchmarkUnavailable, match="lock is unavailable"): + cli.load_dataset(path) + + +@pytest.mark.parametrize( + "mutate", + ( + lambda document: document["cases"][0].update({"caseRef": "INVALID CASE"}), + lambda document: document["cases"][0].update({"unknown": True}), + lambda document: document["documents"].clear(), + lambda document: document["cases"][0].update( + {"expectedDocumentRefs": ["doc-alpha", "doc-alpha"]} ), - encoding="utf-8", - ) + ), +) +def test_loader_and_tracked_schema_refuse_the_same_boundary_documents( + tmp_path: Path, + mutate: Any, +) -> None: + path = tmp_path / "dataset.json" + _write_dataset(path) + document = json.loads(path.read_text(encoding="utf-8")) + mutate(document) + unlocked = {key: value for key, value in document.items() if key != "lock"} + document["lock"]["contentDigest"] = cli.dataset_content_digest(unlocked) + path.write_text(json.dumps(document), encoding="utf-8") + + with pytest.raises(BenchmarkUnavailable, match="schema is unavailable"): + cli.load_dataset(path) + + +def test_locked_document_accepted_by_schema_is_accepted_by_loader( + tmp_path: Path, +) -> None: + path = tmp_path / "dataset.json" + _write_dataset(path) + + dataset = cli.load_dataset(path) + + assert dataset.locked is True def test_cli_run_writes_a_schema_valid_report_for_a_synthetic_set( @@ -46,7 +139,9 @@ def test_cli_run_writes_a_schema_valid_report_for_a_synthetic_set( output_path = Path(".context-engine") / "test" / "embedding-benchmark.json" _write_dataset(dataset_path) - def provider(role: str, _model_dir: Path) -> SyntheticProvider: + def provider( + _backend: cli.SupportedBackend, role: str, _model_dir: Path + ) -> SyntheticProvider: model_id = ( "Qwen/Qwen3-Embedding-0.6B" if role == "primary" @@ -55,7 +150,7 @@ def provider(role: str, _model_dir: Path) -> SyntheticProvider: return SyntheticProvider(_identity(model_id)) monkeypatch.setattr(cli, "build_local_provider", provider) - monkeypatch.setattr(cli, "load_retrieval_judge", lambda _spec: RecordingJudge()) + monkeypatch.setattr(cli, "load_retrieval_judge", lambda: RecordingJudge()) result = cli.main( [ @@ -66,8 +161,6 @@ def provider(role: str, _model_dir: Path) -> SyntheticProvider: str(tmp_path / "primary"), "--baseline-model-dir", str(tmp_path / "baseline"), - "--judge", - "eval.retrieval_judge:create_judge", "--output", str(output_path), "--top-k", @@ -99,8 +192,6 @@ def test_cli_refuses_report_output_outside_the_ignored_state_directory( str(tmp_path / "primary"), "--baseline-model-dir", str(tmp_path / "baseline"), - "--judge", - "eval.retrieval_judge:create_judge", "--output", str(Path("tests") / "tracked.json"), ] @@ -110,9 +201,54 @@ def test_cli_refuses_report_output_outside_the_ignored_state_directory( assert not (Path("tests") / "tracked.json").exists() +def test_cli_reports_a_baseline_win_with_success_exit_status( + tmp_path: Path, + monkeypatch: Any, +) -> None: + dataset_path = tmp_path / "dataset.json" + output_path = Path(".context-engine") / "test" / "embedding-loss.json" + _write_dataset(dataset_path) + + def provider( + _backend: cli.SupportedBackend, role: str, _model_dir: Path + ) -> SyntheticProvider: + model_id = ( + "Qwen/Qwen3-Embedding-0.6B" + if role == "primary" + else "intfloat/multilingual-e5-small" + ) + return SyntheticProvider(_identity(model_id)) + + monkeypatch.setattr(cli, "build_local_provider", provider) + monkeypatch.setattr(cli, "load_retrieval_judge", lambda: BaselineWinningJudge()) + + result = cli.main( + [ + "run", + "--dataset", + str(dataset_path), + "--backend", + "sentence-transformers", + "--primary-model-dir", + str(tmp_path / "primary"), + "--baseline-model-dir", + str(tmp_path / "baseline"), + "--output", + str(output_path), + "--top-k", + "1", + ] + ) + + report = json.loads(output_path.read_text(encoding="utf-8")) + assert result == 0 + assert report["comparison"]["primaryAgainstModelBaseline"] == "lose" + output_path.unlink() + + def test_cli_refuses_a_missing_retrieval_judge() -> None: with pytest.raises(BenchmarkUnavailable, match="judge is unavailable"): - cli.load_retrieval_judge("eval.retrieval_judge:create_judge") + cli.load_retrieval_judge() def test_output_cannot_escape_the_workspace_state_directory(tmp_path: Path) -> None: @@ -126,18 +262,34 @@ def test_output_cannot_escape_the_workspace_state_directory(tmp_path: Path) -> N ) -def test_local_plugin_provider_is_loaded_only_from_the_requested_model_directory( - tmp_path: Path, +def test_missing_real_backend_names_the_optional_extra( + monkeypatch: Any, ) -> None: - model_dir = tmp_path / "model" - model_dir.mkdir() - plugin = model_dir / "context_engine_provider.py" - plugin.write_text( - "def create_provider(model_dir):\n" - " return {'requested': str(model_dir)}\n", - encoding="utf-8", - ) + def missing(_name: str) -> Any: + raise ModuleNotFoundError - provider = cli.build_local_provider("primary", model_dir) + monkeypatch.setattr(importlib, "import_module", missing) - assert cast(Any, provider) == {"requested": str(model_dir)} + with pytest.raises(BenchmarkUnavailable, match=r"context-engine\[benchmark\]"): + cli._load_sentence_transformers() + + +def test_closed_backend_enum_refuses_arbitrary_import_paths() -> None: + parser = cli._parser() + + with pytest.raises(SystemExit, match="2"): + parser.parse_args( + [ + "run", + "--dataset", + "dataset.json", + "--backend", + "some.module:factory", + "--primary-model-dir", + "primary", + "--baseline-model-dir", + "baseline", + "--output", + ".context-engine/report.json", + ] + ) diff --git a/tests/unit/test_embedding_benchmark_metrics.py b/tests/unit/test_embedding_benchmark_metrics.py index fcda03fd..4f8d1e5b 100644 --- a/tests/unit/test_embedding_benchmark_metrics.py +++ b/tests/unit/test_embedding_benchmark_metrics.py @@ -19,6 +19,7 @@ RetrievalJudgeCase, RetrievalMetrics, SliceMetrics, + compare_model_metrics, run_benchmark, ) @@ -170,6 +171,56 @@ def evaluate_retrieval( ] == "lose" +@pytest.mark.parametrize( + ("primary", "baseline", "expected"), + ( + ((0.9, 0.8, 0.7), (0.8, 0.7, 0.6), "win"), + ((0.8, 0.7, 0.6), (0.9, 0.8, 0.7), "lose"), + ((0.8, 0.8, 0.8), (0.8, 0.8, 0.8), "tie"), + ((0.9, 0.6, 0.8), (0.8, 0.7, 0.8), "inconclusive"), + ), +) +def test_model_verdict_uses_pareto_dominance_across_all_retrieval_metrics( + primary: tuple[float, float, float], + baseline: tuple[float, float, float], + expected: str, +) -> None: + verdict = compare_model_metrics( + _retrieval_metrics(*primary), + _retrieval_metrics(*baseline), + ) + + assert verdict.outcome == expected + assert verdict.deltas == { + "caseHit": pytest.approx(primary[0] - baseline[0]), + "macroEvidenceRecall": pytest.approx(primary[1] - baseline[1]), + "microEvidenceRecall": pytest.approx(primary[2] - baseline[2]), + } + + +def _retrieval_metrics( + case_hit: float, + macro: float, + micro: float, +) -> RetrievalMetrics: + return RetrievalMetrics( + case_hit=CaseHitMetric(hits=0, total_cases=1, value=case_hit), + evidence_recall=EvidenceRecallMetric( + macro=MacroRecallMetric(value=macro), + micro=MicroRecallMetric(hits=0, total_expected=1, value=micro), + ), + per_slice={ + "single_doc": SliceMetrics( + case_hit=CaseHitMetric(hits=0, total_cases=1, value=case_hit), + evidence_recall=EvidenceRecallMetric( + macro=MacroRecallMetric(value=macro), + micro=MicroRecallMetric(hits=0, total_expected=1, value=micro), + ), + ) + }, + ) + + def test_runner_requires_the_declared_384_dimension_reduction_per_model() -> None: primary = SyntheticProvider( replace( diff --git a/tests/unit/test_embedding_benchmark_report_schema.py b/tests/unit/test_embedding_benchmark_report_schema.py index 94f88ea6..8db0f640 100644 --- a/tests/unit/test_embedding_benchmark_report_schema.py +++ b/tests/unit/test_embedding_benchmark_report_schema.py @@ -67,6 +67,11 @@ def _report() -> dict[str, object]: ) return { "comparison": { + "metricDeltas": { + "caseHit": 0.0, + "macroEvidenceRecall": 0.0, + "microEvidenceRecall": 0.0, + }, "primaryAgainstModelBaseline": "tie", "primaryAgainstStandingTwinBaseline": "win", }, diff --git a/uv.lock b/uv.lock index 41708788..41badb98 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/4a/4c61d4c84cfd9befb6fa08a702535b27b21fff08c946bc2f6139decbf7f7/alembic-1.16.5-py3-none-any.whl", hash = "sha256:e845dfe090c5ffa7b92593ae6687c5cb1a101e91fa53868497dbd79847f9dbe3", size = 247355, upload-time = "2025-08-27T18:02:07.37Z" }, ] +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -108,6 +117,11 @@ dependencies = [ { name = "uvicorn" }, ] +[package.optional-dependencies] +benchmark = [ + { name = "sentence-transformers" }, +] + [package.dev-dependencies] dev = [ { name = "httpx" }, @@ -125,10 +139,12 @@ requires-dist = [ { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<3.3" }, { name = "pydantic", specifier = ">=2.13,<2.14" }, { name = "rfc8785", specifier = ">=0.1.4,<0.2" }, + { name = "sentence-transformers", marker = "extra == 'benchmark'", specifier = ">=5,<6" }, { name = "sqlalchemy", specifier = ">=2.0,<2.1" }, { name = "starlette", specifier = ">=0.48,<0.49" }, { name = "uvicorn", specifier = ">=0.35,<0.36" }, ] +provides-extras = ["benchmark"] [package.metadata.requires-dev] dev = [ @@ -176,6 +192,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, ] +[[package]] +name = "cuda-bindings" +version = "13.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/6e/2394f8163360f8391f8f1b7e72d300a82724edb81a7b7084c799fbd4c91f/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf", size = 5920504, upload-time = "2026-05-29T23:11:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/34/c2/ef9b6a63f7dc432712a462c816662e662e00d38caa9b861c8c2588195d03/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7", size = 6476660, upload-time = "2026-05-29T23:11:59.188Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/b4/d088047afe39827556df21118cac9ffd20cc3f968c99a7681494d1eb333c/cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51", size = 54591, upload-time = "2026-07-21T15:03:56.224Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.3.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] +cudart = [ + { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] +cufft = [ + { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] +cufile = [ + { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +cupti = [ + { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] +curand = [ + { name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] +cusolver = [ + { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] +cusparse = [ + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] +nvtx = [ + { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, +] + [[package]] name = "fastapi" version = "0.116.2" @@ -190,6 +275,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/e4/c543271a8018874b7f682bf6156863c416e1334b8ed3e51a69495c5d4360/fastapi-0.116.2-py3-none-any.whl", hash = "sha256:c3a7a8fb830b05f7e087d920e0d786ca1fc9892eb4e9a84b227be4c1bc7569db", size = 95670, upload-time = "2025-09-16T18:29:21.329Z" }, ] +[[package]] +name = "filelock" +version = "3.32.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/80/8232b582c4b318b817cf1274ba74976b07b34d35ef439b3eb948f98645a1/filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402", size = 213757, upload-time = "2026-07-21T13:17:42.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, +] + [[package]] name = "greenlet" version = "3.5.3" @@ -217,6 +320,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "hf-xet" +version = "1.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/63/39/67be8d71f900d9a55761b6022821d6679fb56c64f1b6063d5af2c2606727/hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47", size = 903674, upload-time = "2026-07-16T17:29:56.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/ba/2b70603c7552db82baeb2623e2336898304a17328845151be4fe1f48d420/hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799", size = 4033760, upload-time = "2026-07-16T17:29:43.884Z" }, + { url = "https://files.pythonhosted.org/packages/60/ac/b097a86a1e4a6098f3a79382643ab09d5733d87ccc864877ad1e12b49b70/hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed", size = 3841438, upload-time = "2026-07-16T17:29:45.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/35/db860aa3a0780660324a506ad4b3d322ddc6ecbba4b9340aed0942cbf21c/hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c", size = 4428006, upload-time = "2026-07-16T17:29:46.996Z" }, + { url = "https://files.pythonhosted.org/packages/af/6b/832dd980af4b0c3ae0660e309285f2ffcdff2faa38129390dbb47aa4a3f9/hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025", size = 4221099, upload-time = "2026-07-16T17:29:48.525Z" }, + { url = "https://files.pythonhosted.org/packages/9e/05/ae50f0d34e3254e6c3e208beb2519f6b8673016fc4b3643badaf6450d186/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f", size = 4420766, upload-time = "2026-07-16T17:29:50.092Z" }, + { url = "https://files.pythonhosted.org/packages/07/a9/c050bc2743a2bcd68928bfee157b08681667a164a24ec95fbfcfcd717e08/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097", size = 4636716, upload-time = "2026-07-16T17:29:51.62Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f8/68b01c5c2edb56ac9a67b3d076ffddcb90867abaee923923eb34e7a14e76/hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65", size = 3988373, upload-time = "2026-07-16T17:29:53.395Z" }, + { url = "https://files.pythonhosted.org/packages/39/c6/988383e9dc17294d536fcbcd6fd16eed882e411ad16c954984a53e47b09c/hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d", size = 3816957, upload-time = "2026-07-16T17:29:54.991Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -245,6 +364,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "huggingface-hub" +version = "1.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/50/db3771a6e4fad4bd28fb055d4363b51cb0ae98c1aa504b79d41fdcab5483/huggingface_hub-1.25.1.tar.gz", hash = "sha256:21129595ca7a753be479b319913e22cc8808361ac118bd76cc413db831b28a99", size = 928426, upload-time = "2026-07-27T09:24:10.117Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/3f/21e816831c6d16f88a6c784974413fa0421ce8a5d04380c2666ed5b503e5/huggingface_hub-1.25.1-py3-none-any.whl", hash = "sha256:004d4e70350517e24c68a7dbb7dc5e40b2b6aefef8f94bf7a85f6f9835102ea5", size = 774909, upload-time = "2026-07-27T09:24:08.079Z" }, +] + [[package]] name = "hypothesis" version = "6.157.2" @@ -290,6 +429,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + [[package]] name = "mako" version = "1.3.12" @@ -302,6 +462,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -332,6 +504,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + [[package]] name = "mypy" version = "1.17.1" @@ -361,6 +551,195 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "narwhals" +version = "2.24.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/58946e5aab18393e793bd4add6985b95d0e01c3a2d832f38f54468b10dcd/narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d", size = 661143, upload-time = "2026-07-13T10:49:19.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489", size = 461030, upload-time = "2026-07-13T10:49:17.571Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "numpy" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, + { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, + { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, + { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, +] + +[[package]] +name = "nvidia-cublas" +version = "13.1.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cuda-nvrtc" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436", size = 423138758, upload-time = "2026-04-08T18:46:58.655Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.20.0.48" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, + { url = "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304", size = 366173588, upload-time = "2026-03-09T19:29:34.474Z" }, +] + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, +] + +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, +] + +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, + { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.29.7" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/0d/daf50d44177ee0cbc7ff0a0c91eb5ff676c82be42f9a970bc7597f440c3a/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5", size = 206014712, upload-time = "2026-03-03T05:34:20.843Z" }, + { url = "https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d", size = 205976000, upload-time = "2026-03-03T05:36:24.472Z" }, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.3.33" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5", size = 40742423, upload-time = "2026-05-26T16:54:51.613Z" }, + { url = "https://files.pythonhosted.org/packages/69/30/45414e35ff2eee7db3da037e5707037ccf9d2b5218ffbdb055ea4d5aa98a/nvidia_nvjitlink-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ce48b37dfeb3cb1eae4cf85adacb47d7a6539ea2272870c9a3628ce275c2037e", size = 39168635, upload-time = "2026-05-26T16:54:13.906Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -496,6 +875,64 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, +] + +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, +] + [[package]] name = "rfc8785" version = "0.1.4" @@ -505,6 +942,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/78/119878110660b2ad709888c8a1614fce7e2fab39080ab960656dc8605bf6/rfc8785-0.1.4-py3-none-any.whl", hash = "sha256:520d690b448ecf0703691c76e1a34a24ddcd4fc5bc41d589cb7c58ec651bcd48", size = 9240, upload-time = "2024-09-27T16:33:29.683Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "ruff" version = "0.12.12" @@ -531,6 +981,109 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/7e/61c42657f6e4614a4258f1c3b0c5b93adc4d1f8575f5229d1906b483099b/ruff-0.12.12-py3-none-win_arm64.whl", hash = "sha256:2a8199cab4ce4d72d158319b63370abf60991495fb733db96cd923a34c52d093", size = 12256762, upload-time = "2025-09-04T16:50:15.737Z" }, ] +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + +[[package]] +name = "scikit-learn" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib" }, + { name = "narwhals" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/01/cf3310626b6d48d3e9be69a1223f9180360b5e6edb045f50fade723ce494/scikit_learn-1.9.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:80746d63bd4b6eaca54d36fe5feaf4d28bb38dc6f9470f81c7cad7c40155f119", size = 8705188, upload-time = "2026-06-02T11:53:41.964Z" }, + { url = "https://files.pythonhosted.org/packages/3e/04/5acd7ae280c5f93b6ac5ef6cdec14eef4c8d1cd91d85b3292989c94d96b1/scikit_learn-1.9.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5b934c45c252844a91d69fda3a34cff5e7307e1db10d77cb10a3980312c74713", size = 8228299, upload-time = "2026-06-02T11:53:44.817Z" }, + { url = "https://files.pythonhosted.org/packages/0c/39/ffe829a5b8ecb40a518724a997794657fdc354ada5e8fe8e64d998c0bac9/scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:38c3dcb9a1ffb85505ec53d54c7b4aea0cff70050425a7760c2af661ac85df05", size = 8789690, upload-time = "2026-06-02T11:53:47.461Z" }, + { url = "https://files.pythonhosted.org/packages/1f/88/8dab5de10c638c083772a6be83a3d8106ced492f74a928c8693638e5bb50/scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da76d09304a4706db7cc1e3ebaa3b6b98a67365cc11d2996c4f1e58ba47df714", size = 9087723, upload-time = "2026-06-02T11:53:50.702Z" }, + { url = "https://files.pythonhosted.org/packages/20/3f/7917ca72464038f6240ec70c29f94862d08a34a74291ae4d4ec5eb8186a0/scikit_learn-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:5808d98f15c6bf6d9d96d2348c1997392a5888ce7097e664105f930c4bca1277", size = 8184330, upload-time = "2026-06-02T11:53:53.396Z" }, + { url = "https://files.pythonhosted.org/packages/78/c7/15739eb2f61fda3c54639e9942414e5a19ad8a8d1f5a3266afad7cb7df80/scikit_learn-1.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:d77f54c017633791bc0225a43e2f8d03745fdcfe4880268fcc4df15f505dec2e", size = 7840653, upload-time = "2026-06-02T11:53:56.035Z" }, +] + +[[package]] +name = "scipy" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/52/9c0136c2de7ae0779b7b366447766cec6d9f0702c56bb8ffeb04c8fd3af4/scipy-1.18.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:09143f676d157d9f546d663504ef9c1becb819824f1afc018814176411942446", size = 31036107, upload-time = "2026-06-19T15:00:14.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/73/0291a64843270f4efb86cdcf2ee0f2048631b65ec6b405398b2b4dbf11bf/scipy-1.18.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5efe260f69417b97ddae455bfb5a95e8359f7f66ad7fa9522a60feb66f169520", size = 28663303, upload-time = "2026-06-19T15:00:16.819Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0f/10ffa0b697a572f4e0d48b92a88895d366422f019f723e7e14a84c050dac/scipy-1.18.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:68363b7eaacd8b5dd426df56d782cc156468ac79a127a1b87ca597d6e2e82197", size = 20404960, upload-time = "2026-06-19T15:00:19.635Z" }, + { url = "https://files.pythonhosted.org/packages/7e/d2/e896cea21ba8edd6c81d4c55b1ffcc717e79698dcbebf9641b4cfb4c6622/scipy-1.18.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:c5557d8be5da8e41353fcd4d21491fdbab83b062fc579e94dc09a7c8ab4f669b", size = 23034074, upload-time = "2026-06-19T15:00:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b2/e83ea34279a52c03374477c74006256ec78df65fc877baa4617d6de1d202/scipy-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d13bca67c096d89fb95ced0d8921807300fce0275643aef9533cc63a0773468", size = 33942038, upload-time = "2026-06-19T15:00:24.964Z" }, + { url = "https://files.pythonhosted.org/packages/f6/af/e8fe5fb136f51e2b01678b92cb4106d10d8cd68ec147ead2e7cb0ac75398/scipy-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a46f9273dbd0eb1cefba61c9b8648b4dfe3cbc14a080176f9a73e44b8336dc7f", size = 35266390, upload-time = "2026-06-19T15:00:28.059Z" }, + { url = "https://files.pythonhosted.org/packages/3a/49/2c5cbb907b56695fc67517811d1db234dfd83381a84814ec220aded2794d/scipy-1.18.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5aba46108853ddfc77906b6557aac839d2b52e900c1d72a1180adaaab58d265f", size = 35551324, upload-time = "2026-06-19T15:00:31.014Z" }, + { url = "https://files.pythonhosted.org/packages/bb/73/eda39f7a2d306ff0ffc574afd13c0bbb6d10a603d9a413998ee269487a80/scipy-1.18.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b6f758e35f12757b5d95c00bc6de2438e229c2664b7a92e96f205959d9f2dfa4", size = 37404785, upload-time = "2026-06-19T15:00:34.072Z" }, + { url = "https://files.pythonhosted.org/packages/b7/d2/ae881ee28d014f38e0ccbfd974a06a919ba9af34f1f74bf42b5301891d63/scipy-1.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:1afac4a847207c7ff8efd321734a50b06d0280b3b2a2c0fc2f413101747ad7c7", size = 36554943, upload-time = "2026-06-19T15:00:36.903Z" }, + { url = "https://files.pythonhosted.org/packages/70/3a/21154e2d54eb3639c6bf4dbae2e531c68356bfe95990daa30df33b30d556/scipy-1.18.0-cp313-cp313-win_arm64.whl", hash = "sha256:c5dbddf60e58c2312316d097271a8e73d40eaf2eabfa4d95ed7d3695bbf2ce7b", size = 24350911, upload-time = "2026-06-19T15:00:40.062Z" }, +] + +[[package]] +name = "sentence-transformers" +version = "5.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "scikit-learn" }, + { name = "scipy" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "transformers" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/80/573ab31b77bdfa8f18051188adff3405e928386287cd6f756eff5777dd82/sentence_transformers-5.6.1.tar.gz", hash = "sha256:16af5d682ef66672b076d58599a23905800e850ec2bfb1865938306bf684ad72", size = 452185, upload-time = "2026-07-23T14:40:41.589Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/ad/8f73f512dc7ad4031d2b64cbb67f70bdfb355756afbe0db610a5146415c1/sentence_transformers-5.6.1-py3-none-any.whl", hash = "sha256:cefbb17b6325a982a4732c8c49fb013375392687049d1de3d435c4b04060680b", size = 596677, upload-time = "2026-07-23T14:40:40.312Z" }, +] + +[[package]] +name = "setuptools" +version = "83.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -572,6 +1125,136 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/72/2db2f49247d0a18b4f1bb9a5a39a0162869acf235f3a96418363947b3d46/starlette-0.48.0-py3-none-any.whl", hash = "sha256:0764ca97b097582558ecb498132ed0c7d942f233f365b86ba37770e026510659", size = 73736, upload-time = "2025-09-13T08:41:03.869Z" }, ] +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, + { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, + { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, +] + +[[package]] +name = "torch" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/fa/c1c10b7aff4a9a3e8956d4f0a5f468fa6db7abc3208805719076772b4833/torch-2.13.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:33449899ce5496c1b84b4853179d94fd102028ae1407314d9fb956bb79e70d09", size = 111213743, upload-time = "2026-07-08T16:03:28.579Z" }, + { url = "https://files.pythonhosted.org/packages/11/18/9ecb37b56293a0be8d80f810bf672a72fe7e02f8b475d5ef1b9bf8a0d748/torch-2.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1e09d6a722504957c694faceca843acde562786df1144ebcc5a74075ec7f6005", size = 427213008, upload-time = "2026-07-08T16:03:44.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/5a/7c50ba1b7b713d71d34669c6d13dab0a11531a3eceb0307a5162dbfec0f7/torch-2.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a3a9a21312872af8a26950b2c15680335a386a1f56ed03e780653d78b9607e9e", size = 526602329, upload-time = "2026-07-08T16:03:12.649Z" }, + { url = "https://files.pythonhosted.org/packages/91/3d/e7adcc6aaf36961cd18f56cf8ad0f3058c3a5c84ccf391762176c94581b8/torch-2.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:49b58f1e2c52440abb6f17c28f0335fe6c6d01ad1a7f55b0183b81e4b34d64e6", size = 122057920, upload-time = "2026-07-08T16:03:01.808Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, +] + +[[package]] +name = "transformers" +version = "5.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/fb/2a2ba88f325e68a921d8b69ff63b477830b2e73ade9a3c8c8cab2f06d741/transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173", size = 9295927, upload-time = "2026-07-16T09:41:57.773Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/67/8d85ca2323233ae3c0365a659c4e52ee1f587b440e4bc577e7d8e4416d0f/transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725", size = 11625234, upload-time = "2026-07-16T09:41:54.143Z" }, +] + +[[package]] +name = "triton" +version = "3.7.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/42/c5089d4d9327fcd1e862c599cc2927f39418f84dd11a84cb2ccff9d4787a/triton-3.7.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cdbfc09d9ec58bc5e68321525653220de7515c199e7a8097a97c85e62b52cd0a", size = 184694629, upload-time = "2026-06-17T20:03:53.444Z" }, + { url = "https://files.pythonhosted.org/packages/07/42/2c3ac59253ae8892b6f307875263dd23dc875cdf732d3aea40d6d41fb7cb/triton-3.7.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58c0e131da05134a2a4788ccbcc0c1105cf0f54c8e98f19e34cd465396dc15eb", size = 197729241, upload-time = "2026-06-17T19:53:27.801Z" }, +] + +[[package]] +name = "typer" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From db9c0180a3d2313aec89d1110a4b8a27ef39249c Mon Sep 17 00:00:00 2001 From: stone Date: Wed, 29 Jul 2026 22:51:46 +0800 Subject: [PATCH 4/7] fix: enforce benchmark trust contracts --- applications/embedding_benchmark.py | 214 +++++++-------- eval/README.md | 36 ++- eval/embedding-benchmark/input.schema.json | 6 +- eval/embedding-benchmark/model-registry.json | 60 +++++ .../model-registry.schema.json | 64 +++++ eval/embedding-benchmark/report.schema.json | 6 +- eval/embedding_benchmark.py | 111 ++++++-- tests/unit/test_embedding_benchmark_cli.py | 36 ++- .../unit/test_embedding_benchmark_metrics.py | 17 +- ...test_embedding_benchmark_model_registry.py | 248 ++++++++++++++++++ .../test_embedding_benchmark_report_schema.py | 92 +++++++ ...t_embedding_benchmark_runtime_untouched.py | 12 + 12 files changed, 747 insertions(+), 155 deletions(-) create mode 100644 eval/embedding-benchmark/model-registry.json create mode 100644 eval/embedding-benchmark/model-registry.schema.json create mode 100644 tests/unit/test_embedding_benchmark_model_registry.py diff --git a/applications/embedding_benchmark.py b/applications/embedding_benchmark.py index 8622dd4e..e348ffe9 100644 --- a/applications/embedding_benchmark.py +++ b/applications/embedding_benchmark.py @@ -7,7 +7,6 @@ import json import sys from collections.abc import Callable, Sequence -from dataclasses import dataclass from enum import StrEnum from hashlib import sha256 from pathlib import Path @@ -23,6 +22,7 @@ BenchmarkDocument, BenchmarkEmbeddingProvider, BenchmarkUnavailable, + DatasetLockProfile, ModelIdentity, RetrievalJudge, run_benchmark, @@ -33,8 +33,11 @@ EVAL_ROOT = Path(__file__).resolve().parents[1] / "eval" / "embedding-benchmark" INPUT_SCHEMA_PATH = EVAL_ROOT / "input.schema.json" REPORT_SCHEMA_PATH = EVAL_ROOT / "report.schema.json" -LOCK_PROFILE = "sha256-rfc8785-v1" +MODEL_REGISTRY_PATH = EVAL_ROOT / "model-registry.json" +MODEL_REGISTRY_SCHEMA_PATH = EVAL_ROOT / "model-registry.schema.json" +LOCK_PROFILE = DatasetLockProfile.ACCIDENTAL_EDIT_DETECTION BENCHMARK_EXTRA = "context-engine[benchmark]" +_IDENTITY_OVERRIDE_FILENAME = "benchmark-model-identity.json" class SupportedBackend(StrEnum): @@ -43,39 +46,6 @@ class SupportedBackend(StrEnum): SENTENCE_TRANSFORMERS = "sentence-transformers" -@dataclass(frozen=True, slots=True) -class _ExpectedModel: - model_id: str - revision: str - pooling: str - query_prefix: str - document_prefix: str - reduction: str - - -_EXPECTED_MODELS = { - "primary": _ExpectedModel( - model_id="Qwen/Qwen3-Embedding-0.6B", - revision="97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3", - pooling="last_token", - query_prefix=( - "Instruct: Given a web search query, retrieve relevant passages that " - "answer the query\nQuery:" - ), - document_prefix="", - reduction="matryoshka_truncate_384", - ), - "baseline": _ExpectedModel( - model_id="intfloat/multilingual-e5-small", - revision="614241f622f53c4eeff9890bdc4f31cfecc418b3", - pooling="mean", - query_prefix="query: ", - document_prefix="passage: ", - reduction="none_native_384", - ), -} - - class SentenceTransformersProvider: """Local benchmark backend available only through the optional extra.""" @@ -135,58 +105,79 @@ def build_local_provider( ) -> BenchmarkEmbeddingProvider: """Construct one of the two known models through a closed backend enum.""" - if type(backend) is not SupportedBackend or role not in _EXPECTED_MODELS: + if type(backend) is not SupportedBackend or role not in {"primary", "baseline"}: raise BenchmarkUnavailable("benchmark backend is unavailable") - expected = _EXPECTED_MODELS[role] - identity = _load_model_identity(model_dir, expected) + identity = load_registered_model_identity( + role=role, + backend=backend, + model_dir=model_dir, + ) if backend is SupportedBackend.SENTENCE_TRANSFORMERS: return SentenceTransformersProvider(identity=identity, model_dir=model_dir) raise BenchmarkUnavailable("benchmark backend is unavailable") -def _load_model_identity(model_dir: Path, expected: _ExpectedModel) -> ModelIdentity: - manifest_path = model_dir / "benchmark-model-identity.json" +def load_registered_model_identity( + *, + role: str, + backend: SupportedBackend, + model_dir: Path, +) -> ModelIdentity: + """Resolve identity only from the tracked registry and verify local bytes.""" + + override_path = model_dir / _IDENTITY_OVERRIDE_FILENAME + if override_path.exists(): + raise BenchmarkUnavailable("local model identity override is unavailable") try: - manifest = _closed( - json.loads(manifest_path.read_text(encoding="utf-8")), - frozenset( - { - "artifactDigest", - "batchSize", - "dimension", - "documentPrefix", - "modelId", - "normalization", - "pooling", - "precision", - "queryPrefix", - "reduction", - "revision", - } - ), + registry = json.loads(MODEL_REGISTRY_PATH.read_text(encoding="utf-8")) + registry_schema = json.loads( + MODEL_REGISTRY_SCHEMA_PATH.read_text(encoding="utf-8") + ) + try: + validate_json_schema_document(registry, registry_schema) + except BenchmarkUnavailable: + raise BenchmarkUnavailable("model identity is unresolved") from None + registry_root = cast(dict[str, object], registry) + if ( + registry_root["schemaVersion"] + != "context-engine-embedding-model-registry-v1" + ): + raise BenchmarkUnavailable("model identity is unresolved") + models = cast(dict[str, object], registry_root["models"]) + if ( + role not in {"primary", "baseline"} + or frozenset(models).difference({"primary", "baseline"}) + or role not in models + ): + raise BenchmarkUnavailable("model identity is unresolved") + model = cast(dict[str, object], models[role]) + if model["backend"] != backend.value: + raise BenchmarkUnavailable("model identity is unresolved") + artifact_documents = cast(list[object], model["artifacts"]) + expected_artifacts = tuple( + ( + cast(str, cast(dict[str, object], value)["path"]), + cast(str, cast(dict[str, object], value)["sha256"]), + ) + for value in artifact_documents ) + identity_document = cast(dict[str, object], model["identity"]) identity = ModelIdentity( - model_id=cast(str, manifest["modelId"]), - revision=cast(str, manifest["revision"]), - artifact_digest=cast(str, manifest["artifactDigest"]), - dimension=cast(int, manifest["dimension"]), - normalization=cast(str, manifest["normalization"]), - pooling=cast(str, manifest["pooling"]), - query_prefix=cast(str, manifest["queryPrefix"]), - document_prefix=cast(str, manifest["documentPrefix"]), - reduction=cast(str, manifest["reduction"]), - precision=cast(str, manifest["precision"]), - batch_size=cast(int, manifest["batchSize"]), + model_id=cast(str, identity_document["modelId"]), + revision=cast(str, identity_document["revision"]), + artifact_digest=cast(str, identity_document["artifactDigest"]), + dimension=cast(int, identity_document["dimension"]), + normalization=cast(str, identity_document["normalization"]), + pooling=cast(str, identity_document["pooling"]), + query_prefix=cast(str, identity_document["queryPrefix"]), + document_prefix=cast(str, identity_document["documentPrefix"]), + reduction=cast(str, identity_document["reduction"]), + precision=cast(str, identity_document["precision"]), + batch_size=cast(int, identity_document["batchSize"]), ) - if ( - identity.model_id != expected.model_id - or identity.revision != expected.revision - or identity.pooling != expected.pooling - or identity.query_prefix != expected.query_prefix - or identity.document_prefix != expected.document_prefix - or identity.reduction != expected.reduction - or identity.artifact_digest - != _model_artifact_digest(model_dir, excluded=manifest_path) + if identity.artifact_digest != _model_artifact_digest( + model_dir, + expected_artifacts=expected_artifacts, ): raise BenchmarkUnavailable("model identity is unresolved") return identity @@ -196,21 +187,34 @@ def _load_model_identity(model_dir: Path, expected: _ExpectedModel) -> ModelIden raise BenchmarkUnavailable("model identity is unresolved") from None -def _model_artifact_digest(model_dir: Path, *, excluded: Path) -> str: - digest = sha256() +def _model_artifact_digest( + model_dir: Path, + *, + expected_artifacts: tuple[tuple[str, str], ...], +) -> str: files = tuple(sorted(path for path in model_dir.rglob("*") if path.is_file())) - if not files or any(path.is_symlink() for path in files): + if ( + not files + or any(path.is_symlink() for path in files) + or any(not path.resolve().is_relative_to(model_dir.resolve()) for path in files) + ): raise BenchmarkUnavailable("model identity is unresolved") - for path in files: - if path == excluded: - continue - relative = path.relative_to(model_dir).as_posix().encode("utf-8") - digest.update(len(relative).to_bytes(8, "big")) - digest.update(relative) + actual_paths = tuple(path.relative_to(model_dir).as_posix() for path in files) + if actual_paths != tuple(path for path, _digest in expected_artifacts): + raise BenchmarkUnavailable("model identity is unresolved") + manifest: list[dict[str, str]] = [] + for path, (relative_path, expected_digest) in zip( + files, expected_artifacts, strict=True + ): + file_digest = sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() + file_digest.update(block) + actual_digest = file_digest.hexdigest() + if actual_digest != expected_digest: + raise BenchmarkUnavailable("model identity is unresolved") + manifest.append({"path": relative_path, "sha256": actual_digest}) + return sha256(rfc8785.dumps(manifest)).hexdigest() def load_retrieval_judge() -> RetrievalJudge: @@ -236,14 +240,11 @@ def load_dataset(path: Path) -> BenchmarkDataset: raise BenchmarkUnavailable( "benchmark input schema is unavailable" ) from None - root = _closed( - raw_root, - frozenset({"cases", "documents", "lock", "schemaVersion"}), - ) - lock = _closed(root["lock"], frozenset({"contentDigest", "profile"})) + root = cast(dict[str, object], raw_root) + lock = cast(dict[str, object], root["lock"]) unlocked = {key: value for key, value in root.items() if key != "lock"} if ( - lock["profile"] != LOCK_PROFILE + lock["profile"] != LOCK_PROFILE.value or lock["contentDigest"] != dataset_content_digest(unlocked) ): raise BenchmarkUnavailable("benchmark dataset lock is unavailable") @@ -257,7 +258,7 @@ def load_dataset(path: Path) -> BenchmarkDataset: text=_text(document["text"]), ) for value in raw_documents - for document in [_closed(value, frozenset({"documentRef", "text"}))] + for document in [cast(dict[str, object], value)] ) cases = tuple( BenchmarkCase( @@ -267,16 +268,13 @@ def load_dataset(path: Path) -> BenchmarkDataset: slice_name=_text(case["slice"]), ) for value in raw_cases - for case in [ - _closed( - value, - frozenset( - {"caseRef", "expectedDocumentRefs", "query", "slice"} - ), - ) - ] + for case in [cast(dict[str, object], value)] + ) + return BenchmarkDataset( + documents=documents, + cases=cases, + lock_profile=LOCK_PROFILE, ) - return BenchmarkDataset(documents=documents, cases=cases) except BenchmarkUnavailable: raise except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError): @@ -292,14 +290,6 @@ def dataset_content_digest(document: object) -> str: raise BenchmarkUnavailable("benchmark dataset lock is unavailable") from None -def _closed(value: object, fields: frozenset[str]) -> dict[str, object]: - if type(value) is not dict or frozenset(value) != fields: - raise BenchmarkUnavailable("benchmark dataset is unavailable") - if any(type(key) is not str for key in value): - raise BenchmarkUnavailable("benchmark dataset is unavailable") - return value - - def _text(value: object) -> str: if type(value) is not str or not value or value != value.strip(): raise BenchmarkUnavailable("benchmark dataset is unavailable") diff --git a/eval/README.md b/eval/README.md index 0a295beb..ecc246a4 100644 --- a/eval/README.md +++ b/eval/README.md @@ -50,19 +50,31 @@ the exact 384-dimensional primary and baseline model identities required by issue #128 and never composes an `EmbeddingProvider` into Supply or Runtime. The main environment deliberately contains no model framework. The closed `sentence-transformers` backend is installed explicitly with -`uv sync --extra benchmark`; each local pinned model directory supplies -`benchmark-model-identity.json` with its fully resolved repository id, -immutable revision, artifact digest, dimension, normalization, pooling, prompt -prefixes, reduction, precision, and batching. The runner verifies the digest -over the local model artifacts before it embeds anything. +`uv sync --extra benchmark`. Each local pinned model directory is the exact +complete `SentenceTransformer` snapshot declared by the tracked registry: its +weight, model/config, tokenizer, modules, and pooling files must match the +registry's closed path-and-SHA-256 artifact list, with no missing or extra +files. The tracked +`embedding-benchmark/model-registry.json`, not run input or a local manifest, +owns the expected repository id, immutable revision, artifact digest, +dimension, normalization, pooling, prompt prefixes, reduction, precision, and +batching. Unknown models and local identity overrides are refused, and the +runner hashes every on-disk artifact against the tracked expected digests before +it embeds anything. The input follows `embedding-benchmark/input.schema.json`; the output follows `embedding-benchmark/report.schema.json` and must be written below the ignored -`.context-engine/` directory. The input includes an RFC 8785/SHA-256 content -lock and any post-lock edit is refused. Retrieval scores are not implemented by -this runner: it imports the one fixed judge factory owned by issue #129, which -is solely responsible for case hit, macro/micro Evidence recall, and slice -breakdowns. +`.context-engine/` directory. The input includes the typed RFC 8785/SHA-256 +`sha256-rfc8785-accidental-edit-detection-v1` lock. It detects accidental edits, +truncation, and re-serialization performed without relocking, but it is not a +defense against deliberate forgery: the M1 threat model is one trusted local +operator who can recompute a colocated digest. If the boundary changes to an +untrusted caller or remote runner, signing returns as its own ADR. Retrieval +scores are not implemented by this runner: it imports the one fixed judge +factory owned by issue #129, which is solely responsible for case hit, +macro/micro Evidence recall, and slice breakdowns. Until #129 lands, the real +CLI deliberately fails closed with `retrieval judge is unavailable` rather +than degrading to a second metric implementation. The maintainer corpus is private and pending delivery to a durable, maintainer-controlled location outside disposable Git worktrees. Once available, @@ -85,3 +97,7 @@ no worse on all three and strictly better on at least one. Exact equality is a tie; mixed wins are `inconclusive`. Per-slice results remain diagnostic and no weighted composite or tiebreak manufactures a winner. A losing or inconclusive primary is a valid benchmark outcome, not a runner error. + +The issue remains open while the corpus is pending. A real-model run, frozen +numeric result, actual model verdict, and measured comparison to the standing +3.8% twin baseline are not complete until that durable corpus arrives. diff --git a/eval/embedding-benchmark/input.schema.json b/eval/embedding-benchmark/input.schema.json index 61b882b6..5591fc74 100644 --- a/eval/embedding-benchmark/input.schema.json +++ b/eval/embedding-benchmark/input.schema.json @@ -2,6 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://context-engine.invalid/eval/embedding-benchmark/input.schema.json", "title": "ContextEngine offline embedding benchmark input v1", + "description": "M1 assumes one trusted local operator. The colocated digest detects accidental edits, truncation, or re-serialization performed without relocking; it is not a defense against deliberate forgery because an operator can recompute the digest. If callers or runners become remote or untrusted, signing requires its own ADR.", "type": "object", "additionalProperties": false, "required": ["schemaVersion", "lock", "documents", "cases"], @@ -12,7 +13,10 @@ "additionalProperties": false, "required": ["profile", "contentDigest"], "properties": { - "profile": {"const": "sha256-rfc8785-v1"}, + "profile": { + "description": "Typed accidental-edit detector for one trusted local operator; not a tamper-proof or anti-forgery signature.", + "const": "sha256-rfc8785-accidental-edit-detection-v1" + }, "contentDigest": {"type": "string", "pattern": "^[0-9a-f]{64}$"} } }, diff --git a/eval/embedding-benchmark/model-registry.json b/eval/embedding-benchmark/model-registry.json new file mode 100644 index 00000000..3aad5009 --- /dev/null +++ b/eval/embedding-benchmark/model-registry.json @@ -0,0 +1,60 @@ +{ + "schemaVersion": "context-engine-embedding-model-registry-v1", + "models": { + "primary": { + "backend": "sentence-transformers", + "artifacts": [ + {"path": "1_Pooling/config.json", "sha256": "37bf193fa101f19101bfad9c31d3eb0f786e247b7b1e5cb7f007d730eed1ddbd"}, + {"path": "config.json", "sha256": "b5bf1f51fc45be473a54718cef92448d90a1be001bf9b9a44b8c7f10a19feaa9"}, + {"path": "config_sentence_transformers.json", "sha256": "10667c72ddb772627bf1780cb7f86af8e2ae0032b8c243c731172064105c6961"}, + {"path": "generation_config.json", "sha256": "28396d421a2108acce96383f6a7de78008f7f1b17f807958f3c14c51dbfb65fb"}, + {"path": "merges.txt", "sha256": "8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5"}, + {"path": "model.safetensors", "sha256": "0437e45c94563b09e13cb7a64478fc406947a93cb34a7e05870fc8dcd48e23fd"}, + {"path": "modules.json", "sha256": "84e40c8e006c9b1d6c122e02cba9b02458120b5fb0c87b746c41e0207cf642cf"}, + {"path": "tokenizer.json", "sha256": "def76fb086971c7867b829c23a26261e38d9d74e02139253b38aeb9df8b4b50a"}, + {"path": "tokenizer_config.json", "sha256": "253153d0738ceb4c668d2eff957714dd2bea0b56de772a9fdccd96cbf517e6a0"}, + {"path": "vocab.json", "sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910"} + ], + "identity": { + "modelId": "Qwen/Qwen3-Embedding-0.6B", + "revision": "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3", + "artifactDigest": "8cb25677d5be69ce6ac88ebbdfb5dad30980fee39c35c6324a583e325917eddc", + "dimension": 384, + "normalization": "l2", + "pooling": "last_token", + "queryPrefix": "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:", + "documentPrefix": "", + "reduction": "matryoshka_truncate_384", + "precision": "float32", + "batchSize": 8 + } + }, + "baseline": { + "backend": "sentence-transformers", + "artifacts": [ + {"path": "1_Pooling/config.json", "sha256": "987f7a67a38fa564c849bb5d277c52ab9088a84368fc0be31a354125aebb12a0"}, + {"path": "config.json", "sha256": "69137736cab8b8903a07fe8afaafdda25aac55415a12a55d1bffa9f581abf959"}, + {"path": "model.safetensors", "sha256": "1a55775f53449dac10a2bcbc312469fac40b96d53198c407081a831f81c98477"}, + {"path": "modules.json", "sha256": "c6e29747481e8b5dd2b58401966aeac910de39092f90cda9a704b1545f902b04"}, + {"path": "sentence_bert_config.json", "sha256": "948201d8329907aae938fa62f9ceeed53f5694dacc2b87b9f3b78b37ee986529"}, + {"path": "sentencepiece.bpe.model", "sha256": "cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865"}, + {"path": "special_tokens_map.json", "sha256": "d05497f1da52c5e09554c0cd874037a083e1dc1b9cfd48034d1c717f1afc07a7"}, + {"path": "tokenizer.json", "sha256": "0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39"}, + {"path": "tokenizer_config.json", "sha256": "a1d6bc8734a6f635dc158508bef000f8e2e5a759c7d92f984b2c86e5ff53425b"} + ], + "identity": { + "modelId": "intfloat/multilingual-e5-small", + "revision": "614241f622f53c4eeff9890bdc4f31cfecc418b3", + "artifactDigest": "961e0953328daf4df7916c1406751c5c63fe19e14a5c7e977d9f8b38964c5cf3", + "dimension": 384, + "normalization": "l2", + "pooling": "mean", + "queryPrefix": "query: ", + "documentPrefix": "passage: ", + "reduction": "none_native_384", + "precision": "float32", + "batchSize": 8 + } + } + } +} diff --git a/eval/embedding-benchmark/model-registry.schema.json b/eval/embedding-benchmark/model-registry.schema.json new file mode 100644 index 00000000..84e328c5 --- /dev/null +++ b/eval/embedding-benchmark/model-registry.schema.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://context-engine.invalid/eval/embedding-benchmark/model-registry.schema.json", + "title": "ContextEngine offline embedding tracked model registry v1", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "models"], + "properties": { + "schemaVersion": {"const": "context-engine-embedding-model-registry-v1"}, + "models": { + "type": "object", + "additionalProperties": false, + "required": ["primary", "baseline"], + "properties": { + "primary": {"$ref": "#/$defs/model"}, + "baseline": {"$ref": "#/$defs/model"} + } + } + }, + "$defs": { + "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "identity": { + "type": "object", + "additionalProperties": false, + "required": ["modelId", "revision", "artifactDigest", "dimension", "normalization", "pooling", "queryPrefix", "documentPrefix", "reduction", "precision", "batchSize"], + "properties": { + "modelId": {"type": "string", "minLength": 1}, + "revision": {"type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}, + "artifactDigest": {"$ref": "#/$defs/digest"}, + "dimension": {"const": 384}, + "normalization": {"enum": ["l2", "none"]}, + "pooling": {"type": "string", "minLength": 1}, + "queryPrefix": {"type": "string"}, + "documentPrefix": {"type": "string"}, + "reduction": {"enum": ["none_native_384", "matryoshka_truncate_384"]}, + "precision": {"type": "string", "minLength": 1}, + "batchSize": {"type": "integer", "minimum": 1} + } + }, + "model": { + "type": "object", + "additionalProperties": false, + "required": ["backend", "artifacts", "identity"], + "properties": { + "backend": {"const": "sentence-transformers"}, + "artifacts": { + "description": "The exact complete local snapshot consumed by SentenceTransformer. Every required config, tokenizer, pooling, and weight file is present and digest-bound; extra or missing files are refused.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256"], + "properties": { + "path": {"type": "string", "minLength": 1}, + "sha256": {"$ref": "#/$defs/digest"} + } + } + }, + "identity": {"$ref": "#/$defs/identity"} + } + } + } +} diff --git a/eval/embedding-benchmark/report.schema.json b/eval/embedding-benchmark/report.schema.json index d1013c1f..d4b82aef 100644 --- a/eval/embedding-benchmark/report.schema.json +++ b/eval/embedding-benchmark/report.schema.json @@ -29,7 +29,7 @@ "additionalProperties": false, "required": ["primaryAgainstModelBaseline", "primaryAgainstStandingTwinBaseline", "metricDeltas"], "properties": { - "primaryAgainstModelBaseline": {"enum": ["win", "tie", "lose", "inconclusive"]}, + "primaryAgainstModelBaseline": {"$ref": "#/$defs/modelComparisonOutcome"}, "primaryAgainstStandingTwinBaseline": {"enum": ["win", "tie", "lose"]}, "metricDeltas": { "type": "object", @@ -46,6 +46,10 @@ }, "$defs": { "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "modelComparisonOutcome": { + "description": "Closed first-class Pareto outcome. Validation recomputes this value from the recorded primary and baseline case-hit, macro-recall, and micro-recall metrics and refuses any mismatch.", + "enum": ["win", "lose", "tie", "inconclusive"] + }, "run": { "type": "object", "additionalProperties": false, diff --git a/eval/embedding_benchmark.py b/eval/embedding_benchmark.py index b7ce015d..d6efc5e7 100644 --- a/eval/embedding_benchmark.py +++ b/eval/embedding_benchmark.py @@ -7,13 +7,16 @@ import re from collections.abc import Callable, Mapping from dataclasses import dataclass, field +from enum import StrEnum from hashlib import sha256 from pathlib import Path from typing import Any, Final, Protocol, cast import rfc8785 -EMBEDDING_DIMENSION: Final = 384 +from engine.supply.embeddings import CONTEXT_FRAGMENT_EMBEDDING_DIMENSION + +EMBEDDING_DIMENSION: Final = CONTEXT_FRAGMENT_EMBEDDING_DIMENSION REPORT_SCHEMA_VERSION: Final = "context-engine-embedding-benchmark-report-v1" _PINNED_REVISION = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") _SHA256_DIGEST = re.compile(r"^[0-9a-f]{64}$") @@ -25,6 +28,14 @@ class BenchmarkUnavailable(RuntimeError): """The benchmark input, identity, provider, judge, or output is unavailable.""" +class DatasetLockProfile(StrEnum): + """Typed M1 lock that detects accidental edits by one trusted operator.""" + + ACCIDENTAL_EDIT_DETECTION = ( + "sha256-rfc8785-accidental-edit-detection-v1" + ) + + @dataclass(frozen=True, slots=True) class ModelIdentity: """Complete, reproducible identity for one offline embedding provider.""" @@ -126,6 +137,7 @@ class BenchmarkDataset: documents: tuple[BenchmarkDocument, ...] = field(repr=False) cases: tuple[BenchmarkCase, ...] = field(repr=False) locked: bool = True + lock_profile: DatasetLockProfile = DatasetLockProfile.ACCIDENTAL_EDIT_DETECTION digest: str = field(init=False) def __post_init__(self) -> None: @@ -138,6 +150,7 @@ def __post_init__(self) -> None: or any(type(value) is not BenchmarkCase for value in self.cases) or type(self.locked) is not bool or not self.locked + or self.lock_profile is not DatasetLockProfile.ACCIDENTAL_EDIT_DETECTION ): raise BenchmarkUnavailable("benchmark dataset is unavailable") document_refs = tuple(value.document_ref for value in self.documents) @@ -286,15 +299,24 @@ def __post_init__(self) -> None: raise BenchmarkUnavailable("retrieval judge is unavailable") +class ModelComparisonOutcome(StrEnum): + """Closed Pareto outcome; mixed metric wins stay first-class.""" + + WIN = "win" + LOSE = "lose" + TIE = "tie" + INCONCLUSIVE = "inconclusive" + + @dataclass(frozen=True, slots=True) class ModelComparison: """Typed Pareto verdict and primary-minus-baseline metric deltas.""" - outcome: str + outcome: ModelComparisonOutcome deltas: Mapping[str, float] def __post_init__(self) -> None: - if self.outcome not in {"win", "tie", "lose", "inconclusive"}: + if type(self.outcome) is not ModelComparisonOutcome: raise BenchmarkUnavailable("model comparison is unavailable") if frozenset(self.deltas) != frozenset( {"caseHit", "macroEvidenceRecall", "microEvidenceRecall"} @@ -302,7 +324,7 @@ def __post_init__(self) -> None: raise BenchmarkUnavailable("model comparison is unavailable") def public_document(self) -> dict[str, object]: - return {"deltas": dict(self.deltas), "outcome": self.outcome} + return {"deltas": dict(self.deltas), "outcome": self.outcome.value} class RetrievalJudge(Protocol): @@ -355,7 +377,7 @@ def run_benchmark( return { "comparison": { "metricDeltas": dict(model_comparison.deltas), - "primaryAgainstModelBaseline": model_comparison.outcome, + "primaryAgainstModelBaseline": model_comparison.outcome.value, "primaryAgainstStandingTwinBaseline": _comparison(primary_hit, 0.038), }, "models": { @@ -535,20 +557,7 @@ def compare_model_metrics( baseline.evidence_recall.macro.value, baseline.evidence_recall.micro.value, ) - if primary_values == baseline_values: - outcome = "tie" - elif all( - left >= right - for left, right in zip(primary_values, baseline_values, strict=True) - ): - outcome = "win" - elif all( - left <= right - for left, right in zip(primary_values, baseline_values, strict=True) - ): - outcome = "lose" - else: - outcome = "inconclusive" + outcome = _pareto_outcome(primary_values, baseline_values) return ModelComparison( outcome=outcome, deltas={ @@ -559,6 +568,25 @@ def compare_model_metrics( ) +def _pareto_outcome( + primary_values: tuple[float, float, float], + baseline_values: tuple[float, float, float], +) -> ModelComparisonOutcome: + if primary_values == baseline_values: + return ModelComparisonOutcome.TIE + if all( + left >= right + for left, right in zip(primary_values, baseline_values, strict=True) + ): + return ModelComparisonOutcome.WIN + if all( + left <= right + for left, right in zip(primary_values, baseline_values, strict=True) + ): + return ModelComparisonOutcome.LOSE + return ModelComparisonOutcome.INCONCLUSIVE + + def _comparison(left: float, right: float) -> str: if left > right: return "win" @@ -655,8 +683,29 @@ def _validate_report(document: object, schema: object) -> None: ) _require_ratio(standing["caseHitValue"]) models = _closed_object(report["models"], frozenset({"baseline", "primary"})) - _validate_model_report(models["primary"]) - _validate_model_report(models["baseline"]) + primary_values = _validate_model_report(models["primary"]) + baseline_values = _validate_model_report(models["baseline"]) + try: + declared_outcome = ModelComparisonOutcome( + cast(str, comparison["primaryAgainstModelBaseline"]) + ) + except ValueError: + raise BenchmarkUnavailable("benchmark report verdict is unavailable") from None + expected_outcome = _pareto_outcome(primary_values, baseline_values) + expected_deltas = { + "caseHit": primary_values[0] - baseline_values[0], + "macroEvidenceRecall": primary_values[1] - baseline_values[1], + "microEvidenceRecall": primary_values[2] - baseline_values[2], + } + expected_standing = _comparison( + primary_values[0], cast(float, standing["caseHitValue"]) + ) + if ( + declared_outcome is not expected_outcome + or deltas != expected_deltas + or comparison["primaryAgainstStandingTwinBaseline"] != expected_standing + ): + raise BenchmarkUnavailable("benchmark report verdict is unavailable") def validate_json_schema_document(value: object, schema: object) -> None: @@ -774,7 +823,7 @@ def _canonical_json(value: object) -> bytes: raise BenchmarkUnavailable("benchmark report schema is unavailable") from None -def _validate_model_report(value: object) -> None: +def _validate_model_report(value: object) -> tuple[float, float, float]: report = _closed_object(value, frozenset({"identity", "metrics", "timing"})) identity = _closed_object( report["identity"], @@ -807,7 +856,7 @@ def _validate_model_report(value: object) -> None: precision=cast(str, identity["precision"]), batch_size=cast(int, identity["batchSize"]), ) - _validate_metrics(report["metrics"]) + metric_values = _validate_metrics(report["metrics"]) timing = _closed_object( report["timing"], frozenset( @@ -818,9 +867,10 @@ def _validate_model_report(value: object) -> None: raise BenchmarkUnavailable("benchmark report schema is unavailable") _require_nonnegative_number(timing["perDocumentEmbedMilliseconds"]) _require_nonnegative_number(timing["wallClockMilliseconds"]) + return metric_values -def _validate_metrics(value: object) -> None: +def _validate_metrics(value: object) -> tuple[float, float, float]: metrics = _closed_object( value, frozenset({"caseHit", "evidenceRecall", "perSlice"}), @@ -837,6 +887,19 @@ def _validate_metrics(value: object) -> None: ) _validate_case_hit(item["caseHit"]) _validate_recall(item["evidenceRecall"]) + case_hit = _closed_object( + metrics["caseHit"], frozenset({"hits", "totalCases", "value"}) + ) + recall = _closed_object(metrics["evidenceRecall"], frozenset({"macro", "micro"})) + macro = _closed_object(recall["macro"], frozenset({"value"})) + micro = _closed_object( + recall["micro"], frozenset({"hits", "totalExpected", "value"}) + ) + return ( + float(cast(int | float, case_hit["value"])), + float(cast(int | float, macro["value"])), + float(cast(int | float, micro["value"])), + ) def _validate_case_hit(value: object) -> None: diff --git a/tests/unit/test_embedding_benchmark_cli.py b/tests/unit/test_embedding_benchmark_cli.py index f0c94699..5ea59f33 100644 --- a/tests/unit/test_embedding_benchmark_cli.py +++ b/tests/unit/test_embedding_benchmark_cli.py @@ -11,6 +11,7 @@ from eval.embedding_benchmark import ( BenchmarkUnavailable, CaseHitMetric, + DatasetLockProfile, EvidenceRecallMetric, MacroRecallMetric, MicroRecallMetric, @@ -77,7 +78,7 @@ def _write_dataset(path: Path) -> None: content_digest = cli.dataset_content_digest(document) document["lock"] = { "contentDigest": content_digest, - "profile": "sha256-rfc8785-v1", + "profile": "sha256-rfc8785-accidental-edit-detection-v1", } path.write_text(json.dumps(document), encoding="utf-8") @@ -93,6 +94,17 @@ def test_loader_refuses_a_pilot_changed_after_locking(tmp_path: Path) -> None: cli.load_dataset(path) +def test_loader_returns_the_typed_accidental_edit_detection_profile( + tmp_path: Path, +) -> None: + path = tmp_path / "dataset.json" + _write_dataset(path) + + dataset = cli.load_dataset(path) + + assert dataset.lock_profile is DatasetLockProfile.ACCIDENTAL_EDIT_DETECTION + + @pytest.mark.parametrize( "mutate", ( @@ -131,6 +143,28 @@ def test_locked_document_accepted_by_schema_is_accepted_by_loader( assert dataset.locked is True +def test_loader_treats_the_supplied_schema_as_its_only_shape_authority( + tmp_path: Path, + monkeypatch: Any, +) -> None: + path = tmp_path / "dataset.json" + schema_path = tmp_path / "input.schema.json" + _write_dataset(path) + document = json.loads(path.read_text(encoding="utf-8")) + document["operatorAnnotation"] = "ignored by the benchmark domain loader" + unlocked = {key: value for key, value in document.items() if key != "lock"} + document["lock"]["contentDigest"] = cli.dataset_content_digest(unlocked) + path.write_text(json.dumps(document), encoding="utf-8") + schema = json.loads(cli.INPUT_SCHEMA_PATH.read_text(encoding="utf-8")) + schema["properties"]["operatorAnnotation"] = {"type": "string"} + schema_path.write_text(json.dumps(schema), encoding="utf-8") + monkeypatch.setattr(cli, "INPUT_SCHEMA_PATH", schema_path) + + dataset = cli.load_dataset(path) + + assert dataset.cases[0].case_ref == "case-alpha" + + def test_cli_run_writes_a_schema_valid_report_for_a_synthetic_set( tmp_path: Path, monkeypatch: Any, diff --git a/tests/unit/test_embedding_benchmark_metrics.py b/tests/unit/test_embedding_benchmark_metrics.py index 4f8d1e5b..1623ec88 100644 --- a/tests/unit/test_embedding_benchmark_metrics.py +++ b/tests/unit/test_embedding_benchmark_metrics.py @@ -15,6 +15,7 @@ EvidenceRecallMetric, MacroRecallMetric, MicroRecallMetric, + ModelComparisonOutcome, ModelIdentity, RetrievalJudgeCase, RetrievalMetrics, @@ -174,23 +175,27 @@ def evaluate_retrieval( @pytest.mark.parametrize( ("primary", "baseline", "expected"), ( - ((0.9, 0.8, 0.7), (0.8, 0.7, 0.6), "win"), - ((0.8, 0.7, 0.6), (0.9, 0.8, 0.7), "lose"), - ((0.8, 0.8, 0.8), (0.8, 0.8, 0.8), "tie"), - ((0.9, 0.6, 0.8), (0.8, 0.7, 0.8), "inconclusive"), + ((0.9, 0.8, 0.7), (0.8, 0.7, 0.6), ModelComparisonOutcome.WIN), + ((0.8, 0.7, 0.6), (0.9, 0.8, 0.7), ModelComparisonOutcome.LOSE), + ((0.8, 0.8, 0.8), (0.8, 0.8, 0.8), ModelComparisonOutcome.TIE), + ( + (0.9, 0.6, 0.8), + (0.8, 0.7, 0.8), + ModelComparisonOutcome.INCONCLUSIVE, + ), ), ) def test_model_verdict_uses_pareto_dominance_across_all_retrieval_metrics( primary: tuple[float, float, float], baseline: tuple[float, float, float], - expected: str, + expected: ModelComparisonOutcome, ) -> None: verdict = compare_model_metrics( _retrieval_metrics(*primary), _retrieval_metrics(*baseline), ) - assert verdict.outcome == expected + assert verdict.outcome is expected assert verdict.deltas == { "caseHit": pytest.approx(primary[0] - baseline[0]), "macroEvidenceRecall": pytest.approx(primary[1] - baseline[1]), diff --git a/tests/unit/test_embedding_benchmark_model_registry.py b/tests/unit/test_embedding_benchmark_model_registry.py new file mode 100644 index 00000000..dc407bc4 --- /dev/null +++ b/tests/unit/test_embedding_benchmark_model_registry.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import hashlib +import json +from copy import deepcopy +from pathlib import Path +from typing import Any + +import pytest +import rfc8785 + +from applications import embedding_benchmark as cli +from eval.embedding_benchmark import BenchmarkUnavailable + + +def _registry( + *, expected_files: dict[str, bytes], include_primary: bool = True +) -> dict[str, object]: + artifacts = [ + {"path": path, "sha256": hashlib.sha256(content).hexdigest()} + for path, content in sorted(expected_files.items()) + ] + artifact_digest = hashlib.sha256(rfc8785.dumps(artifacts)).hexdigest() + primary: dict[str, Any] = { + "artifacts": artifacts, + "backend": "sentence-transformers", + "identity": { + "artifactDigest": artifact_digest, + "batchSize": 8, + "dimension": 384, + "documentPrefix": "", + "modelId": "Qwen/Qwen3-Embedding-0.6B", + "normalization": "l2", + "pooling": "last_token", + "precision": "float32", + "queryPrefix": "Instruct: retrieve relevant passages\nQuery:", + "reduction": "matryoshka_truncate_384", + "revision": "a" * 40, + }, + } + baseline: dict[str, Any] = deepcopy(primary) + baseline["identity"]["modelId"] = "intfloat/multilingual-e5-small" + models: dict[str, object] = {"baseline": baseline} + if include_primary: + models["primary"] = primary + return { + "models": models, + "schemaVersion": "context-engine-embedding-model-registry-v1", + } + + +def _write_registry( + path: Path, + *, + expected_files: dict[str, bytes], + include_primary: bool = True, +) -> None: + path.write_text( + json.dumps( + _registry( + expected_files=expected_files, + include_primary=include_primary, + ) + ), + encoding="utf-8", + ) + + +def _expected_directory_digest(files: dict[str, bytes]) -> str: + manifest = [ + {"path": path, "sha256": hashlib.sha256(content).hexdigest()} + for path, content in sorted(files.items()) + ] + return hashlib.sha256(rfc8785.dumps(manifest)).hexdigest() + + +def test_model_artifact_must_match_the_tracked_registry_digest( + tmp_path: Path, + monkeypatch: Any, +) -> None: + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "model.safetensors").write_bytes(b"untracked artifact bytes") + registry_path = tmp_path / "model-registry.json" + _write_registry( + registry_path, + expected_files={"model.safetensors": b"tracked artifact bytes"}, + ) + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + with pytest.raises(BenchmarkUnavailable, match="model identity is unresolved"): + cli.load_registered_model_identity( + role="primary", + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) + + +def test_model_absent_from_the_tracked_registry_is_refused( + tmp_path: Path, + monkeypatch: Any, +) -> None: + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "model.safetensors").write_bytes(b"artifact") + registry_path = tmp_path / "model-registry.json" + _write_registry( + registry_path, + expected_files={"model.safetensors": b"artifact"}, + include_primary=False, + ) + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + with pytest.raises(BenchmarkUnavailable, match="model identity is unresolved"): + cli.load_registered_model_identity( + role="primary", + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) + + +def test_local_input_cannot_override_a_tracked_identity_field( + tmp_path: Path, + monkeypatch: Any, +) -> None: + artifact = b"tracked artifact bytes" + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "model.safetensors").write_bytes(artifact) + (model_dir / "benchmark-model-identity.json").write_text( + json.dumps({"revision": "f" * 40}), + encoding="utf-8", + ) + registry_path = tmp_path / "model-registry.json" + _write_registry( + registry_path, + expected_files={"model.safetensors": artifact}, + ) + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + with pytest.raises(BenchmarkUnavailable, match="identity override"): + cli.load_registered_model_identity( + role="primary", + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) + + +def test_matching_artifact_uses_only_the_tracked_identity( + tmp_path: Path, + monkeypatch: Any, +) -> None: + artifact = b"tracked artifact bytes" + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "model.safetensors").write_bytes(artifact) + registry_path = tmp_path / "model-registry.json" + _write_registry( + registry_path, + expected_files={"model.safetensors": artifact}, + ) + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + identity = cli.load_registered_model_identity( + role="primary", + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) + + assert identity.model_id == "Qwen/Qwen3-Embedding-0.6B" + assert identity.artifact_digest == _expected_directory_digest( + {"model.safetensors": artifact} + ) + + +def test_complete_registered_sentence_transformer_layout_is_accepted( + tmp_path: Path, + monkeypatch: Any, +) -> None: + files = { + "1_Pooling/config.json": b"pooling", + "config.json": b"model config", + "model.safetensors": b"weights", + "modules.json": b"modules", + "tokenizer.json": b"tokenizer", + } + model_dir = tmp_path / "model" + for relative_path, content in files.items(): + path = model_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + registry_path = tmp_path / "model-registry.json" + _write_registry(registry_path, expected_files=files) + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + loaded_paths: list[Path] = [] + + class FakeSentenceTransformer: + def __init__( + self, + model_path: str, + *, + local_files_only: bool, + trust_remote_code: bool, + ) -> None: + assert local_files_only is True + assert trust_remote_code is False + loaded_paths.append(Path(model_path)) + + class FakeBackend: + SentenceTransformer = FakeSentenceTransformer + + monkeypatch.setattr(cli, "_load_sentence_transformers", lambda: FakeBackend) + + provider = cli.build_local_provider( + cli.SupportedBackend.SENTENCE_TRANSFORMERS, + "primary", + model_dir, + ) + + assert provider.identity.artifact_digest == _expected_directory_digest(files) + assert loaded_paths == [model_dir] + + +def test_complete_registered_layout_refuses_an_extra_untracked_file( + tmp_path: Path, + monkeypatch: Any, +) -> None: + files = { + "config.json": b"model config", + "model.safetensors": b"weights", + "modules.json": b"modules", + } + model_dir = tmp_path / "model" + model_dir.mkdir() + for relative_path, content in files.items(): + (model_dir / relative_path).write_bytes(content) + (model_dir / "caller-identity.json").write_text("{}", encoding="utf-8") + registry_path = tmp_path / "model-registry.json" + _write_registry(registry_path, expected_files=files) + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + with pytest.raises(BenchmarkUnavailable, match="model identity is unresolved"): + cli.load_registered_model_identity( + role="primary", + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) diff --git a/tests/unit/test_embedding_benchmark_report_schema.py b/tests/unit/test_embedding_benchmark_report_schema.py index 8db0f640..fa29d5be 100644 --- a/tests/unit/test_embedding_benchmark_report_schema.py +++ b/tests/unit/test_embedding_benchmark_report_schema.py @@ -86,6 +86,29 @@ def _report() -> dict[str, object]: } +def _set_model_metric_values( + report: dict[str, Any], + *, + primary: tuple[float, float, float], + baseline: tuple[float, float, float], + declared: str, +) -> None: + for role, values in (("primary", primary), ("baseline", baseline)): + metrics = report["models"][role]["metrics"] + metrics["caseHit"]["value"] = values[0] + metrics["evidenceRecall"]["macro"]["value"] = values[1] + metrics["evidenceRecall"]["micro"]["value"] = values[2] + report["comparison"] = { + "metricDeltas": { + "caseHit": primary[0] - baseline[0], + "macroEvidenceRecall": primary[1] - baseline[1], + "microEvidenceRecall": primary[2] - baseline[2], + }, + "primaryAgainstModelBaseline": declared, + "primaryAgainstStandingTwinBaseline": ( + "win" if primary[0] > 0.038 else "lose" + ), + } def test_report_validates_against_the_tracked_closed_schema() -> None: schema = json.loads(REPORT_SCHEMA.read_text(encoding="utf-8")) @@ -147,3 +170,72 @@ def test_report_refuses_a_missing_identity_metric_or_cost_field( with pytest.raises(BenchmarkUnavailable, match="report schema"): validate_report_document(report, schema_path=REPORT_SCHEMA) + + +@pytest.mark.parametrize( + ("primary", "baseline"), + ( + ((0.0, 0.0, 0.0), (1.0, 1.0, 1.0)), + ((1.0, 0.0, 0.5), (0.0, 1.0, 0.5)), + ), +) +def test_report_refuses_a_claimed_win_not_derived_from_recorded_metrics( + primary: tuple[float, float, float], + baseline: tuple[float, float, float], +) -> None: + report: dict[str, Any] = _report() + _set_model_metric_values( + report, + primary=primary, + baseline=baseline, + declared="win", + ) + + with pytest.raises(BenchmarkUnavailable, match="report verdict"): + validate_report_document(report, schema_path=REPORT_SCHEMA) + + +@pytest.mark.parametrize( + ("primary", "baseline", "declared"), + ( + ((1.0, 1.0, 1.0), (0.0, 0.5, 1.0), "win"), + ((0.0, 0.5, 1.0), (1.0, 1.0, 1.0), "lose"), + ((0.5, 0.5, 0.5), (0.5, 0.5, 0.5), "tie"), + ((1.0, 0.0, 0.5), (0.0, 1.0, 0.5), "inconclusive"), + ), +) +def test_report_accepts_each_self_consistent_typed_pareto_outcome( + primary: tuple[float, float, float], + baseline: tuple[float, float, float], + declared: str, +) -> None: + report: dict[str, Any] = _report() + _set_model_metric_values( + report, + primary=primary, + baseline=baseline, + declared=declared, + ) + + validate_report_document(report, schema_path=REPORT_SCHEMA) + + +@pytest.mark.parametrize( + "mutate", + ( + lambda report: report["comparison"]["metricDeltas"].update( + {"microEvidenceRecall": 0.25} + ), + lambda report: report["comparison"].update( + {"primaryAgainstStandingTwinBaseline": "lose"} + ), + ), +) +def test_report_refuses_each_derived_comparison_field_when_fabricated( + mutate: Any, +) -> None: + report: dict[str, Any] = _report() + mutate(report) + + with pytest.raises(BenchmarkUnavailable, match="report verdict"): + validate_report_document(report, schema_path=REPORT_SCHEMA) diff --git a/tests/unit/test_embedding_benchmark_runtime_untouched.py b/tests/unit/test_embedding_benchmark_runtime_untouched.py index 33573cbc..415fcc5d 100644 --- a/tests/unit/test_embedding_benchmark_runtime_untouched.py +++ b/tests/unit/test_embedding_benchmark_runtime_untouched.py @@ -23,3 +23,15 @@ def test_offline_benchmark_does_not_modify_or_compose_production_runtime() -> No "eval.embedding_benchmark" in path.read_text(encoding="utf-8") for path in (REPOSITORY_ROOT / "engine" / "runtime").glob("*.py") ) + + +def test_benchmark_imports_the_one_production_dimension_constant() -> None: + benchmark_source = (REPOSITORY_ROOT / "eval" / "embedding_benchmark.py").read_text( + encoding="utf-8" + ) + + assert ( + "from engine.supply.embeddings import CONTEXT_FRAGMENT_EMBEDDING_DIMENSION" + in benchmark_source + ) + assert "EMBEDDING_DIMENSION: Final = 384" not in benchmark_source From 3527d9d12fee4bcef4a1d5b4ed756a379dd14533 Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 00:06:39 +0800 Subject: [PATCH 5/7] fix: harden benchmark provenance and metrics --- applications/embedding_benchmark.py | 118 ++++++-- eval/README.md | 16 +- eval/embedding-benchmark/model-registry.json | 6 +- .../model-registry.schema.json | 8 +- eval/embedding-benchmark/report.schema.json | 8 +- eval/embedding_benchmark.py | 274 ++++++++++++++--- tests/unit/test_embedding_benchmark_cli.py | 24 ++ .../unit/test_embedding_benchmark_identity.py | 7 +- .../unit/test_embedding_benchmark_metrics.py | 18 +- ...test_embedding_benchmark_model_registry.py | 277 +++++++++++++++++- .../test_embedding_benchmark_report_schema.py | 135 ++++++++- 11 files changed, 792 insertions(+), 99 deletions(-) diff --git a/applications/embedding_benchmark.py b/applications/embedding_benchmark.py index e348ffe9..5bc78eea 100644 --- a/applications/embedding_benchmark.py +++ b/applications/embedding_benchmark.py @@ -5,6 +5,7 @@ import argparse import importlib import json +import math import sys from collections.abc import Callable, Sequence from enum import StrEnum @@ -24,7 +25,9 @@ BenchmarkUnavailable, DatasetLockProfile, ModelIdentity, + ModelTransformationPipeline, RetrievalJudge, + load_bounded_json, run_benchmark, validate_json_schema_document, validate_report_document, @@ -38,6 +41,10 @@ LOCK_PROFILE = DatasetLockProfile.ACCIDENTAL_EDIT_DETECTION BENCHMARK_EXTRA = "context-engine[benchmark]" _IDENTITY_OVERRIDE_FILENAME = "benchmark-model-identity.json" +_REGISTERED_IDENTITY_DIGESTS = { + "primary": "f748d119fd4d5c6b2843e70e977e9da86000afb195bf8bd5b867d483b6409e1e", + "baseline": "a915cb9d63a908bbd47113d46719f0d582db7a2f787ba83de4ae5eef8f4f890a", +} class SupportedBackend(StrEnum): @@ -77,14 +84,33 @@ def _embed(self, values: tuple[str, ...]) -> tuple[tuple[float, ...], ...]: list(values), batch_size=self.identity.batch_size, convert_to_numpy=True, - normalize_embeddings=self.identity.normalization == "l2", + normalize_embeddings=True, precision=self.identity.precision, show_progress_bar=False, ) - return tuple( - tuple(float(value) for value in vector[:EMBEDDING_DIMENSION]) - for vector in vectors - ) + expected_raw_dimension = self.identity.transformation_pipeline.raw_dimension + normalized: list[tuple[float, ...]] = [] + for vector in vectors: + if len(vector) != expected_raw_dimension: + raise BenchmarkUnavailable("local benchmark model is unavailable") + truncated = tuple( + float(value) for value in vector[:EMBEDDING_DIMENSION] + ) + if not all(math.isfinite(value) for value in truncated): + raise BenchmarkUnavailable("local benchmark model is unavailable") + norm = math.sqrt(sum(value * value for value in truncated)) + if not math.isfinite(norm) or norm == 0.0: + raise BenchmarkUnavailable("local benchmark model is unavailable") + emitted = tuple(value / norm for value in truncated) + if not math.isclose( + math.sqrt(sum(value * value for value in emitted)), + 1.0, + rel_tol=0.0, + abs_tol=1e-6, + ): + raise BenchmarkUnavailable("local benchmark model is unavailable") + normalized.append(emitted) + return tuple(normalized) except Exception: raise BenchmarkUnavailable("local benchmark model is unavailable") from None @@ -107,13 +133,19 @@ def build_local_provider( if type(backend) is not SupportedBackend or role not in {"primary", "baseline"}: raise BenchmarkUnavailable("benchmark backend is unavailable") - identity = load_registered_model_identity( + identity, expected_artifacts = _load_registered_model_contract( role=role, backend=backend, model_dir=model_dir, ) if backend is SupportedBackend.SENTENCE_TRANSFORMERS: - return SentenceTransformersProvider(identity=identity, model_dir=model_dir) + provider = SentenceTransformersProvider(identity=identity, model_dir=model_dir) + if identity.artifact_digest != _model_artifact_digest( + model_dir, + expected_artifacts=expected_artifacts, + ): + raise BenchmarkUnavailable("model identity is unresolved") + return provider raise BenchmarkUnavailable("benchmark backend is unavailable") @@ -125,14 +157,28 @@ def load_registered_model_identity( ) -> ModelIdentity: """Resolve identity only from the tracked registry and verify local bytes.""" + identity, _expected_artifacts = _load_registered_model_contract( + role=role, + backend=backend, + model_dir=model_dir, + ) + return identity + + +def _load_registered_model_contract( + *, + role: str, + backend: SupportedBackend, + model_dir: Path, +) -> tuple[ModelIdentity, tuple[tuple[str, str], ...]]: + """Resolve one identity plus the exact artifact manifest used to verify it.""" + override_path = model_dir / _IDENTITY_OVERRIDE_FILENAME if override_path.exists(): raise BenchmarkUnavailable("local model identity override is unavailable") try: - registry = json.loads(MODEL_REGISTRY_PATH.read_text(encoding="utf-8")) - registry_schema = json.loads( - MODEL_REGISTRY_SCHEMA_PATH.read_text(encoding="utf-8") - ) + registry = load_bounded_json(MODEL_REGISTRY_PATH) + registry_schema = load_bounded_json(MODEL_REGISTRY_SCHEMA_PATH) try: validate_json_schema_document(registry, registry_schema) except BenchmarkUnavailable: @@ -162,28 +208,55 @@ def load_registered_model_identity( for value in artifact_documents ) identity_document = cast(dict[str, object], model["identity"]) + digestable_identity = { + key: value + for key, value in identity_document.items() + if key != "artifactDigest" + } + if sha256(rfc8785.dumps(cast(Any, digestable_identity))).hexdigest() != ( + _REGISTERED_IDENTITY_DIGESTS[role] + ): + raise BenchmarkUnavailable("model identity is unresolved") identity = ModelIdentity( model_id=cast(str, identity_document["modelId"]), revision=cast(str, identity_document["revision"]), artifact_digest=cast(str, identity_document["artifactDigest"]), dimension=cast(int, identity_document["dimension"]), - normalization=cast(str, identity_document["normalization"]), + transformation_pipeline=ModelTransformationPipeline( + cast(str, identity_document["transformationPipeline"]) + ), pooling=cast(str, identity_document["pooling"]), query_prefix=cast(str, identity_document["queryPrefix"]), document_prefix=cast(str, identity_document["documentPrefix"]), - reduction=cast(str, identity_document["reduction"]), precision=cast(str, identity_document["precision"]), batch_size=cast(int, identity_document["batchSize"]), ) + expected_pipeline = ( + ModelTransformationPipeline.PRIMARY + if role == "primary" + else ModelTransformationPipeline.BASELINE + ) + if identity.transformation_pipeline is not expected_pipeline: + raise BenchmarkUnavailable("model identity is unresolved") if identity.artifact_digest != _model_artifact_digest( model_dir, expected_artifacts=expected_artifacts, ): raise BenchmarkUnavailable("model identity is unresolved") - return identity + return identity, expected_artifacts except BenchmarkUnavailable: raise - except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError): + except ( + OSError, + UnicodeDecodeError, + json.JSONDecodeError, + KeyError, + TypeError, + ValueError, + OverflowError, + RecursionError, + MemoryError, + ): raise BenchmarkUnavailable("model identity is unresolved") from None @@ -232,8 +305,8 @@ def load_dataset(path: Path) -> BenchmarkDataset: """Load the strict drop-in input format used by the pending durable corpus.""" try: - raw_root = json.loads(path.read_text(encoding="utf-8")) - schema = json.loads(INPUT_SCHEMA_PATH.read_text(encoding="utf-8")) + raw_root = load_bounded_json(path) + schema = load_bounded_json(INPUT_SCHEMA_PATH) try: validate_json_schema_document(raw_root, schema) except BenchmarkUnavailable: @@ -277,7 +350,16 @@ def load_dataset(path: Path) -> BenchmarkDataset: ) except BenchmarkUnavailable: raise - except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError): + except ( + OSError, + UnicodeDecodeError, + json.JSONDecodeError, + KeyError, + TypeError, + OverflowError, + RecursionError, + MemoryError, + ): raise BenchmarkUnavailable("benchmark dataset is unavailable") from None diff --git a/eval/README.md b/eval/README.md index ecc246a4..0b3cd871 100644 --- a/eval/README.md +++ b/eval/README.md @@ -57,10 +57,12 @@ registry's closed path-and-SHA-256 artifact list, with no missing or extra files. The tracked `embedding-benchmark/model-registry.json`, not run input or a local manifest, owns the expected repository id, immutable revision, artifact digest, -dimension, normalization, pooling, prompt prefixes, reduction, precision, and -batching. Unknown models and local identity overrides are refused, and the -runner hashes every on-disk artifact against the tracked expected digests before -it embeds anything. +dimension, complete transformation pipeline, pooling, prompt prefixes, +precision, and batching. See that registry for the exact ordered pipelines; +both emit unit-norm 384-dimensional vectors under the same geometry. Unknown +models and local identity overrides are refused, and the runner hashes every +on-disk artifact against the tracked expected digests before and after the +backend loads the model. The input follows `embedding-benchmark/input.schema.json`; the output follows `embedding-benchmark/report.schema.json` and must be written below the ignored @@ -69,8 +71,10 @@ The input follows `embedding-benchmark/input.schema.json`; the output follows truncation, and re-serialization performed without relocking, but it is not a defense against deliberate forgery: the M1 threat model is one trusted local operator who can recompute a colocated digest. If the boundary changes to an -untrusted caller or remote runner, signing returns as its own ADR. Retrieval -scores are not implemented by this runner: it imports the one fixed judge +untrusted caller or remote runner, signing returns as its own ADR. All benchmark +JSON entry points reject out-of-range or non-finite numbers and bound document +size, nesting, strings, and containers with a typed refusal. Retrieval scores +are not implemented by this runner: it imports the one fixed judge factory owned by issue #129, which is solely responsible for case hit, macro/micro Evidence recall, and slice breakdowns. Until #129 lands, the real CLI deliberately fails closed with `retrieval judge is unavailable` rather diff --git a/eval/embedding-benchmark/model-registry.json b/eval/embedding-benchmark/model-registry.json index 3aad5009..65962375 100644 --- a/eval/embedding-benchmark/model-registry.json +++ b/eval/embedding-benchmark/model-registry.json @@ -20,11 +20,10 @@ "revision": "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3", "artifactDigest": "8cb25677d5be69ce6ac88ebbdfb5dad30980fee39c35c6324a583e325917eddc", "dimension": 384, - "normalization": "l2", "pooling": "last_token", "queryPrefix": "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:", "documentPrefix": "", - "reduction": "matryoshka_truncate_384", + "transformationPipeline": "l2 -> truncate 1024->384 -> l2", "precision": "float32", "batchSize": 8 } @@ -47,11 +46,10 @@ "revision": "614241f622f53c4eeff9890bdc4f31cfecc418b3", "artifactDigest": "961e0953328daf4df7916c1406751c5c63fe19e14a5c7e977d9f8b38964c5cf3", "dimension": 384, - "normalization": "l2", "pooling": "mean", "queryPrefix": "query: ", "documentPrefix": "passage: ", - "reduction": "none_native_384", + "transformationPipeline": "l2 -> keep native 384 -> l2", "precision": "float32", "batchSize": 8 } diff --git a/eval/embedding-benchmark/model-registry.schema.json b/eval/embedding-benchmark/model-registry.schema.json index 84e328c5..d4765869 100644 --- a/eval/embedding-benchmark/model-registry.schema.json +++ b/eval/embedding-benchmark/model-registry.schema.json @@ -22,17 +22,19 @@ "identity": { "type": "object", "additionalProperties": false, - "required": ["modelId", "revision", "artifactDigest", "dimension", "normalization", "pooling", "queryPrefix", "documentPrefix", "reduction", "precision", "batchSize"], + "required": ["modelId", "revision", "artifactDigest", "dimension", "transformationPipeline", "pooling", "queryPrefix", "documentPrefix", "precision", "batchSize"], "properties": { "modelId": {"type": "string", "minLength": 1}, "revision": {"type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}, "artifactDigest": {"$ref": "#/$defs/digest"}, "dimension": {"const": 384}, - "normalization": {"enum": ["l2", "none"]}, + "transformationPipeline": { + "description": "Complete ordered vector transformation, including upstream normalization, dimensional reduction, and final normalization. Both supported pipelines emit unit-norm 384-dimensional vectors.", + "enum": ["l2 -> truncate 1024->384 -> l2", "l2 -> keep native 384 -> l2"] + }, "pooling": {"type": "string", "minLength": 1}, "queryPrefix": {"type": "string"}, "documentPrefix": {"type": "string"}, - "reduction": {"enum": ["none_native_384", "matryoshka_truncate_384"]}, "precision": {"type": "string", "minLength": 1}, "batchSize": {"type": "integer", "minimum": 1} } diff --git a/eval/embedding-benchmark/report.schema.json b/eval/embedding-benchmark/report.schema.json index d4b82aef..8737fde9 100644 --- a/eval/embedding-benchmark/report.schema.json +++ b/eval/embedding-benchmark/report.schema.json @@ -63,17 +63,19 @@ "identity": { "type": "object", "additionalProperties": false, - "required": ["modelId", "revision", "artifactDigest", "dimension", "normalization", "pooling", "queryPrefix", "documentPrefix", "reduction", "precision", "batchSize"], + "required": ["modelId", "revision", "artifactDigest", "dimension", "transformationPipeline", "pooling", "queryPrefix", "documentPrefix", "precision", "batchSize"], "properties": { "modelId": {"type": "string", "minLength": 1}, "revision": {"type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"}, "artifactDigest": {"$ref": "#/$defs/digest"}, "dimension": {"const": 384}, - "normalization": {"enum": ["l2", "none"]}, + "transformationPipeline": { + "description": "Complete ordered vector transformation, including upstream normalization, dimensional reduction, and final normalization.", + "enum": ["l2 -> truncate 1024->384 -> l2", "l2 -> keep native 384 -> l2"] + }, "pooling": {"type": "string", "minLength": 1}, "queryPrefix": {"type": "string"}, "documentPrefix": {"type": "string"}, - "reduction": {"enum": ["none_native_384", "matryoshka_truncate_384"]}, "precision": {"type": "string", "minLength": 1}, "batchSize": {"type": "integer", "minimum": 1} } diff --git a/eval/embedding_benchmark.py b/eval/embedding_benchmark.py index d6efc5e7..823f2706 100644 --- a/eval/embedding_benchmark.py +++ b/eval/embedding_benchmark.py @@ -20,8 +20,17 @@ REPORT_SCHEMA_VERSION: Final = "context-engine-embedding-benchmark-report-v1" _PINNED_REVISION = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") _SHA256_DIGEST = re.compile(r"^[0-9a-f]{64}$") -_NORMALIZATIONS: Final = frozenset({"l2", "none"}) -_REDUCTIONS: Final = frozenset({"none_native_384", "matryoshka_truncate_384"}) +PRIMARY_TRANSFORMATION_PIPELINE: Final = "l2 -> truncate 1024->384 -> l2" +BASELINE_TRANSFORMATION_PIPELINE: Final = "l2 -> keep native 384 -> l2" +_MAX_JSON_BYTES: Final = 16 * 1024 * 1024 +_MAX_JSON_DEPTH: Final = 64 +_MAX_JSON_CONTAINER_ITEMS: Final = 10_000 +_MAX_JSON_STRING_LENGTH: Final = 1024 * 1024 +_MAX_JSON_NODES: Final = 100_000 +_MAX_JSON_INTEGER: Final = (1 << 63) - 1 +_MIN_JSON_INTEGER: Final = -(1 << 63) +_MAX_ABS_JSON_FLOAT: Final = 1e18 +_METRIC_TOLERANCE: Final = 1e-12 class BenchmarkUnavailable(RuntimeError): @@ -36,6 +45,21 @@ class DatasetLockProfile(StrEnum): ) +class ModelTransformationPipeline(StrEnum): + """Closed, executable vector transformation identities.""" + + PRIMARY = PRIMARY_TRANSFORMATION_PIPELINE + BASELINE = BASELINE_TRANSFORMATION_PIPELINE + + @property + def raw_dimension(self) -> int: + if self is ModelTransformationPipeline.PRIMARY: + return 1024 + if self is ModelTransformationPipeline.BASELINE: + return EMBEDDING_DIMENSION + raise BenchmarkUnavailable("model identity is unresolved") + + @dataclass(frozen=True, slots=True) class ModelIdentity: """Complete, reproducible identity for one offline embedding provider.""" @@ -44,11 +68,10 @@ class ModelIdentity: revision: str artifact_digest: str dimension: int - normalization: str + transformation_pipeline: ModelTransformationPipeline pooling: str query_prefix: str document_prefix: str - reduction: str precision: str batch_size: int @@ -61,13 +84,12 @@ def __post_init__(self) -> None: or not _SHA256_DIGEST.fullmatch(self.artifact_digest) or type(self.dimension) is not int or self.dimension != EMBEDDING_DIMENSION - or self.normalization not in _NORMALIZATIONS + or type(self.transformation_pipeline) is not ModelTransformationPipeline or type(self.pooling) is not str or not self.pooling or self.pooling != self.pooling.strip() or type(self.query_prefix) is not str or type(self.document_prefix) is not str - or self.reduction not in _REDUCTIONS or type(self.precision) is not str or not self.precision or self.precision != self.precision.strip() @@ -83,12 +105,11 @@ def public_document(self) -> dict[str, object]: "dimension": self.dimension, "documentPrefix": self.document_prefix, "modelId": self.model_id, - "normalization": self.normalization, "pooling": self.pooling, "precision": self.precision, "queryPrefix": self.query_prefix, - "reduction": self.reduction, "revision": self.revision, + "transformationPipeline": self.transformation_pipeline.value, } @@ -350,12 +371,18 @@ def run_benchmark( raise BenchmarkUnavailable("benchmark configuration is unavailable") if primary.identity.model_id != "Qwen/Qwen3-Embedding-0.6B": raise BenchmarkUnavailable("primary benchmark model is unavailable") - if primary.identity.reduction != "matryoshka_truncate_384": - raise BenchmarkUnavailable("primary benchmark reduction is unavailable") + if ( + primary.identity.transformation_pipeline + is not ModelTransformationPipeline.PRIMARY + ): + raise BenchmarkUnavailable("primary benchmark pipeline is unavailable") if baseline.identity.model_id != "intfloat/multilingual-e5-small": raise BenchmarkUnavailable("baseline benchmark model is unavailable") - if baseline.identity.reduction != "none_native_384": - raise BenchmarkUnavailable("baseline benchmark reduction is unavailable") + if ( + baseline.identity.transformation_pipeline + is not ModelTransformationPipeline.BASELINE + ): + raise BenchmarkUnavailable("baseline benchmark pipeline is unavailable") primary_report = _run_model(dataset, primary, judge, top_k, clock) baseline_report = _run_model(dataset, baseline, judge, top_k, clock) model_comparison = compare_model_metrics( @@ -399,6 +426,22 @@ def public_document(self) -> dict[str, object]: return dict(self.document) +@dataclass(frozen=True, slots=True) +class _ReportPopulation: + """Structural population identity shared by both model evaluations.""" + + total_cases: int + total_expected: int + slice_counts: tuple[tuple[str, int, int], ...] + document_count: int + + +@dataclass(frozen=True, slots=True) +class _ValidatedModelReport: + values: tuple[float, float, float] + population: _ReportPopulation + + def _run_model( dataset: BenchmarkDataset, provider: BenchmarkEmbeddingProvider, @@ -628,15 +671,106 @@ def validate_report_document( """Validate the closed v1 report without adding a production dependency.""" try: - schema = json.loads(schema_path.read_text(encoding="utf-8")) + schema = load_bounded_json(schema_path) validate_json_schema_document(document, schema) _validate_report(document, schema) except BenchmarkUnavailable: raise - except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError): + except ( + OSError, + UnicodeDecodeError, + json.JSONDecodeError, + KeyError, + TypeError, + ValueError, + OverflowError, + RecursionError, + MemoryError, + ): raise BenchmarkUnavailable("benchmark report schema is unavailable") from None +def load_bounded_json(path: Path) -> object: + """Parse a bounded JSON document and reject hostile numeric extensions.""" + + try: + if path.stat().st_size > _MAX_JSON_BYTES: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + raw = path.read_text(encoding="utf-8") + if len(raw.encode("utf-8")) > _MAX_JSON_BYTES: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + value = json.loads( + raw, + parse_int=_parse_json_integer, + parse_float=_parse_json_float, + parse_constant=_reject_json_constant, + ) + _validate_bounded_json_value(value) + return value + except BenchmarkUnavailable: + raise + except ( + OSError, + UnicodeDecodeError, + json.JSONDecodeError, + OverflowError, + RecursionError, + MemoryError, + ValueError, + ): + raise BenchmarkUnavailable("benchmark JSON is unavailable") from None + + +def _parse_json_integer(raw: str) -> int: + value = int(raw) + if not _MIN_JSON_INTEGER <= value <= _MAX_JSON_INTEGER: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + return value + + +def _parse_json_float(raw: str) -> float: + value = float(raw) + if not math.isfinite(value) or abs(value) > _MAX_ABS_JSON_FLOAT: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + return value + + +def _reject_json_constant(_raw: str) -> None: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + + +def _validate_bounded_json_value(value: object) -> None: + pending: list[tuple[object, int]] = [(value, 0)] + nodes = 0 + while pending: + item, depth = pending.pop() + nodes += 1 + if nodes > _MAX_JSON_NODES or depth > _MAX_JSON_DEPTH: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + if type(item) is str: + if len(item) > _MAX_JSON_STRING_LENGTH: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + elif type(item) is int: + if not _MIN_JSON_INTEGER <= item <= _MAX_JSON_INTEGER: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + elif type(item) is float: + if not math.isfinite(item) or abs(item) > _MAX_ABS_JSON_FLOAT: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + elif type(item) is list: + if len(item) > _MAX_JSON_CONTAINER_ITEMS: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + pending.extend((child, depth + 1) for child in item) + elif type(item) is dict: + if len(item) > _MAX_JSON_CONTAINER_ITEMS or any( + type(key) is not str or len(key) > _MAX_JSON_STRING_LENGTH + for key in item + ): + raise BenchmarkUnavailable("benchmark JSON is unavailable") + pending.extend((child, depth + 1) for child in item.values()) + elif item is not None and type(item) is not bool: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + + def _validate_report(document: object, schema: object) -> None: del schema report = _closed_object( @@ -683,8 +817,12 @@ def _validate_report(document: object, schema: object) -> None: ) _require_ratio(standing["caseHitValue"]) models = _closed_object(report["models"], frozenset({"baseline", "primary"})) - primary_values = _validate_model_report(models["primary"]) - baseline_values = _validate_model_report(models["baseline"]) + primary_report = _validate_model_report(models["primary"]) + baseline_report = _validate_model_report(models["baseline"]) + if primary_report.population != baseline_report.population: + raise BenchmarkUnavailable("benchmark report schema is unavailable") + primary_values = primary_report.values + baseline_values = baseline_report.values try: declared_outcome = ModelComparisonOutcome( cast(str, comparison["primaryAgainstModelBaseline"]) @@ -711,6 +849,8 @@ def _validate_report(document: object, schema: object) -> None: def validate_json_schema_document(value: object, schema: object) -> None: """Validate documents against the tracked schema vocabulary used by eval.""" + _validate_bounded_json_value(value) + _validate_bounded_json_value(schema) root = _object(schema) _validate_json_schema(value, root, root) @@ -823,7 +963,7 @@ def _canonical_json(value: object) -> bytes: raise BenchmarkUnavailable("benchmark report schema is unavailable") from None -def _validate_model_report(value: object) -> tuple[float, float, float]: +def _validate_model_report(value: object) -> _ValidatedModelReport: report = _closed_object(value, frozenset({"identity", "metrics", "timing"})) identity = _closed_object( report["identity"], @@ -834,12 +974,11 @@ def _validate_model_report(value: object) -> tuple[float, float, float]: "dimension", "documentPrefix", "modelId", - "normalization", "pooling", "precision", "queryPrefix", - "reduction", "revision", + "transformationPipeline", } ), ) @@ -848,15 +987,18 @@ def _validate_model_report(value: object) -> tuple[float, float, float]: revision=cast(str, identity["revision"]), artifact_digest=cast(str, identity["artifactDigest"]), dimension=cast(int, identity["dimension"]), - normalization=cast(str, identity["normalization"]), + transformation_pipeline=ModelTransformationPipeline( + cast(str, identity["transformationPipeline"]) + ), pooling=cast(str, identity["pooling"]), query_prefix=cast(str, identity["queryPrefix"]), document_prefix=cast(str, identity["documentPrefix"]), - reduction=cast(str, identity["reduction"]), precision=cast(str, identity["precision"]), batch_size=cast(int, identity["batchSize"]), ) - metric_values = _validate_metrics(report["metrics"]) + metric_values, total_cases, total_expected, slice_counts = _validate_metrics( + report["metrics"] + ) timing = _closed_object( report["timing"], frozenset( @@ -867,42 +1009,73 @@ def _validate_model_report(value: object) -> tuple[float, float, float]: raise BenchmarkUnavailable("benchmark report schema is unavailable") _require_nonnegative_number(timing["perDocumentEmbedMilliseconds"]) _require_nonnegative_number(timing["wallClockMilliseconds"]) - return metric_values + return _ValidatedModelReport( + values=metric_values, + population=_ReportPopulation( + total_cases=total_cases, + total_expected=total_expected, + slice_counts=slice_counts, + document_count=timing["documentCount"], + ), + ) -def _validate_metrics(value: object) -> tuple[float, float, float]: +def _validate_metrics( + value: object, +) -> tuple[ + tuple[float, float, float], + int, + int, + tuple[tuple[str, int, int], ...], +]: metrics = _closed_object( value, frozenset({"caseHit", "evidenceRecall", "perSlice"}), ) - _validate_case_hit(metrics["caseHit"]) - _validate_recall(metrics["evidenceRecall"]) + aggregate_case_hit = _validate_case_hit(metrics["caseHit"]) + aggregate_recall = _validate_recall(metrics["evidenceRecall"]) per_slice = _object(metrics["perSlice"]) if not per_slice: raise BenchmarkUnavailable("benchmark report schema is unavailable") - for slice_metrics in per_slice.values(): + slice_case_hits: list[tuple[int, int, float]] = [] + slice_recalls: list[tuple[float, int, int, float]] = [] + slice_counts: list[tuple[str, int, int]] = [] + for slice_name, slice_metrics in sorted(per_slice.items()): item = _closed_object( slice_metrics, frozenset({"caseHit", "evidenceRecall"}), ) - _validate_case_hit(item["caseHit"]) - _validate_recall(item["evidenceRecall"]) - case_hit = _closed_object( - metrics["caseHit"], frozenset({"hits", "totalCases", "value"}) - ) - recall = _closed_object(metrics["evidenceRecall"], frozenset({"macro", "micro"})) - macro = _closed_object(recall["macro"], frozenset({"value"})) - micro = _closed_object( - recall["micro"], frozenset({"hits", "totalExpected", "value"}) - ) + case_hit = _validate_case_hit(item["caseHit"]) + recall = _validate_recall(item["evidenceRecall"]) + slice_case_hits.append(case_hit) + slice_recalls.append(recall) + slice_counts.append((slice_name, case_hit[1], recall[2])) + if ( + aggregate_case_hit[0] != sum(item[0] for item in slice_case_hits) + or aggregate_case_hit[1] != sum(item[1] for item in slice_case_hits) + or aggregate_recall[1] != sum(item[1] for item in slice_recalls) + or aggregate_recall[2] != sum(item[2] for item in slice_recalls) + ): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + weighted_macro = sum( + recall[0] * case_hit[1] + for recall, case_hit in zip(slice_recalls, slice_case_hits, strict=True) + ) / aggregate_case_hit[1] + if not _same_metric(aggregate_recall[0], weighted_macro): + raise BenchmarkUnavailable("benchmark report schema is unavailable") return ( - float(cast(int | float, case_hit["value"])), - float(cast(int | float, macro["value"])), - float(cast(int | float, micro["value"])), + ( + aggregate_case_hit[2], + aggregate_recall[0], + aggregate_recall[3], + ), + aggregate_case_hit[1], + aggregate_recall[2], + tuple(slice_counts), ) -def _validate_case_hit(value: object) -> None: +def _validate_case_hit(value: object) -> tuple[int, int, float]: metric = _closed_object(value, frozenset({"hits", "totalCases", "value"})) if ( type(metric["hits"]) is not int @@ -912,9 +1085,15 @@ def _validate_case_hit(value: object) -> None: ): raise BenchmarkUnavailable("benchmark report schema is unavailable") _require_ratio(metric["value"]) + hits = metric["hits"] + total = metric["totalCases"] + reported = float(cast(int | float, metric["value"])) + if not _same_metric(reported, hits / total): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + return hits, total, reported -def _validate_recall(value: object) -> None: +def _validate_recall(value: object) -> tuple[float, int, int, float]: recall = _closed_object(value, frozenset({"macro", "micro"})) macro = _closed_object(recall["macro"], frozenset({"value"})) _require_ratio(macro["value"]) @@ -929,6 +1108,17 @@ def _validate_recall(value: object) -> None: ): raise BenchmarkUnavailable("benchmark report schema is unavailable") _require_ratio(micro["value"]) + macro_value = float(cast(int | float, macro["value"])) + hits = micro["hits"] + total = micro["totalExpected"] + reported = float(cast(int | float, micro["value"])) + if not _same_metric(reported, hits / total): + raise BenchmarkUnavailable("benchmark report schema is unavailable") + return macro_value, hits, total, reported + + +def _same_metric(left: float, right: float) -> bool: + return math.isclose(left, right, rel_tol=0.0, abs_tol=_METRIC_TOLERANCE) def _object(value: object) -> dict[str, object]: diff --git a/tests/unit/test_embedding_benchmark_cli.py b/tests/unit/test_embedding_benchmark_cli.py index 5ea59f33..08d99c7c 100644 --- a/tests/unit/test_embedding_benchmark_cli.py +++ b/tests/unit/test_embedding_benchmark_cli.py @@ -327,3 +327,27 @@ def test_closed_backend_enum_refuses_arbitrary_import_paths() -> None: ".context-engine/report.json", ] ) + + +@pytest.mark.parametrize( + "invalid_json", + ( + '{"absurd": 9223372036854775808}', + '{"absurd": 1e1000000}', + '{"absurd": NaN}', + '{"absurd": Infinity}', + '{"absurd": -Infinity}', + "[" * 80 + "0" + "]" * 80, + json.dumps({"absurd": "x" * (1024 * 1024 + 1)}), + json.dumps({"absurd": [0] * 10_001}), + ), +) +def test_loader_refuses_pathological_json_as_a_typed_outcome( + tmp_path: Path, + invalid_json: str, +) -> None: + path = tmp_path / "dataset.json" + path.write_text(invalid_json, encoding="utf-8") + + with pytest.raises(BenchmarkUnavailable): + cli.load_dataset(path) diff --git a/tests/unit/test_embedding_benchmark_identity.py b/tests/unit/test_embedding_benchmark_identity.py index 444f4eff..d91ea797 100644 --- a/tests/unit/test_embedding_benchmark_identity.py +++ b/tests/unit/test_embedding_benchmark_identity.py @@ -9,6 +9,7 @@ EMBEDDING_DIMENSION, BenchmarkUnavailable, ModelIdentity, + ModelTransformationPipeline, ) @@ -18,11 +19,10 @@ def _resolved_identity() -> ModelIdentity: revision="a" * 40, artifact_digest="b" * 64, dimension=EMBEDDING_DIMENSION, - normalization="l2", + transformation_pipeline=ModelTransformationPipeline.PRIMARY, pooling="last_token", query_prefix="Instruct: retrieve relevant passages\nQuery:", document_prefix="", - reduction="matryoshka_truncate_384", precision="float32", batch_size=8, ) @@ -35,11 +35,10 @@ def _resolved_identity() -> ModelIdentity: ("revision", "main"), ("artifact_digest", "pending"), ("dimension", 0), - ("normalization", "unknown"), + ("transformation_pipeline", "unknown"), ("pooling", ""), ("query_prefix", None), ("document_prefix", None), - ("reduction", "unknown"), ("precision", ""), ("batch_size", 0), ), diff --git a/tests/unit/test_embedding_benchmark_metrics.py b/tests/unit/test_embedding_benchmark_metrics.py index 1623ec88..ad41f63e 100644 --- a/tests/unit/test_embedding_benchmark_metrics.py +++ b/tests/unit/test_embedding_benchmark_metrics.py @@ -17,6 +17,7 @@ MicroRecallMetric, ModelComparisonOutcome, ModelIdentity, + ModelTransformationPipeline, RetrievalJudgeCase, RetrievalMetrics, SliceMetrics, @@ -31,15 +32,14 @@ def _identity(model_id: str) -> ModelIdentity: revision="a" * 40, artifact_digest=("b" if model_id.startswith("Qwen") else "c") * 64, dimension=EMBEDDING_DIMENSION, - normalization="l2", + transformation_pipeline=( + ModelTransformationPipeline.PRIMARY + if model_id.startswith("Qwen") + else ModelTransformationPipeline.BASELINE + ), pooling="mean", query_prefix="query: ", document_prefix="passage: ", - reduction=( - "matryoshka_truncate_384" - if model_id.startswith("Qwen") - else "none_native_384" - ), precision="float32", batch_size=8, ) @@ -226,15 +226,15 @@ def _retrieval_metrics( ) -def test_runner_requires_the_declared_384_dimension_reduction_per_model() -> None: +def test_runner_requires_the_declared_transformation_pipeline_per_model() -> None: primary = SyntheticProvider( replace( _identity("Qwen/Qwen3-Embedding-0.6B"), - reduction="none_native_384", + transformation_pipeline=ModelTransformationPipeline.BASELINE, ) ) - with pytest.raises(BenchmarkUnavailable, match="reduction is unavailable"): + with pytest.raises(BenchmarkUnavailable, match="pipeline is unavailable"): run_benchmark( dataset=_dataset(), primary=primary, diff --git a/tests/unit/test_embedding_benchmark_model_registry.py b/tests/unit/test_embedding_benchmark_model_registry.py index dc407bc4..b209e7b1 100644 --- a/tests/unit/test_embedding_benchmark_model_registry.py +++ b/tests/unit/test_embedding_benchmark_model_registry.py @@ -2,6 +2,7 @@ import hashlib import json +import math from copy import deepcopy from pathlib import Path from typing import Any @@ -10,7 +11,14 @@ import rfc8785 from applications import embedding_benchmark as cli -from eval.embedding_benchmark import BenchmarkUnavailable +from eval.embedding_benchmark import ( + BenchmarkUnavailable, + ModelIdentity, + ModelTransformationPipeline, +) + +PRIMARY_PIPELINE = "l2 -> truncate 1024->384 -> l2" +BASELINE_PIPELINE = "l2 -> keep native 384 -> l2" def _registry( @@ -30,16 +38,25 @@ def _registry( "dimension": 384, "documentPrefix": "", "modelId": "Qwen/Qwen3-Embedding-0.6B", - "normalization": "l2", "pooling": "last_token", "precision": "float32", - "queryPrefix": "Instruct: retrieve relevant passages\nQuery:", - "reduction": "matryoshka_truncate_384", - "revision": "a" * 40, + "queryPrefix": ( + "Instruct: Given a web search query, retrieve relevant passages " + "that answer the query\nQuery:" + ), + "revision": "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3", + "transformationPipeline": PRIMARY_PIPELINE, }, } baseline: dict[str, Any] = deepcopy(primary) baseline["identity"]["modelId"] = "intfloat/multilingual-e5-small" + baseline["identity"]["revision"] = ( + "614241f622f53c4eeff9890bdc4f31cfecc418b3" + ) + baseline["identity"]["pooling"] = "mean" + baseline["identity"]["queryPrefix"] = "query: " + baseline["identity"]["documentPrefix"] = "passage: " + baseline["identity"]["transformationPipeline"] = BASELINE_PIPELINE models: dict[str, object] = {"baseline": baseline} if include_primary: models["primary"] = primary @@ -246,3 +263,253 @@ def test_complete_registered_layout_refuses_an_extra_untracked_file( backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, model_dir=model_dir, ) + + +@pytest.mark.parametrize( + ("model_id", "pipeline", "raw_dimension"), + ( + ("Qwen/Qwen3-Embedding-0.6B", PRIMARY_PIPELINE, 1024), + ("intfloat/multilingual-e5-small", BASELINE_PIPELINE, 384), + ), +) +def test_every_emitted_vector_is_unit_norm_after_the_declared_pipeline( + model_id: str, + pipeline: str, + raw_dimension: int, + monkeypatch: Any, +) -> None: + raw_value = 1.0 / math.sqrt(raw_dimension) + encode_arguments: list[dict[str, object]] = [] + + class FakeModel: + def encode(self, _values: list[str], **arguments: object) -> list[list[float]]: + encode_arguments.append(arguments) + return [[raw_value] * raw_dimension] + + class FakeBackend: + @staticmethod + def SentenceTransformer(*_args: object, **_kwargs: object) -> FakeModel: + return FakeModel() + + monkeypatch.setattr(cli, "_load_sentence_transformers", lambda: FakeBackend) + identity = ModelIdentity( + model_id=model_id, + revision="a" * 40, + artifact_digest="b" * 64, + dimension=384, + transformation_pipeline=ModelTransformationPipeline(pipeline), + pooling="mean", + query_prefix="query: ", + document_prefix="passage: ", + precision="float32", + batch_size=8, + ) + provider = cli.SentenceTransformersProvider( + identity=identity, + model_dir=Path("unused"), + ) + + vector = provider.embed_queries(("query",))[0] + + assert len(vector) == 384 + assert math.sqrt(sum(value * value for value in vector)) == pytest.approx( + 1.0, abs=1e-6 + ) + assert encode_arguments[0]["normalize_embeddings"] is True + + +def test_registry_pipeline_round_trips_into_the_report_identity( + tmp_path: Path, + monkeypatch: Any, +) -> None: + artifact = b"tracked artifact bytes" + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "model.safetensors").write_bytes(artifact) + registry_path = tmp_path / "model-registry.json" + _write_registry( + registry_path, + expected_files={"model.safetensors": artifact}, + ) + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + identity = cli.load_registered_model_identity( + role="primary", + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) + + assert identity.public_document()["transformationPipeline"] == PRIMARY_PIPELINE + + +def test_registry_declaring_a_pipeline_the_model_code_does_not_perform_is_refused( + tmp_path: Path, + monkeypatch: Any, +) -> None: + artifact = b"tracked artifact bytes" + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "model.safetensors").write_bytes(artifact) + registry = _registry( + expected_files={"model.safetensors": artifact}, + ) + models = registry["models"] + assert isinstance(models, dict) + primary = models["primary"] + assert isinstance(primary, dict) + identity = primary["identity"] + assert isinstance(identity, dict) + identity["transformationPipeline"] = BASELINE_PIPELINE + registry_path = tmp_path / "model-registry.json" + registry_path.write_text(json.dumps(registry), encoding="utf-8") + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + with pytest.raises(BenchmarkUnavailable, match="model identity is unresolved"): + cli.load_registered_model_identity( + role="primary", + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) + + +@pytest.mark.parametrize( + ("field_name", "changed_value"), + ( + ("revision", "f" * 40), + ("pooling", "mean"), + ), +) +def test_loader_refuses_valid_but_unapproved_non_pipeline_identity_mutations( + tmp_path: Path, + monkeypatch: Any, + field_name: str, + changed_value: str, +) -> None: + artifact = b"tracked artifact bytes" + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "model.safetensors").write_bytes(artifact) + registry = _registry(expected_files={"model.safetensors": artifact}) + models = registry["models"] + assert isinstance(models, dict) + primary = models["primary"] + assert isinstance(primary, dict) + identity = primary["identity"] + assert isinstance(identity, dict) + identity[field_name] = changed_value + registry_path = tmp_path / "model-registry.json" + registry_path.write_text(json.dumps(registry), encoding="utf-8") + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + with pytest.raises(BenchmarkUnavailable, match="model identity is unresolved"): + cli.load_registered_model_identity( + role="primary", + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) + + +def test_artifacts_are_reverified_after_the_backend_loads( + tmp_path: Path, + monkeypatch: Any, +) -> None: + artifact = b"tracked artifact bytes" + model_dir = tmp_path / "model" + model_dir.mkdir() + artifact_path = model_dir / "model.safetensors" + artifact_path.write_bytes(artifact) + registry_path = tmp_path / "model-registry.json" + _write_registry( + registry_path, + expected_files={"model.safetensors": artifact}, + ) + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + class SwappingSentenceTransformer: + def __init__(self, *_args: object, **_kwargs: object) -> None: + artifact_path.write_bytes(b"swapped after verification") + + class FakeBackend: + SentenceTransformer = SwappingSentenceTransformer + + monkeypatch.setattr(cli, "_load_sentence_transformers", lambda: FakeBackend) + + with pytest.raises(BenchmarkUnavailable, match="model identity is unresolved"): + cli.build_local_provider( + cli.SupportedBackend.SENTENCE_TRANSFORMERS, + "primary", + model_dir, + ) + + +def test_tracked_registry_pins_every_load_bearing_identity_field() -> None: + registry = json.loads(cli.MODEL_REGISTRY_PATH.read_text(encoding="utf-8")) + + primary_digest = ( + "8cb25677d5be69ce6ac88ebbdfb5dad30980fee39c35c6324a583e325917eddc" + ) + baseline_digest = ( + "961e0953328daf4df7916c1406751c5c63fe19e14a5c7e977d9f8b38964c5cf3" + ) + qwen_query_prefix = ( + "Instruct: Given a web search query, retrieve relevant passages " + "that answer the query\nQuery:" + ) + assert registry["models"]["primary"]["identity"] == { + "artifactDigest": primary_digest, + "batchSize": 8, + "dimension": 384, + "documentPrefix": "", + "modelId": "Qwen/Qwen3-Embedding-0.6B", + "pooling": "last_token", + "precision": "float32", + "queryPrefix": qwen_query_prefix, + "revision": "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3", + "transformationPipeline": PRIMARY_PIPELINE, + } + assert registry["models"]["baseline"]["identity"] == { + "artifactDigest": baseline_digest, + "batchSize": 8, + "dimension": 384, + "documentPrefix": "passage: ", + "modelId": "intfloat/multilingual-e5-small", + "pooling": "mean", + "precision": "float32", + "queryPrefix": "query: ", + "revision": "614241f622f53c4eeff9890bdc4f31cfecc418b3", + "transformationPipeline": BASELINE_PIPELINE, + } + assert hashlib.sha256( + rfc8785.dumps(registry["models"]["primary"]) + ).hexdigest() == "c6fd8959cf770e872ca441c8b48fec7109eef681af5a7d9f47d08e26f045e152" + assert hashlib.sha256( + rfc8785.dumps(registry["models"]["baseline"]) + ).hexdigest() == "ac247591224422998acfe77af86320dcc440d63476a8fe8cd96f40bf8ba0a693" + + +@pytest.mark.parametrize( + "invalid_json", + ( + '{"models": 9223372036854775808}', + '{"models": 1e1000000}', + '{"models": NaN}', + "[" * 80 + "0" + "]" * 80, + ), +) +def test_model_registry_pathologies_fail_as_typed_unresolved_identity( + tmp_path: Path, + monkeypatch: Any, + invalid_json: str, +) -> None: + model_dir = tmp_path / "model" + model_dir.mkdir() + registry_path = tmp_path / "model-registry.json" + registry_path.write_text(invalid_json, encoding="utf-8") + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + with pytest.raises(BenchmarkUnavailable): + cli.load_registered_model_identity( + role="primary", + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) diff --git a/tests/unit/test_embedding_benchmark_report_schema.py b/tests/unit/test_embedding_benchmark_report_schema.py index fa29d5be..2df99a91 100644 --- a/tests/unit/test_embedding_benchmark_report_schema.py +++ b/tests/unit/test_embedding_benchmark_report_schema.py @@ -7,7 +7,11 @@ import pytest -from eval.embedding_benchmark import BenchmarkUnavailable, validate_report_document +from eval.embedding_benchmark import ( + BenchmarkUnavailable, + validate_json_schema_document, + validate_report_document, +) REPORT_SCHEMA = Path("eval/embedding-benchmark/report.schema.json") @@ -19,11 +23,14 @@ def _identity(model_id: str, *, pooling: str) -> dict[str, object]: "dimension": 384, "documentPrefix": "", "modelId": model_id, - "normalization": "l2", "pooling": pooling, "precision": "float32", "queryPrefix": "query: ", - "reduction": "none_native_384", + "transformationPipeline": ( + "l2 -> truncate 1024->384 -> l2" + if model_id.startswith("Qwen") + else "l2 -> keep native 384 -> l2" + ), "revision": "a" * 40, } @@ -95,9 +102,21 @@ def _set_model_metric_values( ) -> None: for role, values in (("primary", primary), ("baseline", baseline)): metrics = report["models"][role]["metrics"] - metrics["caseHit"]["value"] = values[0] + case_hits = round(values[0] * 2) + micro_hits = round(values[2] * 4) + metrics["caseHit"].update({"hits": case_hits, "value": values[0]}) metrics["evidenceRecall"]["macro"]["value"] = values[1] - metrics["evidenceRecall"]["micro"]["value"] = values[2] + metrics["evidenceRecall"]["micro"].update( + {"hits": micro_hits, "value": values[2]} + ) + slice_metrics = metrics["perSlice"]["single_doc"] + slice_metrics["caseHit"].update( + {"hits": case_hits, "value": values[0]} + ) + slice_metrics["evidenceRecall"]["macro"]["value"] = values[1] + slice_metrics["evidenceRecall"]["micro"].update( + {"hits": micro_hits, "value": values[2]} + ) report["comparison"] = { "metricDeltas": { "caseHit": primary[0] - baseline[0], @@ -239,3 +258,109 @@ def test_report_refuses_each_derived_comparison_field_when_fabricated( with pytest.raises(BenchmarkUnavailable, match="report verdict"): validate_report_document(report, schema_path=REPORT_SCHEMA) + + +@pytest.mark.parametrize( + "mutate", + ( + lambda report: report["models"]["primary"]["metrics"]["caseHit"].update( + {"hits": 0} + ), + lambda report: report["models"]["primary"]["metrics"][ + "evidenceRecall" + ]["micro"].update({"hits": 0}), + ), +) +def test_report_refuses_metric_counts_that_contradict_their_values( + mutate: Any, +) -> None: + report: dict[str, Any] = _report() + mutate(report) + + with pytest.raises(BenchmarkUnavailable, match="report schema"): + validate_report_document(report, schema_path=REPORT_SCHEMA) + + +@pytest.mark.parametrize( + "mutate", + ( + lambda report: report["models"]["primary"]["metrics"]["perSlice"][ + "single_doc" + ]["caseHit"].update({"hits": 1, "value": 0.5}), + lambda report: report["models"]["primary"]["metrics"]["perSlice"][ + "single_doc" + ]["evidenceRecall"]["macro"].update({"value": 0.5}), + lambda report: report["models"]["primary"]["metrics"]["perSlice"][ + "single_doc" + ]["evidenceRecall"]["micro"].update( + {"hits": 2, "totalExpected": 4, "value": 0.5} + ), + ), +) +def test_report_refuses_per_slice_metrics_inconsistent_with_the_aggregate( + mutate: Any, +) -> None: + report: dict[str, Any] = _report() + mutate(report) + + with pytest.raises(BenchmarkUnavailable, match="report schema"): + validate_report_document(report, schema_path=REPORT_SCHEMA) + + +@pytest.mark.parametrize( + "absurd", + ( + 9223372036854775808, + float("inf"), + float("nan"), + "x" * (1024 * 1024 + 1), + [0] * 10_001, + {"nested": {"deeper": {"value": [0] * 10_001}}}, + ), +) +def test_report_refuses_absurd_numeric_or_oversized_values_as_typed_outcomes( + absurd: object, +) -> None: + with pytest.raises(BenchmarkUnavailable): + validate_json_schema_document(absurd, {}) + + +@pytest.mark.parametrize( + "mutate", + ( + lambda report: ( + report["models"]["baseline"]["metrics"]["caseHit"].update( + {"hits": 4, "totalCases": 4, "value": 1.0} + ), + report["models"]["baseline"]["metrics"]["perSlice"]["single_doc"][ + "caseHit" + ].update({"hits": 4, "totalCases": 4, "value": 1.0}), + ), + lambda report: ( + report["models"]["baseline"]["metrics"]["evidenceRecall"][ + "micro" + ].update({"hits": 6, "totalExpected": 8, "value": 0.75}), + report["models"]["baseline"]["metrics"]["perSlice"]["single_doc"][ + "evidenceRecall" + ]["micro"].update({"hits": 6, "totalExpected": 8, "value": 0.75}), + ), + lambda report: report["models"]["baseline"]["metrics"]["perSlice"].update( + { + "renamed": report["models"]["baseline"]["metrics"]["perSlice"].pop( + "single_doc" + ) + } + ), + lambda report: report["models"]["primary"]["timing"].update( + {"documentCount": 99} + ), + ), +) +def test_report_refuses_cross_model_population_contradictions( + mutate: Any, +) -> None: + report: dict[str, Any] = _report() + mutate(report) + + with pytest.raises(BenchmarkUnavailable, match="report schema"): + validate_report_document(report, schema_path=REPORT_SCHEMA) From 4ba5b7cb6d8363cd85a1f346f3ae8dd584729700 Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 00:41:34 +0800 Subject: [PATCH 6/7] test: enforce benchmark identity mutation coverage --- eval/README.md | 9 ++ ...test_embedding_benchmark_model_registry.py | 116 ++++++++++++++++-- 2 files changed, 112 insertions(+), 13 deletions(-) diff --git a/eval/README.md b/eval/README.md index 0b3cd871..c3232822 100644 --- a/eval/README.md +++ b/eval/README.md @@ -64,6 +64,15 @@ models and local identity overrides are refused, and the runner hashes every on-disk artifact against the tracked expected digests before and after the backend loads the model. +`SentenceTransformer` requires a directory path and reopens its artifacts, so +the two checks do not create a tamper-proof snapshot. A concurrent writer could +swap files during backend construction and restore the registered bytes before +the post-load check. That adversary is outside M1's single trusted local operator +threat model: the runner provides provenance under that trusted operator, not +protection against adversarial concurrent writes. If an untrusted party gains +concurrent write access to model storage, stronger snapshot or locking semantics +require their own ADR. + The input follows `embedding-benchmark/input.schema.json`; the output follows `embedding-benchmark/report.schema.json` and must be written below the ignored `.context-engine/` directory. The input includes the typed RFC 8785/SHA-256 diff --git a/tests/unit/test_embedding_benchmark_model_registry.py b/tests/unit/test_embedding_benchmark_model_registry.py index b209e7b1..199ded66 100644 --- a/tests/unit/test_embedding_benchmark_model_registry.py +++ b/tests/unit/test_embedding_benchmark_model_registry.py @@ -5,7 +5,8 @@ import math from copy import deepcopy from pathlib import Path -from typing import Any +from types import SimpleNamespace +from typing import Any, cast import pytest import rfc8785 @@ -91,18 +92,35 @@ def _expected_directory_digest(files: dict[str, bytes]) -> str: return hashlib.sha256(rfc8785.dumps(manifest)).hexdigest() +def _identity_digest(identity: dict[str, object]) -> str: + digestable = { + key: value for key, value in identity.items() if key != "artifactDigest" + } + return hashlib.sha256(rfc8785.dumps(cast(Any, digestable))).hexdigest() + + def test_model_artifact_must_match_the_tracked_registry_digest( tmp_path: Path, monkeypatch: Any, ) -> None: model_dir = tmp_path / "model" model_dir.mkdir() - (model_dir / "model.safetensors").write_bytes(b"untracked artifact bytes") + actual = b"untracked artifact bytes" + (model_dir / "model.safetensors").write_bytes(actual) registry_path = tmp_path / "model-registry.json" - _write_registry( - registry_path, + registry = _registry( expected_files={"model.safetensors": b"tracked artifact bytes"}, ) + models = registry["models"] + assert isinstance(models, dict) + primary = models["primary"] + assert isinstance(primary, dict) + identity = primary["identity"] + assert isinstance(identity, dict) + identity["artifactDigest"] = _expected_directory_digest( + {"model.safetensors": actual} + ) + registry_path.write_text(json.dumps(registry), encoding="utf-8") monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) with pytest.raises(BenchmarkUnavailable, match="model identity is unresolved"): @@ -190,6 +208,76 @@ def test_matching_artifact_uses_only_the_tracked_identity( ) +@pytest.mark.parametrize("role", ("primary", "baseline")) +def test_loader_round_trips_every_registered_identity_field( + tmp_path: Path, + monkeypatch: Any, + role: str, +) -> None: + artifact = b"tracked artifact bytes" + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "model.safetensors").write_bytes(artifact) + registry = _registry(expected_files={"model.safetensors": artifact}) + registry_path = tmp_path / "model-registry.json" + registry_path.write_text(json.dumps(registry), encoding="utf-8") + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + + identity = cli.load_registered_model_identity( + role=role, + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) + + models = registry["models"] + assert isinstance(models, dict) + model = models[role] + assert isinstance(model, dict) + assert identity.public_document() == model["identity"] + + +def test_loader_reads_dimension_from_the_validated_registry( + tmp_path: Path, + monkeypatch: Any, +) -> None: + artifact = b"tracked artifact bytes" + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "model.safetensors").write_bytes(artifact) + registry = _registry(expected_files={"model.safetensors": artifact}) + models = registry["models"] + assert isinstance(models, dict) + primary = models["primary"] + assert isinstance(primary, dict) + identity_document = primary["identity"] + assert isinstance(identity_document, dict) + identity_document["dimension"] = 385 + registry_path = tmp_path / "model-registry.json" + registry_path.write_text(json.dumps(registry), encoding="utf-8") + monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) + monkeypatch.setattr(cli, "validate_json_schema_document", lambda *_args: None) + monkeypatch.setitem( + cli._REGISTERED_IDENTITY_DIGESTS, + "primary", + _identity_digest(identity_document), + ) + captured: dict[str, object] = {} + + def capture_identity(**values: object) -> Any: + captured.update(values) + return SimpleNamespace(**values) + + monkeypatch.setattr(cli, "ModelIdentity", capture_identity) + + cli.load_registered_model_identity( + role="primary", + backend=cli.SupportedBackend.SENTENCE_TRANSFORMERS, + model_dir=model_dir, + ) + + assert captured["dimension"] == 385 + + def test_complete_registered_sentence_transformer_layout_is_accepted( tmp_path: Path, monkeypatch: Any, @@ -243,18 +331,15 @@ def test_complete_registered_layout_refuses_an_extra_untracked_file( tmp_path: Path, monkeypatch: Any, ) -> None: - files = { - "config.json": b"model config", - "model.safetensors": b"weights", - "modules.json": b"modules", - } + artifact = b"same bytes under the wrong path" model_dir = tmp_path / "model" model_dir.mkdir() - for relative_path, content in files.items(): - (model_dir / relative_path).write_bytes(content) - (model_dir / "caller-identity.json").write_text("{}", encoding="utf-8") + (model_dir / "renamed.safetensors").write_bytes(artifact) registry_path = tmp_path / "model-registry.json" - _write_registry(registry_path, expected_files=files) + _write_registry( + registry_path, + expected_files={"model.safetensors": artifact}, + ) monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) with pytest.raises(BenchmarkUnavailable, match="model identity is unresolved"): @@ -360,6 +445,11 @@ def test_registry_declaring_a_pipeline_the_model_code_does_not_perform_is_refuse identity = primary["identity"] assert isinstance(identity, dict) identity["transformationPipeline"] = BASELINE_PIPELINE + monkeypatch.setitem( + cli._REGISTERED_IDENTITY_DIGESTS, + "primary", + _identity_digest(identity), + ) registry_path = tmp_path / "model-registry.json" registry_path.write_text(json.dumps(registry), encoding="utf-8") monkeypatch.setattr(cli, "MODEL_REGISTRY_PATH", registry_path) From 1d805c3903bb4fd10d582bcb4af721f2ffdecb2d Mon Sep 17 00:00:00 2001 From: stone Date: Thu, 30 Jul 2026 00:49:58 +0800 Subject: [PATCH 7/7] fix: harden benchmark input and metric equality --- eval/embedding-benchmark/report.schema.json | 2 +- eval/embedding_benchmark.py | 89 ++++++++++++++----- tests/unit/test_embedding_benchmark_cli.py | 53 +++++++++++ .../unit/test_embedding_benchmark_metrics.py | 27 ++++++ .../test_embedding_benchmark_report_schema.py | 39 ++++++++ 5 files changed, 187 insertions(+), 23 deletions(-) diff --git a/eval/embedding-benchmark/report.schema.json b/eval/embedding-benchmark/report.schema.json index 8737fde9..9652c09b 100644 --- a/eval/embedding-benchmark/report.schema.json +++ b/eval/embedding-benchmark/report.schema.json @@ -24,7 +24,7 @@ "properties": {"caseHitValue": {"type": "number", "minimum": 0, "maximum": 1}} }, "comparison": { - "description": "Frozen Pareto verdict across case hit, macro Evidence recall, and micro Evidence recall. One model wins iff no worse on all three and strictly better on at least one; exact equality ties; mixed wins are inconclusive. Per-slice metrics do not enter the verdict and weighted blending is prohibited.", + "description": "Frozen Pareto verdict across case hit, macro Evidence recall, and micro Evidence recall. All metric consistency checks, delta calculation, standing-baseline comparison, and Pareto comparison use one equality rule: zero relative tolerance and 1e-12 absolute tolerance. Values within that tolerance compare equal and record a zero delta; strict dominance requires exceeding it. One model wins iff no worse on all three and strictly better on at least one; all-equal metrics tie; mixed wins are inconclusive. Per-slice metrics do not enter the verdict and weighted blending is prohibited.", "type": "object", "additionalProperties": false, "required": ["primaryAgainstModelBaseline", "primaryAgainstStandingTwinBaseline", "metricDeltas"], diff --git a/eval/embedding_benchmark.py b/eval/embedding_benchmark.py index 823f2706..b349eff6 100644 --- a/eval/embedding_benchmark.py +++ b/eval/embedding_benchmark.py @@ -4,7 +4,9 @@ import json import math +import os import re +import stat from collections.abc import Callable, Mapping from dataclasses import dataclass, field from enum import StrEnum @@ -604,9 +606,13 @@ def compare_model_metrics( return ModelComparison( outcome=outcome, deltas={ - "caseHit": primary_values[0] - baseline_values[0], - "macroEvidenceRecall": primary_values[1] - baseline_values[1], - "microEvidenceRecall": primary_values[2] - baseline_values[2], + "caseHit": _metric_delta(primary_values[0], baseline_values[0]), + "macroEvidenceRecall": _metric_delta( + primary_values[1], baseline_values[1] + ), + "microEvidenceRecall": _metric_delta( + primary_values[2], baseline_values[2] + ), }, ) @@ -615,25 +621,24 @@ def _pareto_outcome( primary_values: tuple[float, float, float], baseline_values: tuple[float, float, float], ) -> ModelComparisonOutcome: - if primary_values == baseline_values: - return ModelComparisonOutcome.TIE - if all( - left >= right + comparisons = tuple( + _compare_metric(left, right) for left, right in zip(primary_values, baseline_values, strict=True) - ): + ) + if all(comparison == 0 for comparison in comparisons): + return ModelComparisonOutcome.TIE + if all(comparison >= 0 for comparison in comparisons): return ModelComparisonOutcome.WIN - if all( - left <= right - for left, right in zip(primary_values, baseline_values, strict=True) - ): + if all(comparison <= 0 for comparison in comparisons): return ModelComparisonOutcome.LOSE return ModelComparisonOutcome.INCONCLUSIVE def _comparison(left: float, right: float) -> str: - if left > right: + comparison = _compare_metric(left, right) + if comparison > 0: return "win" - if left < right: + if comparison < 0: return "lose" return "tie" @@ -694,16 +699,31 @@ def load_bounded_json(path: Path) -> object: """Parse a bounded JSON document and reject hostile numeric extensions.""" try: - if path.stat().st_size > _MAX_JSON_BYTES: - raise BenchmarkUnavailable("benchmark JSON is unavailable") - raw = path.read_text(encoding="utf-8") - if len(raw.encode("utf-8")) > _MAX_JSON_BYTES: + path_metadata = path.lstat() + if not stat.S_ISREG(path_metadata.st_mode): raise BenchmarkUnavailable("benchmark JSON is unavailable") + flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_size > _MAX_JSON_BYTES + ): + raise BenchmarkUnavailable("benchmark JSON is unavailable") + with os.fdopen(descriptor, "rb", closefd=False) as stream: + encoded = stream.read(_MAX_JSON_BYTES + 1) + if len(encoded) > _MAX_JSON_BYTES: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + raw = encoded.decode("utf-8") + finally: + os.close(descriptor) value = json.loads( raw, parse_int=_parse_json_integer, parse_float=_parse_json_float, parse_constant=_reject_json_constant, + object_pairs_hook=_reject_duplicate_json_keys, ) _validate_bounded_json_value(value) return value @@ -739,6 +759,17 @@ def _reject_json_constant(_raw: str) -> None: raise BenchmarkUnavailable("benchmark JSON is unavailable") +def _reject_duplicate_json_keys( + pairs: list[tuple[str, object]], +) -> dict[str, object]: + document: dict[str, object] = {} + for key, value in pairs: + if key in document: + raise BenchmarkUnavailable("benchmark JSON is unavailable") + document[key] = value + return document + + def _validate_bounded_json_value(value: object) -> None: pending: list[tuple[object, int]] = [(value, 0)] nodes = 0 @@ -831,9 +862,13 @@ def _validate_report(document: object, schema: object) -> None: raise BenchmarkUnavailable("benchmark report verdict is unavailable") from None expected_outcome = _pareto_outcome(primary_values, baseline_values) expected_deltas = { - "caseHit": primary_values[0] - baseline_values[0], - "macroEvidenceRecall": primary_values[1] - baseline_values[1], - "microEvidenceRecall": primary_values[2] - baseline_values[2], + "caseHit": _metric_delta(primary_values[0], baseline_values[0]), + "macroEvidenceRecall": _metric_delta( + primary_values[1], baseline_values[1] + ), + "microEvidenceRecall": _metric_delta( + primary_values[2], baseline_values[2] + ), } expected_standing = _comparison( primary_values[0], cast(float, standing["caseHitValue"]) @@ -1118,7 +1153,17 @@ def _validate_recall(value: object) -> tuple[float, int, int, float]: def _same_metric(left: float, right: float) -> bool: - return math.isclose(left, right, rel_tol=0.0, abs_tol=_METRIC_TOLERANCE) + return _compare_metric(left, right) == 0 + + +def _compare_metric(left: float, right: float) -> int: + if math.isclose(left, right, rel_tol=0.0, abs_tol=_METRIC_TOLERANCE): + return 0 + return 1 if left > right else -1 + + +def _metric_delta(left: float, right: float) -> float: + return 0.0 if _same_metric(left, right) else left - right def _object(value: object) -> dict[str, object]: diff --git a/tests/unit/test_embedding_benchmark_cli.py b/tests/unit/test_embedding_benchmark_cli.py index 08d99c7c..79b5b9b3 100644 --- a/tests/unit/test_embedding_benchmark_cli.py +++ b/tests/unit/test_embedding_benchmark_cli.py @@ -2,6 +2,8 @@ import importlib import json +import os +import stat from pathlib import Path from typing import Any @@ -351,3 +353,54 @@ def test_loader_refuses_pathological_json_as_a_typed_outcome( with pytest.raises(BenchmarkUnavailable): cli.load_dataset(path) + + +def test_loader_refuses_duplicate_json_keys(tmp_path: Path) -> None: + path = tmp_path / "dataset.json" + path.write_text('{"duplicate": 1, "duplicate": 2}', encoding="utf-8") + + with pytest.raises(BenchmarkUnavailable, match="JSON is unavailable"): + cli.load_dataset(path) + + +def test_bounded_loader_refuses_a_fifo_before_reading(tmp_path: Path) -> None: + path = tmp_path / "dataset.pipe" + os.mkfifo(path) + + with pytest.raises(BenchmarkUnavailable, match="JSON is unavailable"): + cli.load_dataset(path) + + +def test_bounded_loader_refuses_a_directory(tmp_path: Path) -> None: + with pytest.raises(BenchmarkUnavailable, match="JSON is unavailable"): + cli.load_dataset(tmp_path) + + +def test_bounded_loader_refuses_a_device_node() -> None: + null_device = Path(os.devnull) + assert stat.S_ISCHR(null_device.stat().st_mode) + + with pytest.raises(BenchmarkUnavailable, match="JSON is unavailable"): + cli.load_dataset(null_device) + + +def test_bounded_loader_refuses_a_dangling_symlink(tmp_path: Path) -> None: + path = tmp_path / "dangling.json" + path.symlink_to(tmp_path / "missing.json") + + with pytest.raises(BenchmarkUnavailable, match="JSON is unavailable"): + cli.load_dataset(path) + + +def test_bounded_loader_refuses_a_symlink_outside_its_expected_root( + tmp_path: Path, +) -> None: + expected_root = tmp_path / "expected" + expected_root.mkdir() + outside = tmp_path / "outside.json" + outside.write_text("{}", encoding="utf-8") + path = expected_root / "dataset.json" + path.symlink_to(outside) + + with pytest.raises(BenchmarkUnavailable, match="JSON is unavailable"): + cli.load_dataset(path) diff --git a/tests/unit/test_embedding_benchmark_metrics.py b/tests/unit/test_embedding_benchmark_metrics.py index ad41f63e..53b46c2e 100644 --- a/tests/unit/test_embedding_benchmark_metrics.py +++ b/tests/unit/test_embedding_benchmark_metrics.py @@ -203,6 +203,33 @@ def test_model_verdict_uses_pareto_dominance_across_all_retrieval_metrics( } +@pytest.mark.parametrize( + ("primary", "baseline", "expected"), + ( + ((0.0, 0.5, 0.5), (0.0, 0.5, 0.5), ModelComparisonOutcome.TIE), + ( + (1e-13, 0.5, 0.5), + (0.0, 0.5, 0.5), + ModelComparisonOutcome.TIE, + ), + ( + (2e-12, 0.5, 0.5), + (0.0, 0.5, 0.5), + ModelComparisonOutcome.WIN, + ), + ), +) +def test_pareto_outcome_uses_the_same_tolerance_as_metric_consistency( + primary: tuple[float, float, float], + baseline: tuple[float, float, float], + expected: ModelComparisonOutcome, +) -> None: + assert compare_model_metrics( + _retrieval_metrics(*primary), + _retrieval_metrics(*baseline), + ).outcome is expected + + def _retrieval_metrics( case_hit: float, macro: float, diff --git a/tests/unit/test_embedding_benchmark_report_schema.py b/tests/unit/test_embedding_benchmark_report_schema.py index 2df99a91..ab6cee6b 100644 --- a/tests/unit/test_embedding_benchmark_report_schema.py +++ b/tests/unit/test_embedding_benchmark_report_schema.py @@ -364,3 +364,42 @@ def test_report_refuses_cross_model_population_contradictions( with pytest.raises(BenchmarkUnavailable, match="report schema"): validate_report_document(report, schema_path=REPORT_SCHEMA) + + +def test_report_consistency_and_pareto_share_one_numeric_tolerance() -> None: + report: dict[str, Any] = _report() + primary_metrics = report["models"]["primary"]["metrics"] + primary_metrics["caseHit"].update( + {"hits": 0, "totalCases": 2, "value": 1e-13} + ) + primary_metrics["perSlice"]["single_doc"]["caseHit"].update( + {"hits": 0, "totalCases": 2, "value": 1e-13} + ) + baseline_metrics = report["models"]["baseline"]["metrics"] + baseline_metrics["caseHit"].update( + {"hits": 0, "totalCases": 2, "value": 0.0} + ) + baseline_metrics["perSlice"]["single_doc"]["caseHit"].update( + {"hits": 0, "totalCases": 2, "value": 0.0} + ) + report["comparison"]["metricDeltas"]["caseHit"] = 0.0 + report["comparison"]["primaryAgainstModelBaseline"] = "tie" + report["comparison"]["primaryAgainstStandingTwinBaseline"] = "lose" + + validate_report_document(report, schema_path=REPORT_SCHEMA) + + +def test_report_refuses_count_value_difference_outside_shared_tolerance() -> None: + report: dict[str, Any] = _report() + primary_metrics = report["models"]["primary"]["metrics"] + primary_metrics["caseHit"].update( + {"hits": 0, "totalCases": 2, "value": 2e-12} + ) + primary_metrics["perSlice"]["single_doc"]["caseHit"].update( + {"hits": 0, "totalCases": 2, "value": 2e-12} + ) + report["comparison"]["metricDeltas"]["caseHit"] = 2e-12 + report["comparison"]["primaryAgainstModelBaseline"] = "win" + + with pytest.raises(BenchmarkUnavailable, match="report schema"): + validate_report_document(report, schema_path=REPORT_SCHEMA)