diff --git a/adapters/embeddings.py b/adapters/embeddings.py index 85787140..c061de37 100644 --- a/adapters/embeddings.py +++ b/adapters/embeddings.py @@ -358,6 +358,10 @@ def _embed_with_prefix( ) except BoundedCallTimedOut: if document and len(batch) == 1: + # Deliberate disclosed conflation: PR #230 records that + # singleton inference timeouts remain sealed under + # unsupported_document_shape pending a dedicated refusal + # category decision. raise EmbeddingDocumentRefused( "Embedding document is outside provider bounds" ) from None diff --git a/docs/evaluation/2026-07-29-embedding-benchmark.json b/docs/evaluation/2026-07-29-embedding-benchmark.json index 05a2c5a8..be7b234d 100644 --- a/docs/evaluation/2026-07-29-embedding-benchmark.json +++ b/docs/evaluation/2026-07-29-embedding-benchmark.json @@ -1,8 +1,70 @@ { - "models": { - "baseline": "pending_corpus", - "primary": "pending_corpus" + "report": { + "datasetDigest": "84ce04e760e57414e5dd7277815ef69727d90e056d1e67c8bbbaa6a6c516bc87", + "documentCount": 4166, + "fullReportRetention": "ignored .context-engine root", + "models": { + "baseline": { + "caseHit": { + "hits": 13, + "totalCases": 30, + "value": 0.43333333333333335 + }, + "evidenceRecall": { + "macroValue": 0.43333333333333335, + "microHits": 13, + "microTotalExpected": 30, + "microValue": 0.43333333333333335 + }, + "modelId": "intfloat/multilingual-e5-small", + "timing": { + "perDocumentEmbedMilliseconds": 2.858616798600621, + "wallClockMilliseconds": 16778.844833839685 + } + }, + "primary": { + "caseHit": { + "hits": 17, + "totalCases": 30, + "value": 0.5666666666666667 + }, + "evidenceRecall": { + "macroValue": 0.5666666666666667, + "microHits": 17, + "microTotalExpected": 30, + "microValue": 0.5666666666666667 + }, + "modelId": "Qwen/Qwen3-Embedding-0.6B", + "timing": { + "perDocumentEmbedMilliseconds": 36.18728767594692, + "wallClockMilliseconds": 155945.89870912023 + } + } + }, + "runIdentity": "201cce2037447f3c29b8eea1da0ce1b6323f0f4311cae17fa24f3aa7c8b0dacb", + "standingTwinBaseline": { + "caseHitValue": 0.038, + "reference": "https://github.com/stone16/context-engine/issues/128" + }, + "topK": 10 }, - "result": "pending_corpus", - "schemaVersion": "context-engine-embedding-benchmark-frozen-result-v1" + "schemaVersion": "context-engine-embedding-benchmark-frozen-result-v1", + "verdict": { + "activationAcceptanceReference": "https://github.com/stone16/context-engine/issues/128#issuecomment-5161569127", + "activationAcceptance": { + "activeProfile": { + "evidenceHits": 7, + "evidenceRecall": 0.25, + "eligibleCases": 28 + }, + "twinBaseline": { + "evidenceHits": 0, + "evidenceRecall": 0.0, + "eligibleCases": 28 + } + }, + "primaryAgainstModelBaseline": "win", + "primaryAgainstStandingTwinBaseline": "win", + "winner": "Qwen/Qwen3-Embedding-0.6B" + } } diff --git a/eval/README.md b/eval/README.md index a29624e5..fc77fcb9 100644 --- a/eval/README.md +++ b/eval/README.md @@ -484,9 +484,9 @@ slice breakdowns. The real CLI fails closed with `retrieval judge is unavailable` if that adapter cannot be loaded rather than degrading to a second metric implementation. -The maintainer corpus is private and pending delivery to a durable, -maintainer-controlled location outside disposable Git worktrees. Once available, -an operator may run: +The private maintainer corpus is retained in a durable, maintainer-controlled +location outside disposable Git worktrees. An operator may reproduce the +benchmark with: ```bash uv run context-engine-embedding-benchmark run \ @@ -497,21 +497,19 @@ uv run context-engine-embedding-benchmark run \ --output .context-engine/eval/embedding-benchmark-v1.json ``` -The tracked frozen result contains metrics only. It currently records -`pending_corpus`; it must never contain queries, note titles, paths, excerpts, -or model weights. The model verdict uses Pareto dominance across case hit, -macro Evidence recall, and micro Evidence recall: one model wins only when it is -no worse on all three and strictly better on at least one. Exact equality is a -tie; mixed wins are `inconclusive`. Per-slice results remain diagnostic and no -weighted composite or tiebreak manufactures a winner. A losing or inconclusive -primary is a valid benchmark outcome, not a runner error. - -The issue remains open while the corpus is pending. A real-model run, frozen -numeric result, actual model verdict, and measured comparison to the standing -3.8% twin baseline are not complete until that durable corpus arrives. - -The first 26-case maintainer corpus is pending delivery. Converting it to v1, -building the 50-case locked pilot, preregistering sample floors, and recording a -real-corpus CLI report remain pending corpus work; synthetic tests fully exercise -the loader, composition validator, lock, judges, floors, privacy boundary, and -security veto meanwhile. +The tracked frozen result contains aggregate metrics and run identity only; it +must never contain queries, note titles, paths, excerpts, model weights, or +per-slice corpus detail. The admitted 30-case, 4,166-document run at top-k 10 +recorded Qwen at 17/30 (`0.5666666667`) and multilingual-e5-small at 13/30 +(`0.4333333333`) for case hit, macro Evidence recall, and micro Evidence recall. +Under the frozen Pareto rule, Qwen wins all three quality metrics and clears the +standing 3.8% twin baseline; e5 remains materially faster. The full schema-valid +report remains below the ignored `.context-engine/` root, while the tracked +aggregate record carries its dataset digest and run identity. + +The winner is activated under the release-bound Qwen profile described by +ADR-0102. Its measured online acceptance resolved 7/28 eligible cases for +`evidenceRecall = 0.25`, versus the standing twin's 0/28 (`0.0`); the tracked +frozen result links the aggregate-only acceptance evidence. The offline and +online measurements cover different retrieval surfaces and are reported +separately rather than blended into one score. diff --git a/tests/unit/test_embedding_benchmark_report_privacy.py b/tests/unit/test_embedding_benchmark_report_privacy.py index 0d5290db..8c968925 100644 --- a/tests/unit/test_embedding_benchmark_report_privacy.py +++ b/tests/unit/test_embedding_benchmark_report_privacy.py @@ -28,10 +28,83 @@ def test_tracked_frozen_report_cannot_carry_personal_content() -> None: assert not FORBIDDEN_KEYS.intersection(_keys(report)) assert PERSONAL_PATH.search(report_text) is None assert report == { - "models": { - "baseline": "pending_corpus", - "primary": "pending_corpus", + "report": { + "datasetDigest": ( + "84ce04e760e57414e5dd7277815ef6972" + "7d90e056d1e67c8bbbaa6a6c516bc87" + ), + "documentCount": 4166, + "fullReportRetention": "ignored .context-engine root", + "models": { + "baseline": { + "caseHit": { + "hits": 13, + "totalCases": 30, + "value": 0.43333333333333335, + }, + "evidenceRecall": { + "macroValue": 0.43333333333333335, + "microHits": 13, + "microTotalExpected": 30, + "microValue": 0.43333333333333335, + }, + "modelId": "intfloat/multilingual-e5-small", + "timing": { + "perDocumentEmbedMilliseconds": 2.858616798600621, + "wallClockMilliseconds": 16778.844833839685, + }, + }, + "primary": { + "caseHit": { + "hits": 17, + "totalCases": 30, + "value": 0.5666666666666667, + }, + "evidenceRecall": { + "macroValue": 0.5666666666666667, + "microHits": 17, + "microTotalExpected": 30, + "microValue": 0.5666666666666667, + }, + "modelId": "Qwen/Qwen3-Embedding-0.6B", + "timing": { + "perDocumentEmbedMilliseconds": 36.18728767594692, + "wallClockMilliseconds": 155945.89870912023, + }, + }, + }, + "runIdentity": ( + "201cce2037447f3c29b8eea1da0ce1b63" + "23f0f4311cae17fa24f3aa7c8b0dacb" + ), + "standingTwinBaseline": { + "caseHitValue": 0.038, + "reference": ( + "https://github.com/stone16/context-engine/issues/128" + ), + }, + "topK": 10, }, - "result": "pending_corpus", "schemaVersion": "context-engine-embedding-benchmark-frozen-result-v1", + "verdict": { + "activationAcceptanceReference": ( + "https://github.com/stone16/context-engine/issues/128" + "#issuecomment-5161569127" + ), + "activationAcceptance": { + "activeProfile": { + "evidenceHits": 7, + "evidenceRecall": 0.25, + "eligibleCases": 28, + }, + "twinBaseline": { + "evidenceHits": 0, + "evidenceRecall": 0.0, + "eligibleCases": 28, + }, + }, + "primaryAgainstModelBaseline": "win", + "primaryAgainstStandingTwinBaseline": "win", + "winner": "Qwen/Qwen3-Embedding-0.6B", + }, } diff --git a/tests/unit/test_local_qwen_embedding_provider.py b/tests/unit/test_local_qwen_embedding_provider.py index 276795c1..f6fdaf02 100644 --- a/tests/unit/test_local_qwen_embedding_provider.py +++ b/tests/unit/test_local_qwen_embedding_provider.py @@ -94,7 +94,7 @@ def encode( assert failure.value.__cause__ is None -def test_local_qwen_documents_are_encoded_in_registered_batches( +def test_local_qwen_documents_are_encoded_one_fragment_per_micro_batch( monkeypatch: Any, ) -> None: class _BatchRecordingModel: