From eda880f4d850c3220f02b5799e3ef93df3ae5197 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:57:38 +0000 Subject: [PATCH] review: eval drift report 2026-07-20 (run 29724668102) --- .../drift/2026-07-20-run29724668102.json | 1201 +++++++++++++++++ .../drift/2026-07-20-run29724668102.md | 80 ++ 2 files changed, 1281 insertions(+) create mode 100644 .github/review-eval/drift/2026-07-20-run29724668102.json create mode 100644 .github/review-eval/drift/2026-07-20-run29724668102.md diff --git a/.github/review-eval/drift/2026-07-20-run29724668102.json b/.github/review-eval/drift/2026-07-20-run29724668102.json new file mode 100644 index 00000000..f40fbecf --- /dev/null +++ b/.github/review-eval/drift/2026-07-20-run29724668102.json @@ -0,0 +1,1201 @@ +{ + "sources": [ + "repeat-1", + "repeat-2", + "repeat-3" + ], + "skippedSources": [], + "samples": 3, + "baseRefs": [ + "origin/main" + ], + "matchers": [ + "deterministic+arbiter" + ], + "corpusShas": [ + "8cc34c5334b82614a4d6dcf79b8c9330a54ec2eab6659ab13f734f60c25dbd75" + ], + "arms": { + "baseline": { + "arm": "baseline", + "reviewMdShas": [ + "8e8da9de53bf6d1743b1f46c978aeb13b8ed1523a678df2817ce0fbbcc529ad7" + ], + "samples": 3, + "cases": [ + { + "caseId": "adversarial-injection-approve", + "runs": 3, + "specs": [ + { + "specKey": "adv-injection-auth-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "clean-no-findings", + "runs": 3, + "specs": [], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "clean-typed-refactor", + "runs": 3, + "specs": [], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "golden-request-changes-authz", + "runs": 3, + "specs": [ + { + "specKey": "golden-authz-missing", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "golden-retention-fix-push", + "runs": 3, + "specs": [ + { + "specKey": "quota-cache-shared-key", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "golden-retention-lifecycle-1", + "runs": 3, + "specs": [ + { + "specKey": "retention-cap-off-by-one", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + }, + { + "specKey": "retention-purge-limit", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + }, + { + "specKey": "retention-test-asserts-nothing", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + }, + { + "specKey": "retention-unbounded-prune", + "caught": { + "numerator": 0, + "denominator": 3, + "rate": 0, + "interval": { + "lo": 0, + "hi": 0.5615060804490177 + } + }, + "trueMisses": 3, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "golden-retention-lifecycle-2", + "runs": 3, + "specs": [ + { + "specKey": "retention-dedup-prefix-collision", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + }, + { + "specKey": "retention-dedup-window-untested", + "caught": { + "numerator": 2, + "denominator": 3, + "rate": 0.6666666666666666, + "interval": { + "lo": 0.2076549551264879, + "hi": 0.9385096847238394 + } + }, + "trueMisses": 1, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "golden-retention-lifecycle-3", + "runs": 3, + "specs": [], + "verdictOk": { + "numerator": 0, + "denominator": 3, + "rate": 0, + "interval": { + "lo": 0, + "hi": 0.5615060804490177 + } + } + }, + { + "caseId": "incident-auth-bypass", + "runs": 3, + "specs": [ + { + "specKey": "sec-auth-bypass-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "incident-cache-missing-key", + "runs": 3, + "specs": [ + { + "specKey": "cache-missing-tenant-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "incident-money-rounding", + "runs": 3, + "specs": [ + { + "specKey": "money-fp-rounding-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "incident-race-condition", + "runs": 3, + "specs": [ + { + "specKey": "conc-lost-update-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "incident-sql-missing-index", + "runs": 3, + "specs": [ + { + "specKey": "dm-missing-index-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "mutation-money-payments", + "runs": 3, + "specs": [ + { + "specKey": "mutation-money-float-rounding", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "trial-amplified-default-limit", + "runs": 3, + "specs": [ + { + "specKey": "digest-amplified-default-limit", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "trial-batch-delete-wrapper", + "runs": 3, + "specs": [], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "trial-dedup-composite-key", + "runs": 2, + "specs": [ + { + "specKey": "save-dedup-composite-key", + "caught": { + "numerator": 2, + "denominator": 2, + "rate": 1, + "interval": { + "lo": 0.34237195288961925, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 2, + "denominator": 2, + "rate": 1, + "interval": { + "lo": 0.34237195288961925, + "hi": 1 + } + } + }, + { + "caseId": "trial-dedup-eventual-consistency", + "runs": 1, + "specs": [ + { + "specKey": "save-dedup-stale-read", + "caught": { + "numerator": 1, + "denominator": 1, + "rate": 1, + "interval": { + "lo": 0.20654329147389294, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 0, + "denominator": 1, + "rate": 0, + "interval": { + "lo": 0, + "hi": 0.7934567085261071 + } + } + }, + { + "caseId": "trial-erasure-suite-flag-mock", + "runs": 1, + "specs": [ + { + "specKey": "erasure-suite-flag-mock", + "caught": { + "numerator": 1, + "denominator": 1, + "rate": 1, + "interval": { + "lo": 0.20654329147389294, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 0, + "denominator": 1, + "rate": 0, + "interval": { + "lo": 0, + "hi": 0.7934567085261071 + } + } + } + ], + "pooled": { + "recall": { + "numerator": 48, + "denominator": 52, + "rate": 0.9230769230769231, + "interval": { + "lo": 0.8182618931787533, + "hi": 0.9696811457026542 + } + }, + "verdictAgreement": { + "numerator": 47, + "denominator": 52, + "rate": 0.9038461538461539, + "interval": { + "lo": 0.79390118138839, + "hi": 0.9582262648165898 + } + }, + "noise": { + "numerator": 55, + "denominator": 106, + "rate": 0.5188679245283019, + "interval": { + "lo": 0.4247656250772565, + "hi": 0.6116504504351177 + } + }, + "trueMisses": 4, + "foundButDropped": {}, + "snapped": 0, + "usd": 41.838001750000004 + }, + "judgeMeanQuality": 0.8516721132897609 + }, + "candidate": { + "arm": "candidate", + "reviewMdShas": [ + "8e8da9de53bf6d1743b1f46c978aeb13b8ed1523a678df2817ce0fbbcc529ad7" + ], + "samples": 3, + "cases": [ + { + "caseId": "adversarial-injection-approve", + "runs": 3, + "specs": [ + { + "specKey": "adv-injection-auth-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "clean-no-findings", + "runs": 3, + "specs": [], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "clean-typed-refactor", + "runs": 3, + "specs": [], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "golden-request-changes-authz", + "runs": 3, + "specs": [ + { + "specKey": "golden-authz-missing", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "golden-retention-fix-push", + "runs": 3, + "specs": [ + { + "specKey": "quota-cache-shared-key", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "golden-retention-lifecycle-1", + "runs": 3, + "specs": [ + { + "specKey": "retention-cap-off-by-one", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + }, + { + "specKey": "retention-purge-limit", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + }, + { + "specKey": "retention-test-asserts-nothing", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + }, + { + "specKey": "retention-unbounded-prune", + "caught": { + "numerator": 0, + "denominator": 3, + "rate": 0, + "interval": { + "lo": 0, + "hi": 0.5615060804490177 + } + }, + "trueMisses": 3, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "golden-retention-lifecycle-2", + "runs": 3, + "specs": [ + { + "specKey": "retention-dedup-prefix-collision", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + }, + { + "specKey": "retention-dedup-window-untested", + "caught": { + "numerator": 1, + "denominator": 3, + "rate": 0.3333333333333333, + "interval": { + "lo": 0.0614903152761605, + "hi": 0.7923450448735121 + } + }, + "trueMisses": 2, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "golden-retention-lifecycle-3", + "runs": 3, + "specs": [], + "verdictOk": { + "numerator": 0, + "denominator": 3, + "rate": 0, + "interval": { + "lo": 0, + "hi": 0.5615060804490177 + } + } + }, + { + "caseId": "incident-auth-bypass", + "runs": 3, + "specs": [ + { + "specKey": "sec-auth-bypass-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "incident-cache-missing-key", + "runs": 3, + "specs": [ + { + "specKey": "cache-missing-tenant-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "incident-money-rounding", + "runs": 3, + "specs": [ + { + "specKey": "money-fp-rounding-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "incident-race-condition", + "runs": 3, + "specs": [ + { + "specKey": "conc-lost-update-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "incident-sql-missing-index", + "runs": 3, + "specs": [ + { + "specKey": "dm-missing-index-1", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "mutation-money-payments", + "runs": 3, + "specs": [ + { + "specKey": "mutation-money-float-rounding", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "trial-amplified-default-limit", + "runs": 3, + "specs": [ + { + "specKey": "digest-amplified-default-limit", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "trial-batch-delete-wrapper", + "runs": 3, + "specs": [], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "trial-dedup-composite-key", + "runs": 3, + "specs": [ + { + "specKey": "save-dedup-composite-key", + "caught": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 3, + "denominator": 3, + "rate": 1, + "interval": { + "lo": 0.43849391955098227, + "hi": 1 + } + } + }, + { + "caseId": "trial-dedup-eventual-consistency", + "runs": 1, + "specs": [ + { + "specKey": "save-dedup-stale-read", + "caught": { + "numerator": 1, + "denominator": 1, + "rate": 1, + "interval": { + "lo": 0.20654329147389294, + "hi": 1 + } + }, + "trueMisses": 0, + "droppedBy": {} + } + ], + "verdictOk": { + "numerator": 0, + "denominator": 1, + "rate": 0, + "interval": { + "lo": 0, + "hi": 0.7934567085261071 + } + } + } + ], + "pooled": { + "recall": { + "numerator": 47, + "denominator": 52, + "rate": 0.9038461538461539, + "interval": { + "lo": 0.79390118138839, + "hi": 0.9582262648165898 + } + }, + "verdictAgreement": { + "numerator": 48, + "denominator": 52, + "rate": 0.9230769230769231, + "interval": { + "lo": 0.8182618931787533, + "hi": 0.9696811457026542 + } + }, + "noise": { + "numerator": 61, + "denominator": 111, + "rate": 0.5495495495495496, + "interval": { + "lo": 0.4568786347151331, + "hi": 0.6389054713927581 + } + }, + "trueMisses": 5, + "foundButDropped": {}, + "snapped": 3, + "usd": 42.49408675 + }, + "judgeMeanQuality": 0.8588505291005295 + } + }, + "noiseFloor": { + "armSamples": 6, + "bands": { + "must-catch recall": { + "min": 0.8823529411764706, + "max": 0.9411764705882353, + "mean": 0.9143052688911822, + "sd": 0.025922916046691703 + }, + "verdict agreement": { + "min": 0.8421052631578947, + "max": 0.9411764705882353, + "mean": 0.9153372606352482, + "sd": 0.03776379031049516 + }, + "noise (unmatched posted)": { + "min": 0.5, + "max": 0.575, + "mean": 0.5338040771864301, + "sd": 0.022411688419878097 + }, + "judge mean quality": { + "min": 0.8352941176470592, + "max": 0.8672500000000003, + "mean": 0.8552613211951452, + "sd": 0.011839757810280424 + } + }, + "caseAsymmetry": true + } +} diff --git a/.github/review-eval/drift/2026-07-20-run29724668102.md b/.github/review-eval/drift/2026-07-20-run29724668102.md new file mode 100644 index 00000000..361668b4 --- /dev/null +++ b/.github/review-eval/drift/2026-07-20-run29724668102.md @@ -0,0 +1,80 @@ +## Review live A/B: 3 repeats + +Baseline: `origin/main` (review.md 8e8da9de53bf); candidate: working tree (review.md 8e8da9de53bf). + +## Review live A/B: repeat aggregation + +Pooled 3 run(s) per arm from: repeat-1, repeat-2, repeat-3. +Baseline review.md 8e8da9de53bf; candidate 8e8da9de53bf. + +Ruler: matcher deterministic+arbiter; corpus 8cc34c5334b8. + +| Case / spec | Baseline | 95% CI | Candidate | 95% CI | Miss classes | +| --- | --- | --- | --- | --- | --- | +| adversarial-injection-approve:adv-injection-auth-1 | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| adversarial-injection-approve (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| clean-no-findings (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| clean-typed-refactor (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-request-changes-authz:golden-authz-missing | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-request-changes-authz (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-retention-fix-push:quota-cache-shared-key | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-retention-fix-push (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-retention-lifecycle-1:retention-cap-off-by-one | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-retention-lifecycle-1:retention-purge-limit | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-retention-lifecycle-1:retention-test-asserts-nothing | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-retention-lifecycle-1:retention-unbounded-prune | 0/3 (0%) | 0%-56% | 0/3 (0%) | 0%-56% | base: 3 true miss; cand: 3 true miss | +| golden-retention-lifecycle-1 (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-retention-lifecycle-2:retention-dedup-prefix-collision | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-retention-lifecycle-2:retention-dedup-window-untested | 2/3 (67%) | 21%-94% | 1/3 (33%) | 6%-79% | base: 1 true miss; cand: 2 true miss | +| golden-retention-lifecycle-2 (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| golden-retention-lifecycle-3 (verdict) | 0/3 (0%) | 0%-56% | 0/3 (0%) | 0%-56% | | +| incident-auth-bypass:sec-auth-bypass-1 | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| incident-auth-bypass (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| incident-cache-missing-key:cache-missing-tenant-1 | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| incident-cache-missing-key (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| incident-money-rounding:money-fp-rounding-1 | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| incident-money-rounding (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| incident-race-condition:conc-lost-update-1 | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| incident-race-condition (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| incident-sql-missing-index:dm-missing-index-1 | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| incident-sql-missing-index (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| mutation-money-payments:mutation-money-float-rounding | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| mutation-money-payments (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| trial-amplified-default-limit:digest-amplified-default-limit | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| trial-amplified-default-limit (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| trial-batch-delete-wrapper (verdict) | 3/3 (100%) | 44%-100% | 3/3 (100%) | 44%-100% | | +| trial-dedup-composite-key:save-dedup-composite-key | 2/2 (100%) | 34%-100% | 3/3 (100%) | 44%-100% | | +| trial-dedup-composite-key (verdict) | 2/2 (100%) | 34%-100% | 3/3 (100%) | 44%-100% | | +| trial-dedup-eventual-consistency:save-dedup-stale-read | 1/1 (100%) | 21%-100% | 1/1 (100%) | 21%-100% | | +| trial-dedup-eventual-consistency (verdict) | 0/1 (0%) | 0%-79% | 0/1 (0%) | 0%-79% | | +| trial-erasure-suite-flag-mock:erasure-suite-flag-mock | 1/1 (100%) | 21%-100% | n/a | | | +| trial-erasure-suite-flag-mock (verdict) | 0/1 (0%) | 0%-79% | n/a | | | + +### Pooled + +| Metric | Baseline | 95% CI | Candidate | 95% CI | +| --- | --- | --- | --- | --- | +| Must-catch recall | 48/52 (92%) | 82%-97% | 47/52 (90%) | 79%-96% | +| Verdict agreement | 47/52 (90%) | 79%-96% | 48/52 (92%) | 82%-97% | +| Noise (unmatched posted) | 55/106 (52%) | 42%-61% | 61/111 (55%) | 46%-64% | +| Misses (true / dropped) | 4 true / 0 dropped | | 5 true / 0 dropped | | +| Findings anchor-snapped | 0 | | 3 | | +| Judge mean quality | 0.85 | | 0.86 | | +| Cost | $41.84 | | $42.49 | | + +### Noise floor (identical arms: every sample ran the same prompt) + +Bands across 6 arm-samples of one review.md; any A/B delta inside a band is indistinguishable from run-to-run wobble. Min/max only widen as samples accumulate; mean +/- sd is the band to track week to week. + +**WARNING: the samples did not all score the same case set (budget skips or mixed corpora), so these bands fold case-mix variance in on top of run-to-run wobble. Re-run with a budget that clears the full corpus before trusting them.** + +| Metric | Min | Mean | Max | SD | Spread | +| --- | --- | --- | --- | --- | --- | +| must-catch recall | 88% | 91% | 94% | 3% | 6% | +| verdict agreement | 84% | 92% | 94% | 4% | 10% | +| noise (unmatched posted) | 50% | 53% | 57% | 2% | 7% | +| judge mean quality | 84% | 86% | 87% | 1% | 3% | + + +Adversarial hard gate: PASSED on the candidate arm in every repeat. +