From 9c79a96ae237df8730f3019e3d9ce6abde5666b8 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 11:53:04 -0400 Subject: [PATCH 01/15] Add super plan for #187: faster grade defaults (Haiku + per-provider) Co-Authored-By: Claude Opus 4.8 (1M context) --- plans/super/187-fast-grade-defaults.md | 239 +++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 plans/super/187-fast-grade-defaults.md diff --git a/plans/super/187-fast-grade-defaults.md b/plans/super/187-fast-grade-defaults.md new file mode 100644 index 00000000..aa75ae5b --- /dev/null +++ b/plans/super/187-fast-grade-defaults.md @@ -0,0 +1,239 @@ +# Super Plan — #187: Faster default grader model (Haiku + per-provider fast defaults) + +## Meta + +- **Ticket:** [#187](https://github.com/wjduenow/SignalForge/issues/187) — *grade: switch default grader to Haiku-4-5 (Sonnet's reasoning depth is wasted on rubric scoring)* +- **Plus user addendum:** also set faster model defaults for OpenAI and Gemini providers. +- **Base branch:** `dev` (0.6.0.dev0). PRs target `dev`. +- **Worktree:** `../worktrees/SignalForge/187-fast-grade-defaults` +- **Branch:** `feature/187-fast-grade-defaults` +- **Phase:** detailing (awaiting PR review / approval) +- **Sessions:** 1 (2026-06-02) + +--- + +## What / Why + +**What.** The grade layer is an LLM-as-judge: read an artifact (test SQL / column description / model rationale), score against ~4 rubric criterion texts, emit JSON `{score, evidence, reasoning}`. That is a small, well-bounded classification task — the Haiku-4-5 sweet spot. The default grader is currently `claude-sonnet-4-6`, whose reasoning depth is wasted here at ~3× the latency and ~3–5× the cost. + +This ticket flips the **grade-stage default** to a faster/cheaper model, and — per the user's addendum — makes the **OpenAI and Gemini** grade providers resolve to a fast default model too (today, selecting a non-Anthropic provider forces the operator to also hand-pick a model, or the Anthropic default leaks across providers). + +**Why.** Combined with #186 (grade parallelism), per-model grade phase drops from ~280s → ~10–15s. Cost per grade run drops materially. This is the cheap perf+cost win; it directly serves adoption (Architectural Commitment #4, OSS-first / "try it cheaply"). + +**Drafter stays on Sonnet** — the drafter's reasoning depth is load-bearing for SQL/description authoring. This ticket touches the grade stage only. + +--- + +## Discovery + +### Codebase findings (origin/dev, 0.6.0.dev0) + +**Provider seam (multi-provider, post #135/#136/#137).** +- `src/signalforge/llm/providers.py` — process registry `provider_for(name) -> LLMProvider`; three registered: `anthropic`, `openai`, `gemini`. No model→provider auto-detection in this module. +- `src/signalforge/llm/cost/_rollup.py:75-92` — SKU-prefix dispatch for cost attribution: `claude-`→anthropic, `gpt-`→openai, `gemini-`→gemini. +- Provider capability flags (`supports_prompt_caching`, `supports_token_count`) gate cache markers + pre-send token counting. Anthropic both `True`; OpenAI/Gemini both `False`. + +**Grade config — the landing surface.** +- `src/signalforge/grade/config.py:97` — `model: str = "claude-sonnet-4-6"` (single field). +- `:127` — `provider: str = "anthropic"`, validated against the registry (`provider_for`). +- `model` field validator only checks non-empty/non-whitespace; **no model↔provider compatibility check.** +- `grade/engine.py:364,371` — `call_llm_async(..., model=config.model, provider=config.provider, ...)`. +- **Consequence:** setting `provider: openai` while leaving `model` at its default sends `claude-sonnet-4-6` to OpenAI → runtime API error. Live tests (`test_smoke_real_api_openai.py:163`, `test_gemini_grade_live.py:141`) sidestep this by setting **both** `provider` and `model` explicitly (`gpt-4o`, `gemini-2.5-flash`). + +**Draft config `cheap_model` placeholder.** +- `src/signalforge/draft/config.py:110` — `cheap_model: str = "claude-haiku-4-5-20251001"` — declared, documented "informational; not selected automatically," **never consumed** in `src/`. + +**Pricing table — exact-match lookup.** +- `src/signalforge/llm/pricing.py` — `lookup(model)` is **exact key match**; unknown id → `EstimateUnknownModelError` (CLI tier 2). +- Anthropic keys: `claude-sonnet-4-6`, `claude-opus-4-7`, **`claude-haiku-4-5`** (note: bare SKU, NOT the dated `-20251001`). +- OpenAI keys: `gpt-4o`, **`gpt-4o-mini`**, `gpt-4.1`, `gpt-4-turbo`. +- Gemini keys: `gemini-2.5-pro`, `gemini-2.5-flash`, **`gemini-2.0-flash`**. +- `PRICE_TABLE_VERSION = "2026-05-28"`. +- **Risk:** the unused draft placeholder uses the dated `claude-haiku-4-5-20251001`, which is NOT a pricing key. Adopting that exact string as the grade default would break `--estimate`/cost-rollup. The bare SKU `claude-haiku-4-5` matches pricing and the `sonnet-4-6`/`opus-4-7` convention. + +**Fastest/cheapest known SKU per provider (from pricing):** +| Provider | Cheapest known SKU | input $/MTok | output $/MTok | +|---|---|---|---| +| Anthropic | `claude-haiku-4-5` | 0.80 | 4.00 | +| OpenAI | `gpt-4o-mini` | 0.15 | 0.60 | +| Gemini | `gemini-2.0-flash` | 0.10 | 0.40 | +| Gemini (current live-test default) | `gemini-2.5-flash` | 0.30 | 2.50 | + +**No CLI model flag.** `cli/generate.py` has no `--model`/`--grade-model`; model selection is config-only (`signalforge.yml grade:`). + +**`_PROMPT_VERSION` is model-agnostic.** `grade/prompts.py:prompt_version_template` hashes `_SYSTEM_PROMPT + rubric block + envelope tags` — model id is **not** in the hash. Flipping the default model does **not** rotate the snapshot (`b1e609fae240ac1c`); `test_prompts.py:288` and `test_prompt_cache_stability.py:57` stay green. + +**Tests impacted by a default-literal flip (model-agnostic unit suite via fakes):** +- `tests/fixtures/grade/grade_event_v1.jsonl` — carries `"model":"claude-sonnet-4-6"` (drift-detector fixture; data update, no assertion break). +- `tests/grade/test_models.py:91,139` — `_make_event()` helper default (fixture value, not assertion). +- `tests/grade/test_smoke_real_api.py` (gated `@pytest.mark.anthropic`) — uses the **default** config, so it would start calling the new default model. Shape-only assertions; no score pinned. +- `test_provider_neutrality*.py`, `test_gemini_neutrality.py` — provider-neutral; unaffected. +- `tests/research/` — **does not exist yet**; the issue asks to pin a calibration sample there per the #179 precedent. + +### Rule constraints that bind this work + +- **grade-layer.md** — DEC-023..027 locked defaults list (includes `model="claude-sonnet-4-6"`); must update docstring + locked list. `grade:` namespace, `extra="forbid"` inner / `extra="ignore"` outer. Drift detectors mandatory for `GradeEvent`/`GradingReport`. `_PROMPT_VERSION` rotation policy (text-only; stays stable here). +- **llm-drafter.md** — provider capability flags; `cheap_model` precedent (draft side); `llm:` namespace; per-provider byte-identity on `estimate_input_tokens`; non-clean finish_reason → `LLMResponseFormatError`. +- **testing-signal.md** — drift mirror + fixture lockstep; `apply_provider_override` per-test overlay (don't globally bump `max_output_tokens`); gated markers (`anthropic`/`openai`/`gemini`/`e2e`) + runtime skip; engineered determinism for LLM-driven assertions; 12 AST audit-completeness scans must still pass. +- **cli-layer.md** — four-tier exit codes; 7th AST scan over `errors.py` (new typed errors must map). Pricing `EstimateUnknownModelError` is tier 2. +- **No `workflow-project.md`** present. + +### Scoping answers (session 1) + +- **SCOPE-1 — Per-provider fast default = provider-keyed table + sentinel.** Add a `PROVIDER_FAST_MODELS` mapping `{anthropic: claude-haiku-4-5, openai: gpt-4o-mini, gemini: gemini-2.5-flash}`. Change `GradeConfig.model` default from `"claude-sonnet-4-6"` to a sentinel (`None`); resolve to the provider's fast model when unset. Setting just `provider: openai` Just Works. Explicit `model:` still honoured. +- **SCOPE-2 — Calibration ships as a gated story; maintainer runs the eval.** Add a re-grade concordance harness + `tests/research/187-*.md` writeup, gated behind the `anthropic` live-API marker. Decision rule: ≥85% pass/fail agreement vs the #179 Sonnet baseline → Haiku ships as resolved default. The maintainer runs it as a human pre-merge gate; code lands with Haiku as the resolved Anthropic default. +- **SCOPE-3 — Model ids:** `claude-haiku-4-5` / `gpt-4o-mini` / `gemini-2.5-flash`. All three are exact pricing keys. +- **SCOPE-4 — Reconcile `DraftConfig.cheap_model`** `claude-haiku-4-5-20251001` → `claude-haiku-4-5` (bare SKU, matches pricing + new grade default). Field stays unused; lockstep consistency only. + +--- + +## Architecture Review (session 1) + +| Area | Rating | Finding | +|---|---|---| +| Resolution timing | **concern → resolved** | None must resolve **at config-load**, not per-call, so `.model` is concrete everywhere downstream (estimate, engine, `GradeEvent.model`, cost-rollup). A `None` leaking past config-load crashes the cost rollup's prefix dispatch (`_rollup.py:75-92` → `CostRollupUnknownModelError`) and the estimate path's exact-match `pricing.lookup`. **Decision:** resolve in `GradeConfig`. | +| Frozen-model mechanism | **concern → resolved** | `GradeConfig` is `frozen=True` (`extra="forbid"`). A `@model_validator(mode="after")` doing `self.model = ...` raises `FrozenInstanceError`. **Decision (DEC-003):** resolve in a `@model_validator(mode="before")` that injects the provider's fast model into the raw dict when `model` is absent/None, so the field is concrete after field-validation and frozen-ness is preserved. Unknown provider → no injection → existing `provider` field-validator raises the proper `UnknownProviderError`. | +| Gemini truncation | **BLOCKER (needs user call)** | `GradeConfig.max_output_tokens` default is `256` (`grade/config.py:109`). Per `plans/super/155-gemini-truncation-e2e-gap.md`, `gemini-2.5-flash`'s verbose `reasoning` field routinely exceeds small caps and hits `MAX_TOKENS` → non-clean finish → `LLMResponseFormatError` → degraded grade (`score=None`). #155 deliberately kept the production `max_output_tokens=256` and used **per-test overlays** for Gemini. But #187 makes `gemini-2.5-flash` a *one-line production default* for `provider: gemini`, so 256 becomes a first-run footgun for Gemini operators. See refinement Q. | +| Model↔provider compat | **concern (improvement)** | No validator today ensures `model` matches `provider` (`provider: openai, model: claude-sonnet-4-6` parses, fails at runtime). The sentinel change makes a cheap fix natural: when `model` is explicitly set, check its SKU prefix matches `provider` (reusing the single prefix-dispatch source in `cost/_rollup.py`, not a duplicate). Fails loud at config-load. **Decision (DEC-006):** add it. | +| estimate byte-identity | **concern** | `--estimate` embeds `grader_model=grade_config.model` (resolved) in its report; any estimate snapshot test that pins the grade model id flips sonnet→haiku. Must update those snapshots in lockstep. Per-provider `estimate_input_tokens` byte-identity is unaffected (model id isn't in the token-count payload). | +| `_PROMPT_VERSION` snapshot | **pass** | Model id is NOT in the grade prompt-version hash (`prompts.py` hashes system prompt + rubric block + envelope only). `b1e609fae240ac1c` stays pinned; `test_prompts.py` / `test_prompt_cache_stability.py` green. | +| Unit suite (fakes) | **pass** | Grade unit tests inject `FakeAnthropicClient` / override `model="claude-fake"`; model-agnostic. Only `test_config.py` default assertions (`:190`, `:480`, `:508`), `test_models.py` `_make_event()` helper, and `grade_event_v1.jsonl` carry literal model ids → fixture/assertion updates, no logic break. | +| AST audit scans | **pass** | A module-level `PROVIDER_FAST_MODELS` constant trips no scan (scans gate event construction / SDK clients / errors.py — not data constants). No new typed errors → 7th scan unaffected. | +| Drift detectors | **concern** | `StrictGradeEvent` mirror + `grade_event_v1.jsonl` fixture carry `model`; update fixture to the resolved value, keep strict mirror in lockstep (testing-signal.md DEC-001). `GradeConfig` has no read-back fixture (it's write-only input). | +| Calibration | **pass (gated story)** | `tests/research/` does not exist; `docs/research/179-test-primitive-expansion-retest.md` is the writeup-format precedent. No reusable concordance harness exists — build a one-off per the #179 pattern, gated behind `@pytest.mark.anthropic`. | +| Constant home | **pass** | `PROVIDER_FAST_MODELS` lives in `llm/providers.py` (next to the registry; add to `__all__`), reusable by a future draft `--cheap`. Grade config imports it. | +| Draft per-provider footgun | **accepted / out of scope** | `DraftConfig` has the same latent `provider≠model` footgun, but the issue scopes the drafter to stay on Sonnet. Note as a v0.3 follow-up; this ticket only aligns `cheap_model`'s SKU (SCOPE-4). | + +No blockers remain after the refinement decisions below. + +--- + +## Refinement Log — Decisions + +- **DEC-001 — Per-provider fast grade default via sentinel.** `GradeConfig.model` changes from `str = "claude-sonnet-4-6"` to `str | None = None`. When unset, it resolves to the calling provider's fast model. The resolved Anthropic default becomes `claude-haiku-4-5` (the ticket's headline change). Explicit `model:` is always honoured. *Traces: SCOPE-1, issue #187.* +- **DEC-002 — `PROVIDER_FAST_MODELS` table in `llm/providers.py`.** `{"anthropic": "claude-haiku-4-5", "openai": "gpt-4o-mini", "gemini": "gemini-2.5-flash"}`, added to `__all__`. Every value is an exact `pricing.PRICES` key (so `--estimate`/cost-rollup never raise `EstimateUnknownModelError`). Home chosen for reuse by a future draft `--cheap` and proximity to the provider registry. *Traces: SCOPE-1, SCOPE-3.* +- **DEC-003 — Resolve in a frozen-safe `@model_validator(mode="before")`.** `GradeConfig` is `frozen=True`; a `mode="after"` `self.model = …` raises. The before-validator injects `PROVIDER_FAST_MODELS[provider]` into the raw dict when `model` is absent/None, so `.model` is concrete after field-validation. Resolution at **config-load**, never per-call — a `None` reaching the engine / `GradeEvent.model` / cost-rollup is the failure mode this prevents. Unknown provider → no injection → existing `provider` field-validator raises `UnknownProviderError`. *Traces: architecture review "resolution timing" + "frozen-model mechanism".* +- **DEC-004 — Raise `GradeConfig.max_output_tokens` default 256 → 1024 (all providers).** Prevents `gemini-2.5-flash` truncation (`MAX_TOKENS` → `LLMResponseFormatError` → degraded grade) out of the box now that Gemini is a one-line production default. It is a cap, not a target — Haiku / gpt-4o-mini rarely approach it, so the Anthropic/OpenAI cost ceiling barely moves. Supersedes #155's "keep 256 production default" *for the production default only*; #155's per-test overlays remain valid for tighter test scoping. **Verification owed:** the live calibration story must confirm `gemini-2.5-flash` does not still truncate at 1024 on real artifacts; if it does, revisit (bump further or per-provider floor). *Traces: refinement Q, #155 DEC-008/DEC-009.* +- **DEC-005 — Calibration is a maintainer-run gated story.** Re-grade a pinned sample from the #179 Phase-B `grade.jsonl` (Sonnet baseline) with the new Haiku default; compute per-criterion pass/fail concordance. Decision rule: **≥85% agreement → Haiku ships as the resolved Anthropic default** (this PR); <85% → fall back to the issue's "opt-in knob" option in a follow-up. Harness + pinned sample live under `tests/research/187-haiku-calibration/`, gated by `@pytest.mark.anthropic` + runtime env skip; prose writeup at `docs/research/187-haiku-calibration.md` mirroring `docs/research/179-test-primitive-expansion-retest.md`. The eval is a human pre-merge gate (CI can't run live API); the PR records the result. *Traces: SCOPE-2, issue #187 "Empirical calibration step (required before merge)".* +- **DEC-006 — Add a model↔provider compatibility validator.** When `model` is explicitly set, reject a SKU-prefix/provider mismatch (`provider: openai, model: claude-…`) at config-load with `GradeConfigError`, instead of failing at runtime. Single source of truth: define `PROVIDER_SKU_PREFIXES = {"anthropic": "claude-", "openai": "gpt-", "gemini": "gemini-"}` in `llm/providers.py` and refactor `cost/_rollup.py` to import it (removes the existing duplicate prefix map). No new error class → 7th AST scan unaffected. *Traces: architecture review "model↔provider compat".* +- **DEC-007 — Reconcile `DraftConfig.cheap_model` SKU.** `claude-haiku-4-5-20251001` → `claude-haiku-4-5` (bare SKU; matches pricing + the new grade default + the `sonnet-4-6`/`opus-4-7` convention). Field remains unused; lockstep consistency only. *Traces: SCOPE-4.* +- **DEC-008 — Drafter stays on Sonnet; draft per-provider resolution is out of scope.** The analogous `DraftConfig` `provider≠model` footgun is noted as a v0.3 follow-up. *Traces: issue #187 ("The drafter stays on Sonnet").* +- **DEC-009 — 5-surface lockstep for the default-flip graduation.** The default change updates: (1) `.claude/rules/grade-layer.md` (DEC-026 + locked-defaults), (2) `docs/grade-ops.md` (default model, `max_output_tokens`, cost table) + `docs/llm-providers-ops.md` (per-provider fast SKU + the Gemini token note) + `docs/draft-ops.md` (cheap_model SKU), (3) `CLAUDE.md` public-API surface + the `GradeConfig` docstring, (4) the grade config/fixture tests, (5) this plan's DEC list. CHANGELOG entry under "Changed"; README "blessed IDs" only if it enumerates the grade default. *Traces: grade-layer.md/prune-engine.md 5-surface parity rule, cli-layer.md multi-surface parity.* + +--- + +## Detailed Breakdown — Stories + +> Validation command (all stories' final gate): `pip install -e ".[dev]" && ruff check . && ruff format --check . && pyright && pytest` + +### US-001 — `PROVIDER_FAST_MODELS` + `PROVIDER_SKU_PREFIXES` constants +**Description.** Add the two provider→string mapping constants to `src/signalforge/llm/providers.py` and export them in `__all__`. Refactor `src/signalforge/llm/cost/_rollup.py` to import `PROVIDER_SKU_PREFIXES` so the SKU-prefix dispatch has a single source of truth. +**Traces to:** DEC-002, DEC-006. +**TDD (write first):** +- `PROVIDER_FAST_MODELS` covers exactly the three registered providers (`anthropic`/`openai`/`gemini`). +- Every `PROVIDER_FAST_MODELS` value is a key in `signalforge.llm.pricing.PRICES` (guards the `--estimate` contract). +- `PROVIDER_SKU_PREFIXES` keys == `PROVIDER_FAST_MODELS` keys; each fast model id `startswith` its provider's prefix. +- `_rollup.py` prefix dispatch still classifies `claude-…`/`gpt-…`/`gemini-…` correctly after the refactor (existing rollup tests stay green). +**Files:** `src/signalforge/llm/providers.py` (constants + `__all__`), `src/signalforge/llm/cost/_rollup.py` (import the prefix map, drop the local copy), `tests/llm/test_providers.py` (or `test_pricing.py`) for the new invariants. +**Depends on:** none. +**Done When:** +- [ ] Both constants defined + in `__all__`; values are exact pricing keys. +- [ ] `_rollup.py` imports `PROVIDER_SKU_PREFIXES`; no duplicate prefix literals remain. +- [ ] New invariant tests pass; existing cost-rollup tests green. +- [ ] `make verify` / canonical validation passes. + +### US-002 — GradeConfig sentinel resolver + compat validator + token-cap bump +**Description.** Migrate `GradeConfig.model` to `str | None = None` with a `mode="before"` resolver (DEC-003), add the explicit-model↔provider compatibility check (DEC-006), and raise the `max_output_tokens` default 256→1024 (DEC-004). Update the class docstring + the DEC-023..027 locked-defaults list. +**Traces to:** DEC-001, DEC-003, DEC-004, DEC-006. +**TDD (write first):** +- `GradeConfig().model == "claude-haiku-4-5"` (resolved Anthropic default). +- `GradeConfig(provider="openai").model == "gpt-4o-mini"`; `GradeConfig(provider="gemini").model == "gemini-2.5-flash"`. +- `GradeConfig(model="claude-sonnet-4-6").model == "claude-sonnet-4-6"` (explicit honoured). +- `GradeConfig(provider="openai", model="claude-sonnet-4-6")` → `ValidationError` (compat reject); same-provider explicit (`provider="openai", model="gpt-4o"`) passes. +- `GradeConfig(model=" ")` still rejected (empty-string guard runs before/independent of resolution). +- `GradeConfig().max_output_tokens == 1024`. +- `load_grade_config` with a `grade:` block omitting `model:` → resolves; with `model:` set → honoured; with mismatched provider/model → `GradeConfigError`. +- Unknown provider still raises `UnknownProviderError` (resolution doesn't mask it). +**Files:** `src/signalforge/grade/config.py`, `tests/grade/test_config.py`. +**Depends on:** US-001. +**Done When:** +- [ ] `model: str | None = None`; `mode="before"` resolver injects fast model; frozen-ness preserved (no `FrozenInstanceError`). +- [ ] Compat validator rejects explicit prefix/provider mismatch at load. +- [ ] `max_output_tokens` default == 1024; docstring + locked-defaults list updated. +- [ ] All TDD cases pass; `make verify` passes. + +### US-003 — Fixture + downstream test lockstep +**Description.** Update committed fixtures and assertions that carry the old default model id, and any `--estimate` snapshot that pins the grade model id, so the suite reflects the resolved Haiku default. Keep the drift detector green. +**Traces to:** DEC-001, DEC-009; testing-signal.md DEC-001 (drift lockstep). +**TDD / checks:** +- `tests/grade/test_config.py` default assertions updated (`:190` resolved-value; review `:480`, `:508`). +- `tests/fixtures/grade/grade_event_v1.jsonl` `model` value + `StrictGradeEvent` mirror remain consistent (drift detector passes). +- `tests/grade/test_models.py` `_make_event()` helper reviewed (explicit fixture value — update only if a test asserts the default). +- Grep for estimate snapshots embedding the grade model id (`tests/cli/` estimate tests, `_estimate` report); update any in lockstep. +**Files:** `tests/grade/test_config.py`, `tests/fixtures/grade/grade_event_v1.jsonl`, `tests/grade/test_models.py`, plus any estimate snapshot fixture surfaced by grep. +**Depends on:** US-002. +**Done When:** +- [ ] Drift detector + grade unit suite green against the new default. +- [ ] No stale `claude-sonnet-4-6` default assertion remains where the resolved default now applies. +- [ ] `make verify` passes. + +### US-004 — `DraftConfig.cheap_model` SKU alignment +**Description.** Change the unused `DraftConfig.cheap_model` default from the dated `claude-haiku-4-5-20251001` to the bare SKU `claude-haiku-4-5`; update its docstring + the DEC-017 reference. +**Traces to:** DEC-007. +**Files:** `src/signalforge/draft/config.py`, `tests/draft/test_config.py` (if it asserts the value). +**Depends on:** none (independent; can land any time before QG). +**Done When:** +- [ ] `cheap_model == "claude-haiku-4-5"`; docstring updated. +- [ ] Draft config tests green; `make verify` passes. + +### US-005 — Haiku calibration harness + writeup (gated) +**Description.** Build the one-off concordance harness under `tests/research/187-haiku-calibration/`: re-grade a pinned sample of artifacts with the new Haiku default vs the #179 Sonnet baseline, compute per-criterion pass/fail agreement, and assert the ≥85% decision rule (or report it). Gate behind `@pytest.mark.anthropic` + a runtime env skip. Add the prose writeup at `docs/research/187-haiku-calibration.md` mirroring the #179 retest format. Also verify (live) that the new `max_output_tokens=1024` default keeps `gemini-2.5-flash` grading clean (no `score=None` degrade from truncation). +**Traces to:** DEC-004 (gemini verification), DEC-005. +**TDD / determinism:** per testing-signal.md "engineered determinism" — pin the sample artifacts + Sonnet baseline as committed data so the comparison is reproducible; the only live variable is the Haiku re-grade. +**Files:** `tests/research/187-haiku-calibration/` (gated test + pinned sample json), `docs/research/187-haiku-calibration.md`, `pyproject.toml` (only if a new marker is needed — reuse `anthropic`/`gemini`). +**Depends on:** US-002. +**Done When:** +- [ ] Gated harness runs under `pytest -m anthropic --no-cov` and emits the concordance metric; deselected by default CI. +- [ ] Writeup documents method, sample, decision rule, and (maintainer-filled) result. +- [ ] Gemini-at-1024 no-truncation check present (gated `@pytest.mark.gemini`). +- [ ] Default `pytest` (non-gated) + `make verify` pass. + +### US-006 — Docs + CHANGELOG + rule lockstep (5-surface parity) +**Description.** Update every non-code surface for the default flip: `docs/grade-ops.md` (default model, `max_output_tokens=1024`, cost table with Haiku + per-provider rows), `docs/llm-providers-ops.md` (per-provider fast SKU + the Gemini token-cap note), `docs/draft-ops.md` (cheap_model SKU), `.claude/rules/grade-layer.md` (DEC-026 + locked defaults), `.claude/rules/llm-drafter.md` (`PROVIDER_FAST_MODELS` note), `CLAUDE.md` public-API surface, and a `CHANGELOG` "Changed" entry. README "blessed IDs" only if it enumerates the grade default. +**Traces to:** DEC-009. +**Files:** the docs/rules/CLAUDE.md/CHANGELOG files above. +**Depends on:** US-002 (docs reflect final shape). +**Done When:** +- [ ] All five parity surfaces agree on the new default + `max_output_tokens`. +- [ ] CHANGELOG entry added; cost table updated. +- [ ] `make verify` passes (doc-example round-trip tests, if any, green). + +### US-007 — Quality Gate +**Description.** Run the code reviewer 4× across the full changeset, fixing every real bug each pass; run CodeRabbit if available. Canonical validation must pass after all fixes. +**Traces to:** all DECs. +**Depends on:** US-001..US-006. +**Done When:** +- [ ] 4 review passes complete; all real findings fixed. +- [ ] CodeRabbit (if available) addressed. +- [ ] `ruff check . && ruff format --check . && pyright && pytest` all pass; gated markers (`anthropic`/`gemini`) spot-run by maintainer. + +### US-008 — Patterns & Memory +**Description.** Capture new patterns: the `PROVIDER_FAST_MODELS` sentinel-resolution pattern, the frozen-model `mode="before"` resolver convention, and the per-provider-default graduation. Update `.claude/rules/` / `docs/` / memory as warranted. +**Traces to:** DEC-001..DEC-009. +**Depends on:** US-007. +**Done When:** +- [ ] Rules/docs/memory updated with the sentinel-resolution + frozen-resolver patterns. +- [ ] `make verify` passes. + +--- + +## Story dependency graph + +``` +US-001 ─┬─> US-002 ─┬─> US-003 ─┐ + │ ├─> US-005 ─┤ + │ └─> US-006 ─┤ +US-004 ─────────────────────────┼─> US-007 (Quality Gate) ─> US-008 (Patterns & Memory) + ┘ +``` + From 82142142a3bd42e1d1a89b828fc7b2114365d144 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 11:53:32 -0400 Subject: [PATCH 02/15] Mark #187 plan published (PR #193) Co-Authored-By: Claude Opus 4.8 (1M context) --- plans/super/187-fast-grade-defaults.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plans/super/187-fast-grade-defaults.md b/plans/super/187-fast-grade-defaults.md index aa75ae5b..39462266 100644 --- a/plans/super/187-fast-grade-defaults.md +++ b/plans/super/187-fast-grade-defaults.md @@ -7,7 +7,8 @@ - **Base branch:** `dev` (0.6.0.dev0). PRs target `dev`. - **Worktree:** `../worktrees/SignalForge/187-fast-grade-defaults` - **Branch:** `feature/187-fast-grade-defaults` -- **Phase:** detailing (awaiting PR review / approval) +- **Phase:** published (awaiting approval) +- **PR:** [#193](https://github.com/wjduenow/SignalForge/pull/193) (draft, base `dev`) - **Sessions:** 1 (2026-06-02) --- From 0c080432618cb609968e49ea778bba533b8c4407 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 12:04:47 -0400 Subject: [PATCH 03/15] Devolve #187 plan to beads (epic SignalForge-dpy + 8 tasks) Co-Authored-By: Claude Opus 4.8 (1M context) --- plans/super/187-fast-grade-defaults.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/plans/super/187-fast-grade-defaults.md b/plans/super/187-fast-grade-defaults.md index 39462266..ad9ecd72 100644 --- a/plans/super/187-fast-grade-defaults.md +++ b/plans/super/187-fast-grade-defaults.md @@ -7,10 +7,28 @@ - **Base branch:** `dev` (0.6.0.dev0). PRs target `dev`. - **Worktree:** `../worktrees/SignalForge/187-fast-grade-defaults` - **Branch:** `feature/187-fast-grade-defaults` -- **Phase:** published (awaiting approval) -- **PR:** [#193](https://github.com/wjduenow/SignalForge/pull/193) (draft, base `dev`) +- **Phase:** devolved +- **PR:** [#193](https://github.com/wjduenow/SignalForge/pull/193) (base `dev`) - **Sessions:** 1 (2026-06-02) +### Beads Manifest + +- **Epic:** `SignalForge-dpy` — #187: Faster grade defaults — Haiku + per-provider fast models +- **Worktree (planning):** `../worktrees/SignalForge/187-fast-grade-defaults` (`feature/187-fast-grade-defaults`) + +| Bead | Story | Depends on | Ready at devolve | +|---|---|---|---| +| `SignalForge-dpy.1` | US-001 — PROVIDER_FAST_MODELS + PROVIDER_SKU_PREFIXES constants | — | ✅ ready | +| `SignalForge-dpy.2` | US-002 — GradeConfig sentinel resolver + compat validator + token-cap bump | .1 | blocked | +| `SignalForge-dpy.3` | US-003 — Fixture + downstream test lockstep | .2 | blocked | +| `SignalForge-dpy.4` | US-004 — DraftConfig.cheap_model SKU alignment | — | ✅ ready | +| `SignalForge-dpy.5` | US-005 — Haiku calibration harness + writeup (gated) | .2 | blocked | +| `SignalForge-dpy.6` | US-006 — Docs + CHANGELOG + rule lockstep | .2 | blocked | +| `SignalForge-dpy.7` | Quality Gate — code review x4 + CodeRabbit | .3, .4, .5, .6 | blocked | +| `SignalForge-dpy.8` | Patterns & Memory | .7 | blocked | + +*Note: `bd` auto-push to `origin/main` warns "no common ancestor" — the dolt remote tracks `main` but the active line is `dev`. Local beads DB is committed and intact; this is the pre-existing environment quirk, not a devolve failure.* + --- ## What / Why From 0bf3d5f8149c20d9b3f40e44f93fdf8b81521126 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 12:10:00 -0400 Subject: [PATCH 04/15] =?UTF-8?q?SignalForge-dpy.4:=20US-004=20=E2=80=94?= =?UTF-8?q?=20align=20DraftConfig.cheap=5Fmodel=20to=20bare=20SKU=20claude?= =?UTF-8?q?-haiku-4-5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dated id claude-haiku-4-5-20251001 is not a key in signalforge.llm.pricing.PRICES (which uses bare SKUs). Change the unused (declared-not-consumed) cheap_model default to the bare SKU so it matches pricing and the new grade default; update the DEC-017 reference comment and field docstring in lockstep. Lockstep consistency fix only — not wired into any code path. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/signalforge/draft/config.py | 12 +++++++----- tests/draft/test_config.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/signalforge/draft/config.py b/src/signalforge/draft/config.py index b2a637a8..2a2bf917 100644 --- a/src/signalforge/draft/config.py +++ b/src/signalforge/draft/config.py @@ -15,9 +15,10 @@ read-back / response-shaped models which use ``extra="ignore"`` for forward-compat. * **DEC-017** — Defaults: ``model="claude-sonnet-4-6"``, - ``cheap_model="claude-haiku-4-5-20251001"``, ``max_output_tokens=4096``, + ``cheap_model="claude-haiku-4-5"``, ``max_output_tokens=4096``, ``cache_ttl="5m"``, ``max_retries_429=3``, ``max_retries_5xx=1``, - ``max_retries_conn=1``. + ``max_retries_conn=1``. The bare SKU (no date suffix) matches the + keys in :data:`signalforge.llm.pricing.PRICES`. * **DEC-027** — ``signalforge.yml`` top-level namespace key for this layer is ``llm:``. Other top-level keys (``safety:``, ``prune:``, ``grade:``, …) are reserved for other stages and silently ignored by @@ -105,11 +106,12 @@ class DraftConfig(BaseModel): model: str = "claude-sonnet-4-6" """Default Anthropic model. Any string the SDK accepts is allowed — the three blessed IDs are documented in the README; ``cheap_model`` - holds the v0.1 Haiku ID.""" + holds the bare Haiku SKU.""" - cheap_model: str = "claude-haiku-4-5-20251001" + cheap_model: str = "claude-haiku-4-5" """Informational; not selected automatically. The CLI (#9) flips on - ``--cheap`` to swap ``model`` for this value.""" + ``--cheap`` to swap ``model`` for this value. The bare SKU (no date + suffix) matches the keys in :data:`signalforge.llm.pricing.PRICES`.""" max_output_tokens: int = 4096 """Anthropic ``max_tokens`` ceiling. Must be positive (validator).""" diff --git a/tests/draft/test_config.py b/tests/draft/test_config.py index eaba83da..5574bf57 100644 --- a/tests/draft/test_config.py +++ b/tests/draft/test_config.py @@ -39,7 +39,7 @@ def test_draft_config_defaults_match_dec_017() -> None: """DEC-017: every default field value matches the spec.""" cfg = DraftConfig() assert cfg.model == "claude-sonnet-4-6" - assert cfg.cheap_model == "claude-haiku-4-5-20251001" + assert cfg.cheap_model == "claude-haiku-4-5" assert cfg.max_output_tokens == 4096 assert cfg.cache_ttl == "5m" assert cfg.max_retries_429 == 3 From 418bcb7459a9ea772c3c4ad410072bd7266b48d9 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 12:11:17 -0400 Subject: [PATCH 05/15] =?UTF-8?q?SignalForge-dpy.1:=20US-001=20=E2=80=94?= =?UTF-8?q?=20PROVIDER=5FFAST=5FMODELS=20+=20PROVIDER=5FSKU=5FPREFIXES=20c?= =?UTF-8?q?onstants?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two provider->string mapping constants to signalforge.llm.providers, keyed by the three registered provider names (anthropic/openai/gemini), and export both in __all__: - PROVIDER_FAST_MODELS — cheap/fast judge SKU per provider; every value is an exact key in signalforge.llm.pricing.PRICES (claude-haiku-4-5 / gpt-4o-mini / gemini-2.5-flash) so lookup()/--estimate never raise. - PROVIDER_SKU_PREFIXES — SKU-string prefix per provider. Refactor signalforge.llm.cost._rollup so its SKU-prefix dispatch derives _PROVIDER_PREFIXES by inverting PROVIDER_SKU_PREFIXES instead of duplicating the claude-/gpt-/gemini- literals. Classification behaviour is unchanged; existing rollup tests stay green and a new parametrised test pins the per-SKU classification plus the derived-from-source-of-truth invariant. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/signalforge/llm/cost/_rollup.py | 13 +++-- src/signalforge/llm/providers.py | 41 ++++++++++++++ tests/llm/cost/test_rollup.py | 43 +++++++++++++++ tests/llm/test_providers.py | 84 +++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 4 deletions(-) diff --git a/src/signalforge/llm/cost/_rollup.py b/src/signalforge/llm/cost/_rollup.py index e80f8fd0..8fe6e1de 100644 --- a/src/signalforge/llm/cost/_rollup.py +++ b/src/signalforge/llm/cost/_rollup.py @@ -62,6 +62,7 @@ ) from signalforge.llm.errors import EstimateUnknownModelError from signalforge.llm.pricing import PRICE_TABLE_VERSION, PRICES, lookup +from signalforge.llm.providers import PROVIDER_SKU_PREFIXES # --------------------------------------------------------------------------- # Provider derivation. The pricing table groups entries by provider in @@ -72,10 +73,14 @@ # Prefix -> canonical provider name (matches the names registered in # ``signalforge.llm.providers``). Order doesn't matter — prefixes are # disjoint as of PRICE_TABLE_VERSION 2026-05-28. -_PROVIDER_PREFIXES: tuple[tuple[str, str], ...] = ( - ("claude-", "anthropic"), - ("gpt-", "openai"), - ("gemini-", "gemini"), +# +# Derived from the single-source-of-truth :data:`PROVIDER_SKU_PREFIXES` +# (provider -> prefix) in :mod:`signalforge.llm.providers`, inverted to +# prefix -> provider for the dispatch below (#187 US-001). The constant +# was previously a duplicated literal here; importing keeps the prefix set +# in lockstep with the provider registry. +_PROVIDER_PREFIXES: tuple[tuple[str, str], ...] = tuple( + (prefix, provider) for provider, prefix in PROVIDER_SKU_PREFIXES.items() ) diff --git a/src/signalforge/llm/providers.py b/src/signalforge/llm/providers.py index aa0d231d..046c1381 100644 --- a/src/signalforge/llm/providers.py +++ b/src/signalforge/llm/providers.py @@ -339,6 +339,45 @@ def provider_for(name: str) -> LLMProvider: raise UnknownProviderError(name, available=tuple(_REGISTRY)) from None +# --------------------------------------------------------------------------- +# Provider -> string mappings (#187 US-001). +# +# Two read-only constants keyed by the canonical provider names registered +# below (``anthropic`` / ``openai`` / ``gemini``). They are the single +# source of truth for two cross-cutting facts that previously lived as +# duplicated literals scattered across stages: +# +# * ``PROVIDER_FAST_MODELS`` — the cheap/fast judge SKU per provider, used by +# the faster-grade defaults (#187). Every value MUST be an exact key in +# :data:`signalforge.llm.pricing.PRICES` so ``pricing.lookup(model)`` and the +# ``--estimate`` cost-preview path never raise. +# * ``PROVIDER_SKU_PREFIXES`` — the SKU-string prefix per provider, used by +# the cost-rollup's prefix dispatch (``signalforge.llm.cost._rollup``) to map +# a priced SKU back to its provider. Consumers that need the inverse +# (prefix -> provider) iterate ``.items()`` and invert. +# +# These are plain ``dict`` literals (not ``MappingProxyType``) for the same +# reason the cost-rollup's prefix table is a plain tuple — they are internal +# lookup tables, the values are immutable strings, and no caller mutates them. +# --------------------------------------------------------------------------- + +#: Cheap/fast judge SKU per provider (#187 US-001). Every value is an exact +#: key in :data:`signalforge.llm.pricing.PRICES`. +PROVIDER_FAST_MODELS: dict[str, str] = { + "anthropic": "claude-haiku-4-5", + "openai": "gpt-4o-mini", + "gemini": "gemini-2.5-flash", +} + +#: SKU-string prefix per provider (#187 US-001). The cost-rollup's +#: prefix dispatch consumes this (inverting to prefix -> provider). +PROVIDER_SKU_PREFIXES: dict[str, str] = { + "anthropic": "claude-", + "openai": "gpt-", + "gemini": "gemini-", +} + + class AnthropicProvider(LLMProvider): """Anthropic strategy behind the generic LLM orchestrator (DEC-002/003/004). @@ -1424,6 +1463,8 @@ def estimate_input_tokens( __all__ = ( + "PROVIDER_FAST_MODELS", + "PROVIDER_SKU_PREFIXES", "AnthropicProvider", "ExceptionCategory", "GeminiProvider", diff --git a/tests/llm/cost/test_rollup.py b/tests/llm/cost/test_rollup.py index 5f6c7a10..a8cb3a10 100644 --- a/tests/llm/cost/test_rollup.py +++ b/tests/llm/cost/test_rollup.py @@ -757,3 +757,46 @@ def test_verify_provider_prefix_coverage_raises_when_model_unmapped() -> None: # find the SKU to fix. assert "mistral-large-2" in str(exc_info.value) assert "_PROVIDER_PREFIXES" in str(exc_info.value) + + +# --------------------------------------------------------------------------- +# #187 US-001 — SKU-prefix dispatch now sources PROVIDER_SKU_PREFIXES from +# signalforge.llm.providers instead of a duplicated local literal. The +# classification behaviour must be byte-identical to the pre-refactor +# literal. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + ("claude-sonnet-4-6", "anthropic"), + ("claude-haiku-4-5", "anthropic"), + ("gpt-4o", "openai"), + ("gpt-4o-mini", "openai"), + ("gemini-2.5-flash", "gemini"), + ("gemini-2.5-pro", "gemini"), + ("mistral-large-2", None), # no matching prefix → None + ("", None), + ], +) +def test_provider_for_model_classifies_each_sku(model: str, expected: str | None) -> None: + """``_provider_for_model`` maps a SKU to its provider via the SKU-prefix + dispatch (now sourced from ``PROVIDER_SKU_PREFIXES`` — #187 US-001). The + claude-/gpt-/gemini- classification is byte-identical to the pre-refactor + local literal; an unknown prefix returns ``None``.""" + from signalforge.llm.cost._rollup import _provider_for_model + + assert _provider_for_model(model) == expected + + +def test_rollup_prefix_table_is_derived_from_provider_sku_prefixes() -> None: + """The cost-rollup's ``_PROVIDER_PREFIXES`` is the inversion of the + single-source-of-truth ``PROVIDER_SKU_PREFIXES`` (#187 US-001) — no + duplicated literal. A drift between the two tables breaks this loudly.""" + from signalforge.llm.cost._rollup import _PROVIDER_PREFIXES + from signalforge.llm.providers import PROVIDER_SKU_PREFIXES + + assert dict((prefix, provider) for prefix, provider in _PROVIDER_PREFIXES) == { + prefix: provider for provider, prefix in PROVIDER_SKU_PREFIXES.items() + } diff --git a/tests/llm/test_providers.py b/tests/llm/test_providers.py index 9f1a44ca..8f51a2b9 100644 --- a/tests/llm/test_providers.py +++ b/tests/llm/test_providers.py @@ -19,6 +19,8 @@ from signalforge.llm.errors import UnknownProviderError from signalforge.llm.providers import ( + PROVIDER_FAST_MODELS, + PROVIDER_SKU_PREFIXES, AnthropicProvider, ExceptionCategory, LLMProvider, @@ -1433,3 +1435,85 @@ def test_unclean_finish_reason_message_default_returns_generic_diagnostic() -> N # Mentions the "stop reason" concept generically (the default doesn't # know which vendor field to name — that's the override's job). assert "stop reason" in message + + +# --------------------------------------------------------------------------- +# #187 US-001 — PROVIDER_FAST_MODELS + PROVIDER_SKU_PREFIXES constants +# --------------------------------------------------------------------------- + + +#: The three providers registered at import time in +#: :mod:`signalforge.llm.providers` (anthropic / openai / gemini). +_REGISTERED_PROVIDER_NAMES = frozenset({"anthropic", "openai", "gemini"}) + + +@pytest.mark.unit +@pytest.mark.llm +def test_provider_fast_models_keys_are_the_three_registered_providers() -> None: + """``PROVIDER_FAST_MODELS`` is keyed by exactly the three provider names + registered in the module (#187 US-001). A new provider that ships without + a fast-model entry — or a dropped/renamed key — breaks this loudly.""" + assert set(PROVIDER_FAST_MODELS) == _REGISTERED_PROVIDER_NAMES + # Cross-check against the live registry, not just a hard-coded set, so a + # future registry change forces a fast-models update in lockstep. + for name in PROVIDER_FAST_MODELS: + assert provider_for(name).name == name + + +@pytest.mark.unit +@pytest.mark.llm +def test_provider_fast_models_values_are_all_priced_skus() -> None: + """Every ``PROVIDER_FAST_MODELS`` value MUST be an exact key in + :data:`signalforge.llm.pricing.PRICES` so ``pricing.lookup(model)`` and + the ``--estimate`` cost-preview path never raise on a fast default + (#187 US-001).""" + from signalforge.llm.pricing import PRICES, lookup + + for provider, model in PROVIDER_FAST_MODELS.items(): + assert model in PRICES, f"{provider} fast model {model!r} is not a priced SKU" + # lookup() raising would surface the same gap as a hard failure; pin it. + lookup(model) + + +@pytest.mark.unit +@pytest.mark.llm +def test_provider_sku_prefixes_keys_match_fast_models_keys() -> None: + """``PROVIDER_SKU_PREFIXES`` and ``PROVIDER_FAST_MODELS`` cover the same + provider names — the two tables stay in lockstep (#187 US-001).""" + assert set(PROVIDER_SKU_PREFIXES) == set(PROVIDER_FAST_MODELS) + assert set(PROVIDER_SKU_PREFIXES) == _REGISTERED_PROVIDER_NAMES + + +@pytest.mark.unit +@pytest.mark.llm +def test_each_fast_model_starts_with_its_provider_prefix() -> None: + """Each provider's fast model id begins with that provider's SKU prefix + (#187 US-001) — a guard that the two tables describe the same SKUs.""" + for provider, model in PROVIDER_FAST_MODELS.items(): + prefix = PROVIDER_SKU_PREFIXES[provider] + assert model.startswith(prefix), ( + f"{provider} fast model {model!r} does not start with prefix {prefix!r}" + ) + + +@pytest.mark.unit +@pytest.mark.llm +def test_provider_sku_prefixes_values_match_expected() -> None: + """Pin the exact prefix strings the cost-rollup dispatch depends on + (#187 US-001) — a typo in any prefix would silently route a priced SKU + to ``CostRollupUnknownModelError`` at rollup time.""" + assert PROVIDER_SKU_PREFIXES == { + "anthropic": "claude-", + "openai": "gpt-", + "gemini": "gemini-", + } + + +@pytest.mark.unit +@pytest.mark.llm +def test_both_constants_are_exported() -> None: + """Both constants are part of the module's public surface (#187 US-001).""" + from signalforge.llm import providers as providers_module + + assert "PROVIDER_FAST_MODELS" in providers_module.__all__ + assert "PROVIDER_SKU_PREFIXES" in providers_module.__all__ From 699a5555ec6fd4cab5983d5b4f09d507fcb771ba Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 12:22:28 -0400 Subject: [PATCH 06/15] =?UTF-8?q?SignalForge-dpy.2:=20US-002=20=E2=80=94?= =?UTF-8?q?=20GradeConfig=20per-provider=20fast-model=20resolver=20+=20com?= =?UTF-8?q?pat=20validator=20+=201024=20cap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the grade-judge model default a per-provider fast model resolved at config-load, plus a model<->provider compat validator and a raised max_output_tokens cap. - model: str -> str | None = None sentinel. A @model_validator(mode="before") resolves the sentinel to PROVIDER_FAST_MODELS[provider] (frozen-safe; no mutation of the constructed instance). Unknown provider is NOT injected so the existing provider field-validator still raises UnknownProviderError rather than being masked. After construction model is always concrete. - _model_non_empty field-validator now passes None through cleanly (the only None-survival path is the unknown-provider case that raises downstream). - New @model_validator(mode="after") _validate_model_provider_compat rejects a SKU-prefix/provider mismatch (e.g. provider=openai + claude-* model), reusing PROVIDER_SKU_PREFIXES. Only the three known-prefix providers participate; custom/plugin providers (not in the table) may use any model name. - max_output_tokens default 256 -> 1024 (DEC-004) to avoid one-line gemini-2.5-flash grade-JSON truncation. It is a cap, not a target. - Module + class docstrings and the DEC-023..027 locked-defaults list updated. - tests/grade/test_config.py: updated the defaults regression test to the new resolved defaults (haiku, 1024) and added coverage for per-provider resolution, explicit-honour, compat reject/accept, whitespace guard, unknown-provider-not-masked, and the loader round-trips. - grade/engine.py + cli/_estimate.py: read-only assert narrows for the now str|None field (runtime-guaranteed concrete) to keep pyright green. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/signalforge/cli/_estimate.py | 8 ++ src/signalforge/grade/config.py | 141 ++++++++++++++++++++++++++++--- src/signalforge/grade/engine.py | 7 ++ tests/grade/test_config.py | 128 +++++++++++++++++++++++++++- 4 files changed, 270 insertions(+), 14 deletions(-) diff --git a/src/signalforge/cli/_estimate.py b/src/signalforge/cli/_estimate.py index 8dd6f693..ec1ca36b 100644 --- a/src/signalforge/cli/_estimate.py +++ b/src/signalforge/cli/_estimate.py @@ -360,6 +360,10 @@ def _count_grade_criterion_tokens( per criterion. See CHANGELOG. """ dynamic_block = render_grade_dynamic_block(artifact_id, artifact_text, criterion) + # ``grade_config.model`` is invariantly concrete post-construction (#187 + # US-002: the sentinel ``None`` resolves to the provider's fast model at + # config-load). Narrow the static ``str | None``. + assert grade_config.model is not None return provider_for(grade_config.provider).estimate_input_tokens( grade_config.model, dynamic_block, system=system_and_rubric, client=client ) @@ -488,6 +492,10 @@ def estimate( ) * pricing_draft.output_per_mtok # ---- Grader cost projection ----------------------------------- + # ``grade_config.model`` is invariantly concrete post-construction (#187 + # US-002: the sentinel ``None`` resolves to the provider's fast model at + # config-load). Narrow the static ``str | None`` once for this block. + assert grade_config.model is not None pricing_grade = _pricing.lookup(grade_config.model) rubric = grade_config.rubric or DEFAULT_RUBRIC column_count = len(model.columns_list) diff --git a/src/signalforge/grade/config.py b/src/signalforge/grade/config.py index 797b62ae..03aac9f3 100644 --- a/src/signalforge/grade/config.py +++ b/src/signalforge/grade/config.py @@ -24,11 +24,19 @@ here. The loader takes it as a required argument so the caller is explicit about the resolution base. * **DEC-023..DEC-027** — Locked default values: - ``model="claude-sonnet-4-6"``, ``cache_ttl="1h"``, - ``max_output_tokens=256``, ``max_retries_429=3``, ``max_retries_5xx=1``, + ``model=None`` (resolves to the calling provider's fast model at + config-load — ``anthropic`` -> ``claude-haiku-4-5`` per + :data:`signalforge.llm.providers.PROVIDER_FAST_MODELS`; #187 US-002 / + DEC-004), ``cache_ttl="1h"``, ``max_output_tokens=1024`` (#187 DEC-004 + — raised from 256 so a one-line ``gemini-2.5-flash`` grade JSON is not + truncated), ``max_retries_429=3``, ``max_retries_5xx=1``, ``max_retries_conn=1``, ``total_budget_seconds=300``, ``min_pass_rate=0.7``, ``min_mean_score=0.5``, ``rubric=None``, ``fail_on_below_threshold=False``. +* **#187 US-002 / DEC-006** — when ``model`` is set explicitly, a + SKU-prefix/provider mismatch (e.g. ``provider="openai"`` with + ``model="claude-sonnet-4-6"``) fails loud at config-load. The + prefix table is :data:`signalforge.llm.providers.PROVIDER_SKU_PREFIXES`. Resolution order (mirrors :func:`signalforge.draft.config.load_draft_config`): @@ -65,13 +73,14 @@ import math from pathlib import Path -from typing import Literal +from typing import Any, Literal import yaml from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator from signalforge.grade.errors import GradeConfigError, GradeRubricError from signalforge.grade.rubric import Rubric, validate_rubric +from signalforge.llm.providers import PROVIDER_FAST_MODELS, PROVIDER_SKU_PREFIXES _DEFAULT_CONFIG_FILENAME = "signalforge.yml" @@ -94,10 +103,22 @@ class GradeConfig(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) - model: str = "claude-sonnet-4-6" - """LLM-judge model id (DEC-026). Mirrors the drafter's default. - Haiku 4.5 is documented as a v0.2 ``cheap_model`` option for - cost-conscious mode but is not exposed in v0.1.""" + model: str | None = None + """LLM-judge model id (DEC-026; #187 US-002 / DEC-004). + + The sentinel default ``None`` means "use the calling provider's fast + model" — resolved at config-load by the + :meth:`_resolve_model_default` before-validator to + :data:`signalforge.llm.providers.PROVIDER_FAST_MODELS` keyed on + :attr:`provider` (``anthropic`` -> ``claude-haiku-4-5``, ``openai`` + -> ``gpt-4o-mini``, ``gemini`` -> ``gemini-2.5-flash``). An explicit + ``model:`` is always honoured verbatim. After construction this + field is always a concrete non-empty string — never ``None``. + + When set explicitly, a SKU-prefix/provider mismatch (e.g. + ``provider="openai"`` with a ``claude-`` model) fails loud at + config-load via :meth:`_validate_model_provider_compat` (#187 + DEC-006), reusing :data:`signalforge.llm.providers.PROVIDER_SKU_PREFIXES`.""" cache_ttl: Literal["5m", "1h"] = "1h" """Anthropic prompt-cache TTL (DEC-024). Defaults to ``"1h"`` (vs. @@ -106,10 +127,16 @@ class GradeConfig(BaseModel): gives margin at no extra cost (cache writes are one-shot regardless of TTL).""" - max_output_tokens: int = 256 - """Per-criterion judge response cap (DEC-025). The expected JSON - response is ~150 tokens; 256 gives 2× safety. Independent of - :attr:`signalforge.draft.DraftConfig.max_output_tokens`.""" + max_output_tokens: int = 1024 + """Per-criterion judge response cap (DEC-025; #187 DEC-004). + + Raised from 256 to 1024 now that ``gemini-2.5-flash`` is a one-line + default judge model — a verbose-but-valid one-line grade JSON from a + cheaper/faster model can exceed 256 tokens, and a truncated response + surfaces as the wrong typed degrade. This is a **cap**, not a target; + the expected JSON is still ~150 tokens, so the larger ceiling costs + nothing on the happy path while removing the truncation risk. + Independent of :attr:`signalforge.draft.DraftConfig.max_output_tokens`.""" max_retries_429: int = 3 """Mirrors :attr:`signalforge.draft.DraftConfig.max_retries_429`. @@ -212,9 +239,46 @@ class GradeConfig(BaseModel): ``signalforge generate`` invocation in CI can gate on threshold compliance — see ``docs/cli-ops.md`` for the exit-code tier.""" + @model_validator(mode="before") + @classmethod + def _resolve_model_default(cls, data: Any) -> Any: + """Resolve the sentinel ``model=None`` to the provider's fast model. + + Runs BEFORE field validation (and before the frozen instance + exists) so the injected value flows through the normal + construction path — :class:`GradeConfig` is ``frozen=True`` and a + ``mode="after"`` mutation would raise. Only a dict input is + rewritten; an already-constructed instance (e.g. from + ``model_validate`` of a :class:`GradeConfig`) passes through + untouched. + + When ``model`` is absent or ``None``, inject + :data:`signalforge.llm.providers.PROVIDER_FAST_MODELS` keyed on + the requested ``provider`` (defaulting to ``"anthropic"`` to + match the field default). A provider NOT in the fast-model table + is left alone — no injection — so the existing ``provider`` + field-validator raises :class:`UnknownProviderError` rather than + this masking it with a ``KeyError`` (#187 US-002 / DEC-004). + """ + if not isinstance(data, dict): + return data + if data.get("model") is None: + provider = data.get("provider", "anthropic") + resolved = PROVIDER_FAST_MODELS.get(provider) + if resolved is not None: + # Copy-on-write so we don't mutate a caller-owned dict. + data = {**data, "model": resolved} + return data + @field_validator("model") @classmethod - def _model_non_empty(cls, v: str) -> str: + def _model_non_empty(cls, v: str | None) -> str | None: + # ``None`` only survives to here when the provider was unknown and + # the before-validator deliberately declined to inject a default + # (so the provider field-validator can raise the typed error). + # Pass it through cleanly rather than tripping the non-empty guard. + if v is None: + return v if not v or not v.strip(): raise ValueError("must be a non-empty, non-whitespace string") return v @@ -284,6 +348,59 @@ def _bounded_unit(cls, v: float) -> float: raise ValueError("must be in the closed interval [0.0, 1.0]") return v + @model_validator(mode="after") + def _validate_model_provider_compat(self) -> GradeConfig: + """Reject a SKU-prefix/provider mismatch (#187 US-002 / DEC-006). + + After field validation ``model`` is always a concrete string (the + before-validator resolved the sentinel, OR the operator set it + explicitly, OR the provider was unknown and the provider + field-validator already raised before reaching here). When + :attr:`provider` is one of the *known-prefix* providers in + :data:`signalforge.llm.providers.PROVIDER_SKU_PREFIXES` AND the + model carries a *different* known provider's SKU prefix, fail + loud: e.g. ``provider="openai"`` with ``model="claude-sonnet-4-6"`` + is an operator mistake that would otherwise send a ``claude-`` SKU + through the OpenAI strategy. + + The prefix table is + :data:`signalforge.llm.providers.PROVIDER_SKU_PREFIXES` (single + source of truth — no hardcoded prefixes here). Two cases are + deliberately left alone: + + * A model whose prefix matches no known provider (forward-compat: + a future SKU the table doesn't yet enumerate must not be + rejected as a mismatch). + * A registry-valid provider that is NOT in the prefix table + (a custom/plugin provider). Such a provider may use any model + name — the cross-vendor mismatch concept only applies among the + three known-prefix vendors, so the check does not fire when + :attr:`provider` is outside the table. + + This is a read-only check — no mutation — so it is safe on the + frozen instance. + """ + # ``model`` is concrete by this point on every reachable path. + model = self.model + if model is None: # pragma: no cover - defensive; resolution + provider guard cover it + return self + # Only the known-prefix providers participate in the mismatch check. + if self.provider not in PROVIDER_SKU_PREFIXES: + return self + if model.startswith(PROVIDER_SKU_PREFIXES[self.provider]): + return self + # If the model carries ANOTHER known provider's prefix, that's a mismatch. + for other_provider, prefix in PROVIDER_SKU_PREFIXES.items(): + if other_provider == self.provider: + continue + if model.startswith(prefix): + raise ValueError( + f"model {model!r} has the {other_provider!r} SKU prefix " + f"{prefix!r} but provider is {self.provider!r}; set a " + f"{self.provider!r}-compatible model or change the provider" + ) + return self + @model_validator(mode="after") def _validate_rubric_structure(self) -> GradeConfig: # Per-criterion shape is enforced by ``Criterion`` (extra=forbid, diff --git a/src/signalforge/grade/engine.py b/src/signalforge/grade/engine.py index 4e6d6030..da6a0139 100644 --- a/src/signalforge/grade/engine.py +++ b/src/signalforge/grade/engine.py @@ -356,6 +356,11 @@ async def _grade_one_async( # 2. Issue the LLM call. Wrap LLMError -> GradeLLMError once at # the seam (DEC-015 of #5 mirror: one-level adapter). + # ``config.model`` is invariantly a concrete string post-construction + # (#187 US-002: the sentinel ``None`` is resolved to the provider's fast + # model at config-load; an unknown provider raises before any consumer + # reads it). Narrow the static ``str | None`` once for this function. + assert config.model is not None try: result = await call_llm_async( system=_SYSTEM_PROMPT, @@ -459,6 +464,8 @@ def _build_degraded( receipt) carry the same ``score=None`` / ``passed=False`` shape so a downstream replay round-trips cleanly. """ + # ``config.model`` is invariantly concrete post-construction (#187 US-002). + assert config.model is not None grading_result = GradingResult( artifact_id=artifact_id, criterion_id=criterion.id, diff --git a/tests/grade/test_config.py b/tests/grade/test_config.py index 10fe8ab9..de81dbc3 100644 --- a/tests/grade/test_config.py +++ b/tests/grade/test_config.py @@ -187,9 +187,13 @@ def test_grade_config_defaults_match_dec_023_to_027() -> None: """Regression guard: every locked default must match the plan. A drift here is a behaviour change masquerading as a refactor.""" cfg = GradeConfig() - assert cfg.model == "claude-sonnet-4-6" + # #187 US-002 / DEC-004: ``model`` now defaults to the sentinel that + # resolves to the calling provider's fast model. With the default + # provider (``anthropic``) that is ``claude-haiku-4-5``. + assert cfg.model == "claude-haiku-4-5" assert cfg.cache_ttl == "1h" - assert cfg.max_output_tokens == 256 + # #187 DEC-004: raised from 256 to avoid one-line gemini-flash truncation. + assert cfg.max_output_tokens == 1024 assert cfg.max_retries_429 == 3 assert cfg.max_retries_5xx == 1 assert cfg.max_retries_conn == 1 @@ -269,6 +273,126 @@ def test_load_grade_config_unknown_provider_fails_loud(tmp_path: Path) -> None: assert "anthropic" in str(excinfo.value) +# ----- Per-provider fast-model resolution (#187 US-002 / DEC-004) ----- + + +def test_grade_config_model_resolves_anthropic_fast_default() -> None: + """The sentinel ``model=None`` (default) resolves to the anthropic + fast model via :data:`PROVIDER_FAST_MODELS`.""" + assert GradeConfig().model == "claude-haiku-4-5" + + +def test_grade_config_model_resolves_openai_fast_default() -> None: + """With ``provider="openai"`` and no explicit model, resolution + yields the openai fast model.""" + assert GradeConfig(provider="openai").model == "gpt-4o-mini" + + +def test_grade_config_model_resolves_gemini_fast_default() -> None: + """With ``provider="gemini"`` and no explicit model, resolution + yields the gemini fast model.""" + assert GradeConfig(provider="gemini").model == "gemini-2.5-flash" + + +def test_grade_config_explicit_model_is_honoured_over_default() -> None: + """An explicit ``model:`` always wins over the per-provider default.""" + assert GradeConfig(model="claude-sonnet-4-6").model == "claude-sonnet-4-6" + + +def test_grade_config_resolved_model_is_never_none() -> None: + """After construction on the happy path, ``model`` is a concrete + string — the sentinel never leaks out.""" + cfg = GradeConfig() + assert isinstance(cfg.model, str) + assert cfg.model.strip() != "" + + +def test_grade_config_unknown_provider_not_masked_by_resolution() -> None: + """An unknown provider must still raise the typed provider error — + the model-resolution before-validator declines to inject (the + provider isn't in the fast-model table) so the provider + field-validator surfaces :class:`UnknownProviderError` rather than a + masked ``KeyError`` (#187 US-002 / DEC-004).""" + from signalforge.llm.errors import UnknownProviderError + + with pytest.raises(UnknownProviderError) as excinfo: + GradeConfig(provider="bogus") + assert excinfo.value.name == "bogus" + + +# ----- Model<->provider compatibility validator (#187 US-002 / DEC-006) ----- + + +def test_grade_config_provider_model_mismatch_rejected() -> None: + """A ``claude-`` model under ``provider="openai"`` is an operator + mistake — reject at config-load via the SKU-prefix compat check.""" + from pydantic import ValidationError + + with pytest.raises(ValidationError): + GradeConfig(provider="openai", model="claude-sonnet-4-6") + + +def test_grade_config_provider_model_match_accepted() -> None: + """A ``gpt-`` model under ``provider="openai"`` passes the compat + check (the prefix matches the provider).""" + cfg = GradeConfig(provider="openai", model="gpt-4o") + assert cfg.provider == "openai" + assert cfg.model == "gpt-4o" + + +def test_grade_config_whitespace_model_still_rejected() -> None: + """A whitespace-only explicit model must still trip the non-empty + guard — the sentinel resolution does not relax that defence.""" + from pydantic import ValidationError + + with pytest.raises(ValidationError): + GradeConfig(model=" ") + + +def test_grade_config_max_output_tokens_default_is_1024() -> None: + """#187 DEC-004: the per-criterion cap default is raised to 1024.""" + assert GradeConfig().max_output_tokens == 1024 + + +# ----- load_grade_config fast-model resolution + compat (#187 US-002) ----- + + +def test_load_grade_config_block_without_model_resolves_fast_model( + tmp_path: Path, +) -> None: + """A ``grade:`` block that omits ``model:`` resolves the provider's + fast model at load time.""" + (tmp_path / "signalforge.yml").write_text( + "grade:\n provider: openai\n", + encoding="utf-8", + ) + cfg = load_grade_config(tmp_path) + assert cfg.provider == "openai" + assert cfg.model == "gpt-4o-mini" + + +def test_load_grade_config_block_with_model_honours_it(tmp_path: Path) -> None: + """An explicit ``model:`` in the ``grade:`` block is honoured.""" + (tmp_path / "signalforge.yml").write_text( + "grade:\n model: claude-sonnet-4-6\n", + encoding="utf-8", + ) + cfg = load_grade_config(tmp_path) + assert cfg.model == "claude-sonnet-4-6" + + +def test_load_grade_config_provider_model_mismatch_raises(tmp_path: Path) -> None: + """A mismatched provider/model in ``signalforge.yml`` surfaces as + :class:`GradeConfigError` at the loader boundary (the underlying + ``ValidationError`` is wrapped).""" + (tmp_path / "signalforge.yml").write_text( + "grade:\n provider: openai\n model: claude-sonnet-4-6\n", + encoding="utf-8", + ) + with pytest.raises(GradeConfigError): + load_grade_config(tmp_path) + + # ----- Numeric validators ----- From 0668a3b08bd8d4d2e8140e38373ecbf1ed9bed5b Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 12:28:01 -0400 Subject: [PATCH 07/15] =?UTF-8?q?SignalForge-dpy.3:=20US-003=20=E2=80=94?= =?UTF-8?q?=20re-baseline=20estimate=20goldens=20+=20grade=20fixtures=20fo?= =?UTF-8?q?r=20Haiku=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #187 US-002 grade default flip (anthropic resolves to claude-haiku-4-5, max_output_tokens 256 -> 1024) drifted two estimate fixtures pinned to the old claude-sonnet-4-6 grade default. Re-baseline in lockstep: - tests/fixtures/estimate/anthropic_byte_identity_golden.txt: regenerated. Only diff is the grade-model id (sonnet -> haiku) and the corresponding grade USD figures (per-criterion $0.0292 -> $0.0078, grade $0.1170 -> $0.0312, total $0.1814 -> $0.0956). Verified the draft section, token counts, call counts, artifact counts, and warehouse section are unchanged. - tests/cli/test_estimate_engine.py::test_estimate_total_llm_usd_matches_hand_calculation: the hand calc now keys the grade half on claude-haiku-4-5 pricing ($0.80/MTok input, $4/MTok output) instead of reusing the draft model's pricing. The grade output-token figure stays the fixed _GRADE_OUTPUT_TOKENS_PER_CALL (50) — the 256->1024 max_output_tokens bump is a response cap, not the estimate's per-call output projection. - tests/fixtures/grade/grade_event_v1.jsonl: cosmetic model id sonnet -> haiku for consistency with the new default (drift detector validates shape only; still green). No production source changed. Full suite green: 3329 passed, 8 skipped. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/cli/test_estimate_engine.py | 57 ++++++++++++------- .../anthropic_byte_identity_golden.txt | 14 ++--- tests/fixtures/grade/grade_event_v1.jsonl | 2 +- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/tests/cli/test_estimate_engine.py b/tests/cli/test_estimate_engine.py index 18d7f258..cda21c54 100644 --- a/tests/cli/test_estimate_engine.py +++ b/tests/cli/test_estimate_engine.py @@ -239,27 +239,37 @@ def test_estimate_total_llm_usd_matches_hand_calculation( ) -> None: """Pin USD to four decimals against a hand-computed expected. - Hand calculation with the default ``DraftConfig``/``GradeConfig`` - (``claude-sonnet-4-6`` for both, ``$3/MTok`` input, ``$15/MTok`` - output) and the default rubric (4 criteria): - - Draft input: 1_000_000 tokens (1 MTok) → $3.00. - Draft output: 4096 tokens (default ``max_output_tokens``) - → 4096 / 1e6 * 15 ≈ $0.06144. - Draft USD ≈ 3.06144. - - Grade per criterion (4 criteria): + Hand calculation with the default ``DraftConfig`` / ``GradeConfig`` + and the default rubric (4 criteria). Post #187 US-002 the two stages + use DIFFERENT default models — the drafter stays on + ``claude-sonnet-4-6`` (``$3/MTok`` input, ``$15/MTok`` output) while + the grader resolves to the provider's fast model + ``claude-haiku-4-5`` (``$0.80/MTok`` input, ``$4/MTok`` output). The + draft and grade halves therefore key on separate price rows: + + Draft (sonnet pricing): + Draft input: 1_000_000 tokens (1 MTok) → 1 * 3.00 = $3.00. + Draft output: 4096 tokens (default ``DraftConfig.max_output_tokens``) + → 4096 / 1e6 * 15 ≈ $0.06144. + Draft USD ≈ 3.06144. + + Grade (haiku pricing — 4 criteria): artifact_count for our 2-column model: 2*2 (column desc+rationale) + 2 (model desc+rationale) + int(3.5*2) (test rationales) = 4 + 2 + 7 = 13. Input tokens per call (queued) = 500 → 500 * 13 = 6500. - Per-criterion input USD: 6500/1e6 * 3 = 0.0195. - Per-criterion output USD: 50 * 13 / 1e6 * 15 = 650/1e6*15 - = 0.00975. - Per-criterion total: 0.0195 + 0.00975 = 0.02925. - Across 4 criteria: 4 * 0.02925 = 0.117. - - Grand total: 3.06144 + 0.117 = 3.17844. + Per-criterion input USD: 6500/1e6 * 0.80 = 0.0052. + Per-criterion output USD: 50 * 13 / 1e6 * 4 = 650/1e6*4 + = 0.0026. + (The grade output-token figure is the fixed + ``_GRADE_OUTPUT_TOKENS_PER_CALL`` of 50, NOT + ``GradeConfig.max_output_tokens`` — the 256→1024 default bump in + #187 US-002 is a response cap, not the estimate's per-call + output projection, so it does not enter this math.) + Per-criterion total: 0.0052 + 0.0026 = 0.0078. + Across 4 criteria: 4 * 0.0078 = 0.0312. + + Grand total: 3.06144 + 0.0312 = 3.09264. Test pins to 4 decimals. """ @@ -277,13 +287,16 @@ def test_estimate_total_llm_usd_matches_hand_calculation( fake_anthropic, ) - pricing = pricing_lookup(draft_config.model) - expected_draft = (1_000_000 / 1_000_000.0) * pricing.input_per_mtok + ( + # The drafter and grader key on separate price rows post #187 US-002. + draft_pricing = pricing_lookup(draft_config.model) + assert grade_config.model is not None # resolved to the fast model at config-load + grade_pricing = pricing_lookup(grade_config.model) + expected_draft = (1_000_000 / 1_000_000.0) * draft_pricing.input_per_mtok + ( 4096 / 1_000_000.0 - ) * pricing.output_per_mtok + ) * draft_pricing.output_per_mtok artifact_count = 2 * 2 + 2 + int(3.5 * 2) - per_crit_in = (500 * artifact_count) / 1_000_000.0 * pricing.input_per_mtok - per_crit_out = (50 * artifact_count) / 1_000_000.0 * pricing.output_per_mtok + per_crit_in = (500 * artifact_count) / 1_000_000.0 * grade_pricing.input_per_mtok + per_crit_out = (50 * artifact_count) / 1_000_000.0 * grade_pricing.output_per_mtok expected_grade = n_criteria * (per_crit_in + per_crit_out) expected_total = expected_draft + expected_grade diff --git a/tests/fixtures/estimate/anthropic_byte_identity_golden.txt b/tests/fixtures/estimate/anthropic_byte_identity_golden.txt index e3fb969b..7ee3fedf 100644 --- a/tests/fixtures/estimate/anthropic_byte_identity_golden.txt +++ b/tests/fixtures/estimate/anthropic_byte_identity_golden.txt @@ -1,6 +1,6 @@ Estimate for model.shop.customers drafter: claude-sonnet-4-6 - grader: claude-sonnet-4-6 + grader: claude-haiku-4-5 Estimated draft cost: input tokens: 1,000 @@ -10,11 +10,11 @@ Estimated draft cost: Estimated grade cost: artifacts: 13 criteria: 4 calls: 52 per criterion: - clarity 13 calls 6,500 tokens $0.0292 - consistency 13 calls 6,500 tokens $0.0292 - rationale 13 calls 6,500 tokens $0.0292 - no-redundant 13 calls 6,500 tokens $0.0292 - cost: $0.1170 + clarity 13 calls 6,500 tokens $0.0078 + consistency 13 calls 6,500 tokens $0.0078 + rationale 13 calls 6,500 tokens $0.0078 + no-redundant 13 calls 6,500 tokens $0.0078 + cost: $0.0312 Estimated warehouse cost: bytes-per-row: ~1 (BigQuery dryRun) @@ -22,7 +22,7 @@ Estimated warehouse cost: sample size: 100,000 rows total bytes: ~68.4 KB -Total estimated LLM cost: $0.1814 +Total estimated LLM cost: $0.0956 Total estimated warehouse: ~68.4 KB Price table: 2026-05-28 | Heuristic: ~3.5 tests/column (canonical fixture average) diff --git a/tests/fixtures/grade/grade_event_v1.jsonl b/tests/fixtures/grade/grade_event_v1.jsonl index f27999d7..cf66ae88 100644 --- a/tests/fixtures/grade/grade_event_v1.jsonl +++ b/tests/fixtures/grade/grade_event_v1.jsonl @@ -1 +1 @@ -{"audit_schema_version":1,"signalforge_version":"0.1.0.dev0","run_id":"a1b2c3d4e5f6478890aabbccddeeff00","timestamp":"2026-05-01T17:42:13.123456Z","model_unique_id":"model.shop.dim_customers","artifact_id":"column.email.description","criterion_id":"clarity","score":0.8,"passed":true,"evidence":"The description states 'Email address of the customer at the time of order'.","reasoning":"The description is clear and specific about which email is captured. It would be improved by noting whether the value is normalised, but the meaning is unambiguous as written.","rubric_hash":"0123456789abcdef","prompt_version_template":"fedcba9876543210","criterion_prompt_hash":"1111222233334444","response_text_hash":"5555666677778888","model":"claude-sonnet-4-6","input_tokens":1820,"output_tokens":140,"cache_creation_input_tokens":0,"cache_read_input_tokens":1500} +{"audit_schema_version":1,"signalforge_version":"0.1.0.dev0","run_id":"a1b2c3d4e5f6478890aabbccddeeff00","timestamp":"2026-05-01T17:42:13.123456Z","model_unique_id":"model.shop.dim_customers","artifact_id":"column.email.description","criterion_id":"clarity","score":0.8,"passed":true,"evidence":"The description states 'Email address of the customer at the time of order'.","reasoning":"The description is clear and specific about which email is captured. It would be improved by noting whether the value is normalised, but the meaning is unambiguous as written.","rubric_hash":"0123456789abcdef","prompt_version_template":"fedcba9876543210","criterion_prompt_hash":"1111222233334444","response_text_hash":"5555666677778888","model":"claude-haiku-4-5","input_tokens":1820,"output_tokens":140,"cache_creation_input_tokens":0,"cache_read_input_tokens":1500} From 4a8a297e616945c2835f89b33babb50c99b94808 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 12:36:34 -0400 Subject: [PATCH 08/15] =?UTF-8?q?SignalForge-dpy.5:=20US-005=20=E2=80=94?= =?UTF-8?q?=20Haiku=20calibration=20harness=20+=20writeup=20(gated)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build the maintainer-run empirical gate for the #187 Haiku grade default. - tests/research/187-haiku-calibration/_substrate.py: pinned Model + CandidateSchema (engineered-deterministic, spans the rubric calibration space) + curated Sonnet-baseline loader. Artifact-id set derived from the grade engine itself so it can't drift from the formatter. - sonnet_baseline_sample.json: curated 44-verdict Sonnet baseline (11 artifacts x 4 DEFAULT_RUBRIC criteria), NOT the un-committed #179 Phase-B dump; provenance documented. - test_haiku_calibration.py (@pytest.mark.anthropic + runtime skip on ANTHROPIC_API_KEY): re-grades the sample with the resolved Haiku default (claude-haiku-4-5, max_output_tokens=1024), joins to the baseline by (artifact_id, criterion_id), asserts >=85% per-criterion concordance, prints the breakdown for the writeup. - test_gemini_1024_no_truncation.py (@pytest.mark.gemini + runtime skip on SF_RUN_GEMINI/GOOGLE_API_KEY): grades a verbose artifact on gemini-2.5-flash @ 1024 tokens, asserts no score=None truncation degrade (verifies DEC-004). - docs/research/187-haiku-calibration.md: writeup mirroring #179's structure with a clearly-marked "Result (maintainer-filled)" TODO. Gating is belt-and-suspenders (existing anthropic/gemini markers excluded in addopts + runtime skip). No production default changed; src/ untouched; no new marker registered. Default `uv run pytest` stays fully green (3329 passed, gated tests deselected) at 97% coverage. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/research/187-haiku-calibration.md | 219 ++++++++++++++++++ .../187-haiku-calibration/_substrate.py | 197 ++++++++++++++++ .../sonnet_baseline_sample.json | 61 +++++ .../test_gemini_1024_no_truncation.py | 197 ++++++++++++++++ .../test_haiku_calibration.py | 182 +++++++++++++++ 5 files changed, 856 insertions(+) create mode 100644 docs/research/187-haiku-calibration.md create mode 100644 tests/research/187-haiku-calibration/_substrate.py create mode 100644 tests/research/187-haiku-calibration/sonnet_baseline_sample.json create mode 100644 tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py create mode 100644 tests/research/187-haiku-calibration/test_haiku_calibration.py diff --git a/docs/research/187-haiku-calibration.md b/docs/research/187-haiku-calibration.md new file mode 100644 index 00000000..80285950 --- /dev/null +++ b/docs/research/187-haiku-calibration.md @@ -0,0 +1,219 @@ +# Issue #187 — Haiku grade-default calibration gate + +**Status:** harness BUILT (2026-06-02); maintainer RUN pending. The #187 +plan ships `claude-haiku-4-5` as the new grade-default SKU (US-001..US-003 — +the provider fast-model resolver now resolves `model=None` to +`claude-haiku-4-5` on the Anthropic provider, and `max_output_tokens` +defaults to `1024`). The default ships behind this empirical gate: a +maintainer runs the gated harness below with a live `ANTHROPIC_API_KEY` +and transcribes the result into the **"Result (maintainer-filled)"** +section. Until then this writeup records the substrate and method only. + +**Companion artefacts:** + +- `tests/research/187-haiku-calibration/_substrate.py` — pinned `Model` + + `CandidateSchema` + Sonnet-baseline loader. +- `tests/research/187-haiku-calibration/sonnet_baseline_sample.json` — + the curated baseline verdict sample. +- `tests/research/187-haiku-calibration/test_haiku_calibration.py` — the + `@pytest.mark.anthropic`-gated concordance gate. +- `tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py` — + the `@pytest.mark.gemini`-gated 1024-token no-truncation check (DEC-004). + +## tl;dr + +- **The question:** does `claude-haiku-4-5` grade rubric artifacts + concordantly with the prior `claude-sonnet-4-6` baseline? The decision + rule is **≥ 85% per-criterion pass/fail agreement** over a pinned + sample. +- **The harness:** re-grades a hand-authored, calibration-spanning + candidate (strong / adequate / vague artifacts) with the resolved + Haiku default over the locked four-criterion `DEFAULT_RUBRIC`, joins + each `GradingResult` to a curated Sonnet baseline by + `(artifact_id, criterion_id)`, and asserts the agreement rate clears + 85%. Degraded (`score=None`) pairs are excluded from the denominator + and reported separately. +- **A second gated check** verifies DEC-004's claim that the new + `max_output_tokens=1024` default leaves Gemini enough headroom: it + grades a deliberately verbose artifact on `gemini-2.5-flash` @ 1024 + tokens and asserts no `GradingResult` degraded to `score=None` from a + truncation. +- **Default CI is untouched:** both checks are deselected by the + existing `anthropic` / `gemini` markers in `pyproject.toml`'s + `addopts`, and skip-with-reason at runtime if collected without keys. + No live API call happens during normal validation. + +## Substrate + +### Pinned candidate (the artifacts under grade) + +`_substrate.build_candidate()` returns a `CandidateSchema` for a fictional +`dim_customers` model, hand-authored to span the rubric's calibration +space (engineered determinism per `.claude/rules/testing-signal.md` +§ "Engineered determinism over snapshot normalisation"): + +| Artifact | Shape | Intended baseline signal | +|---|---|---| +| `customer_id` description | Strong, specific, sourced | passes every criterion | +| `customer_id` rationale | Strong, names downstream consumers | passes every criterion | +| `email` description | Adequate, concrete | passes clarity / consistency | +| `email` rationale | Thin ("Contact channel.") | fails clarity / rationale | +| `status` description | Deliberately vague ("A status field…") | fails clarity / rationale | +| `status` rationale | Restates the description | fails clarity / rationale / no-redundant | +| `model` description / rationale | Strong, conformed-dimension framing | passes every criterion | +| `customer_id` `not_null` / `unique` tests | Well-justified | passes every criterion | +| `status` `accepted_values` test | Well-justified closed set | passes every criterion | + +The engine's `_stable_artifact_pairs(candidate)` derives **11 artifacts** +from this shape (3 column descriptions + 3 column rationales + model +description + model rationale + 3 test rationales). The harness derives +the `artifact_id` set from the engine itself (via +`_substrate.expected_artifact_ids`) rather than hand-listing it, so the +sample can never silently drift from the formatter +(`.claude/rules/grade-layer.md` § "`_artifact_id_for` … hoist"). + +Over the locked four-criterion `DEFAULT_RUBRIC` (`clarity`, +`consistency`, `rationale`, `no-redundant`) this is **11 × 4 = 44 judge +calls** per run — a reasonable maintainer-gate budget on Haiku +(materially cheaper than the Sonnet baseline; cf. the ~$0.005/call Sonnet +figure in `docs/research/179-test-primitive-expansion-retest.md`). + +### Curated Sonnet baseline + +`sonnet_baseline_sample.json` is a **curated sample, NOT the raw #179 +Phase-B `grade.jsonl` dump**. That dump is not committed anywhere in this +repo (`find . -name grade.jsonl` finds only the drift-detector fixture at +`tests/fixtures/grade/grade_event_v1.jsonl`), and the #179 retest was run +against a private `intuit_airflow` fixture with transient `/tmp/phaseB/` +sidecars (see `docs/research/179-test-primitive-expansion-retest.md` +§ "Reproducing this retest"). Rather than depend on an un-committed dump, +the baseline here is a small representative sample of **44 hand-assigned +plausible `claude-sonnet-4-6` pass/fail verdicts** — one per +`(artifact_id, criterion_id)` pair — whose distribution tracks the +engineered candidate shape above (strong artifacts pass; vague / thin / +redundant artifacts fail on the relevant criteria). + +This makes the comparison reproducible with the **only live variable +being the Haiku re-grade**: the candidate is pinned bytes, the rubric is +locked, the baseline is committed. A concordant Haiku run reproduces the +same verdict distribution; a discordant one surfaces the specific +`(artifact, criterion)` pairs where Haiku and the baseline disagree. + +### Config under test + +`GradeConfig()` with all defaults — after US-002 this resolves to: + +- `model` → `claude-haiku-4-5` (provider fast-model resolver, + `provider="anthropic"`), +- `max_output_tokens` → `1024`, +- `provider` → `anthropic`. + +The harness asserts both resolved values before grading, so a regression +in the resolver fails the gate loud rather than silently measuring the +wrong SKU. + +## Method — the ≥ 85% concordance rule + +1. Build the resolved Haiku-default `GradeConfig()`; assert + `model == "claude-haiku-4-5"` and `max_output_tokens == 1024`. +2. Assert the committed baseline covers every `artifact_id` the engine + will grade (no silent gaps). +3. Run `grade_artifacts(model, candidate, prune_result, config=...)` — + 44 live Haiku judge calls. +4. For each returned `GradingResult`, join to the baseline by + `(artifact_id, criterion_id)`: + - `score is None` (degraded, DEC-015 of #7) → counted as **degraded**, + excluded from the agreement denominator (neither concordant nor + discordant — the pair could not be positively evaluated). + - otherwise → **comparable**; `agreement` iff + `result.passed == baseline_passed`. +5. `agreement_rate = agreements / comparable`. Assert + `agreement_rate >= 0.85`. The harness prints the full breakdown + (model, comparable count, agreements, degraded count, rate, and each + discordance) regardless of pass/fail so a sub-threshold run still + surfaces the disagreements for the writeup. + +**Decision:** if the rate clears 85%, the Haiku default ships as planned. +If it falls short, the printed discordances name the specific +`(artifact, criterion)` shapes where Haiku diverges — those become the +follow-on (prompt-engineering, rubric-tuning, or reconsidering the +default), not silent acceptance. (Same disposition as the #179 epic's +"name the shapes that fell through" acceptance criterion.) + +### Running the gate (maintainer) + +```bash +# From the repo root, with a live key: +ANTHROPIC_API_KEY=sk-... \ + uv run pytest -m anthropic --no-cov -s \ + tests/research/187-haiku-calibration/test_haiku_calibration.py +``` + +`--no-cov` is required because the gated path exercises only a fraction +of the codebase and would trip the 80% coverage floor in `addopts` +(mirrors the `pytest -m bigquery --no-cov` precedent in +`.claude/rules/testing-signal.md`). `-s` surfaces the printed concordance +breakdown. + +For the Gemini 1024-token check: + +```bash +SF_RUN_GEMINI=1 GOOGLE_API_KEY=... \ + uv run pytest -m gemini --no-cov -s \ + tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py +``` + +## Result (maintainer-filled) + +> **TODO (maintainer):** run `pytest -m anthropic --no-cov -s +> tests/research/187-haiku-calibration/test_haiku_calibration.py` with a +> live `ANTHROPIC_API_KEY` and fill in the table + verdict below from the +> printed breakdown. Then run the Gemini check and record its outcome. + +**Run metadata** + +- Date run: `TODO` +- SignalForge version: `TODO` (e.g. `0.x.y.dev0`) +- Grade model resolved: `claude-haiku-4-5` (assert in-test) +- `max_output_tokens`: `1024` + +**Haiku-vs-Sonnet concordance** + +| Metric | Value | +|---|---| +| Comparable verdicts | `TODO / 44` | +| Agreements | `TODO` | +| Degraded (`score=None`) | `TODO` | +| **Agreement rate** | `TODO %` | +| Decision threshold | 85% | +| **Verdict** | `TODO` PASS / FAIL | + +**Discordances** (if any — `(artifact_id, criterion, sonnet_passed, haiku_passed)`): + +- `TODO` (or "none — full concordance") + +**Gemini 1024-token no-truncation check** + +- Outcome: `TODO` (PASS = no `score=None` degrade / FAIL = truncation observed) +- Notes: `TODO` + +**Disposition:** `TODO` — ship the Haiku default as planned, OR name the +follow-on if concordance fell short. + +## References + +- Issue **#187** — the epic this writeup gates (Haiku grade default). + US-001..US-003 ship the resolver + 1024 cap; US-005 (this) ships the + gated harness + writeup; US-006 owns the docs/rules/CHANGELOG updates. +- `docs/research/179-test-primitive-expansion-retest.md` — the prior + empirical-retest writeup whose structure this mirrors; source of the + "name the shapes that fell through" disposition and the cost reference. +- `tests/grade/test_smoke_real_api.py` — the `anthropic`-gated grade + smoke whose marker + env-skip pattern the concordance gate reuses. +- `tests/grade/test_gemini_grade_live.py` — the `gemini`-gated grade + smoke whose `SF_RUN_GEMINI` + `GOOGLE_API_KEY` env gating the + truncation check reuses. +- `.claude/rules/testing-signal.md` § "End-to-end gated tests" + + § "Engineered determinism" — the gating + determinism conventions. +- `.claude/rules/grade-layer.md` — the grade-layer contract (artifact-id + formatter, DEC-015 degraded path, four-criterion `DEFAULT_RUBRIC`). diff --git a/tests/research/187-haiku-calibration/_substrate.py b/tests/research/187-haiku-calibration/_substrate.py new file mode 100644 index 00000000..dc5d30b6 --- /dev/null +++ b/tests/research/187-haiku-calibration/_substrate.py @@ -0,0 +1,197 @@ +"""Shared substrate for the #187 Haiku-calibration harness (US-005). + +Builds the pinned :class:`Model` + :class:`CandidateSchema` whose +artifacts the calibration harness re-grades, and the loader for the +committed Sonnet-baseline verdict sample. + +The substrate is *engineered-deterministic* per +:file:`.claude/rules/testing-signal.md` § "Engineered determinism over +snapshot normalisation": the candidate is hand-authored so the engine's +:func:`signalforge.grade.engine._stable_artifact_pairs` emits a fixed, +known set of ``artifact_id`` strings. The committed baseline JSON keys on +exactly those ``(artifact_id, criterion_id)`` pairs, so the only live +variable when the maintainer runs the gate is the Haiku re-grade verdict. + +The Sonnet baseline is a **curated sample**, not the raw #179 Phase-B +``grade.jsonl`` dump (which is not committed anywhere in this repo — +``find . -name grade.jsonl`` finds only the drift-detector fixture). +The ``passed`` verdicts in :data:`BASELINE_PATH` are hand-assigned +plausible Sonnet outcomes that span the rubric's calibration space +(clear/strong artifacts pass; vague/weak/redundant artifacts fail), so a +concordant Haiku run reproduces the same verdict distribution. See +:file:`docs/research/187-haiku-calibration.md` for the full provenance +note. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import signalforge as _sf +from signalforge.draft.models import ( + CandidateColumn, + CandidateSchema, + CandidateTestAcceptedValues, + CandidateTestNotNull, + CandidateTestUnique, +) +from signalforge.manifest.models import Column, Model +from signalforge.prune.models import PruneResult + +# The committed Sonnet-baseline verdict sample lives next to this module. +BASELINE_PATH = Path(__file__).with_name("sonnet_baseline_sample.json") + + +def build_model() -> Model: + """Return the pinned manifest :class:`Model` the harness grades. + + Carries exactly the columns referenced by :func:`build_candidate` + so the candidate's tests resolve against real columns. + """ + return Model( + unique_id="model.sf_calib.dim_customers", + name="dim_customers", + resource_type="model", + package_name="sf_calib", + original_file_path="models/marts/dim_customers.sql", + path="marts/dim_customers.sql", + database="sf-calib-proj", + schema="main", # type: ignore[call-arg] + columns={ + "customer_id": Column(name="customer_id"), + "email": Column(name="email"), + "status": Column(name="status"), + }, + raw_code=("select customer_id, email, status from {{ ref('stg_customers') }}"), + ) + + +def build_candidate() -> CandidateSchema: + """Return the pinned :class:`CandidateSchema` the harness grades. + + Hand-authored to span the rubric's calibration space: + + * ``customer_id`` — strong, specific description + rationale + (expected baseline ``passed=True`` on every criterion). + * ``email`` — adequate description, thin rationale (mixed). + * ``status`` — deliberately vague description ("a status field") + and a redundant rationale that restates the description + (expected baseline ``passed=False`` on clarity / rationale / + no-redundant). + + The engine's :func:`_stable_artifact_pairs` derives the + ``artifact_id`` set from this shape; the committed baseline keys on + exactly those ids. See :func:`expected_artifact_ids`. + """ + return CandidateSchema( + name="dim_customers", + description=( + "Curated one-row-per-customer dimension joining stg_customers " + "with stg_customer_status to expose the current lifecycle state " + "of every customer for analytics." + ), + rationale=( + "Materialises the conformed customer dimension consumed by the " + "orders and subscriptions fact tables; resolves status at load " + "time so downstream marts never re-derive lifecycle logic." + ), + columns=( + CandidateColumn( + name="customer_id", + description=( + "Surrogate primary key uniquely identifying each " + "customer. Generated from the source system's natural " + "key via dbt_utils.generate_surrogate_key." + ), + rationale=( + "Used as the join key by every downstream fact table; " + "stability across loads is contractually required." + ), + tests=( + CandidateTestNotNull( + column="customer_id", + rationale="Primary keys must never be null.", + ), + CandidateTestUnique( + column="customer_id", + rationale=( + "One row per customer is the table's declared " + "grain; duplicates indicate a broken join." + ), + ), + ), + ), + CandidateColumn( + name="email", + description=( + "Customer's primary contact email address, lower-cased " + "and trimmed at load time." + ), + rationale="Contact channel.", + tests=(), + ), + CandidateColumn( + name="status", + description="A status field for the customer.", + rationale="Stores the status of the customer.", + tests=( + CandidateTestAcceptedValues( + column="status", + values=("active", "churned", "trialing"), + rationale=( + "The customer lifecycle is a closed set of " + "three states; any other value is a data error." + ), + ), + ), + ), + ), + tests=(), + ) + + +def empty_prune_result(model: Model) -> PruneResult: + """Return an empty :class:`PruneResult` linked to ``model``. + + The no-redundant criterion is the only consumer of dropped tests; + the curated baseline grades the artifacts standalone, so an empty + decision tuple is correct here. + """ + return PruneResult( + model_unique_id=model.unique_id, + decisions=(), + elapsed_ms=0, + signalforge_version=_sf.__version__, + ) + + +def expected_artifact_ids(candidate: CandidateSchema) -> list[str]: + """Return the engine's canonical artifact_id set for ``candidate``. + + Thin wrapper over the engine's own + :func:`signalforge.grade.engine._stable_artifact_pairs` so the + harness never hand-enumerates ids (which would drift the moment the + formatter changes). Importing the private helper is acceptable here: + this is research-tier test code, and the alternative — duplicating + the dotted-path grammar — is exactly the drift risk + :file:`.claude/rules/grade-layer.md` § "_artifact_id_for ... hoist" + warns against. + """ + from signalforge.grade.engine import _stable_artifact_pairs + + return [artifact_id for artifact_id, _text in _stable_artifact_pairs(candidate)] + + +def load_baseline() -> dict[tuple[str, str], bool]: + """Load the committed Sonnet baseline as ``{(artifact_id, crit): passed}``. + + The on-disk shape is a JSON object with a ``"verdicts"`` array of + ``{"artifact_id", "criterion_id", "baseline_passed"}`` records. + """ + raw = json.loads(BASELINE_PATH.read_text(encoding="utf-8")) + out: dict[tuple[str, str], bool] = {} + for record in raw["verdicts"]: + key = (record["artifact_id"], record["criterion_id"]) + out[key] = bool(record["baseline_passed"]) + return out diff --git a/tests/research/187-haiku-calibration/sonnet_baseline_sample.json b/tests/research/187-haiku-calibration/sonnet_baseline_sample.json new file mode 100644 index 00000000..68e05dfe --- /dev/null +++ b/tests/research/187-haiku-calibration/sonnet_baseline_sample.json @@ -0,0 +1,61 @@ +{ + "_provenance": "Curated Sonnet-baseline verdict sample for the #187 Haiku-calibration gate (US-005). NOT the raw #179 Phase-B grade.jsonl dump (which is not committed in this repo). Verdicts are hand-assigned plausible claude-sonnet-4-6 outcomes spanning the four DEFAULT_RUBRIC criteria (clarity, consistency, rationale, no-redundant). Engineered determinism per testing-signal.md: strong/specific artifacts pass; vague/thin/redundant artifacts fail. The artifact_id set is derived from tests/research/187-haiku-calibration/_substrate.build_candidate() via the grade engine's _stable_artifact_pairs; see docs/research/187-haiku-calibration.md for full provenance.", + "baseline_model": "claude-sonnet-4-6", + "rubric_criteria": ["clarity", "consistency", "rationale", "no-redundant"], + "verdicts": [ + {"artifact_id": "column.customer_id.description", "criterion_id": "clarity", "baseline_passed": true}, + {"artifact_id": "column.customer_id.description", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "column.customer_id.description", "criterion_id": "rationale", "baseline_passed": true}, + {"artifact_id": "column.customer_id.description", "criterion_id": "no-redundant", "baseline_passed": true}, + + {"artifact_id": "column.email.description", "criterion_id": "clarity", "baseline_passed": true}, + {"artifact_id": "column.email.description", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "column.email.description", "criterion_id": "rationale", "baseline_passed": true}, + {"artifact_id": "column.email.description", "criterion_id": "no-redundant", "baseline_passed": true}, + + {"artifact_id": "column.status.description", "criterion_id": "clarity", "baseline_passed": false}, + {"artifact_id": "column.status.description", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "column.status.description", "criterion_id": "rationale", "baseline_passed": false}, + {"artifact_id": "column.status.description", "criterion_id": "no-redundant", "baseline_passed": true}, + + {"artifact_id": "column.customer_id.rationale", "criterion_id": "clarity", "baseline_passed": true}, + {"artifact_id": "column.customer_id.rationale", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "column.customer_id.rationale", "criterion_id": "rationale", "baseline_passed": true}, + {"artifact_id": "column.customer_id.rationale", "criterion_id": "no-redundant", "baseline_passed": true}, + + {"artifact_id": "column.email.rationale", "criterion_id": "clarity", "baseline_passed": false}, + {"artifact_id": "column.email.rationale", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "column.email.rationale", "criterion_id": "rationale", "baseline_passed": false}, + {"artifact_id": "column.email.rationale", "criterion_id": "no-redundant", "baseline_passed": true}, + + {"artifact_id": "column.status.rationale", "criterion_id": "clarity", "baseline_passed": false}, + {"artifact_id": "column.status.rationale", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "column.status.rationale", "criterion_id": "rationale", "baseline_passed": false}, + {"artifact_id": "column.status.rationale", "criterion_id": "no-redundant", "baseline_passed": false}, + + {"artifact_id": "model.description", "criterion_id": "clarity", "baseline_passed": true}, + {"artifact_id": "model.description", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "model.description", "criterion_id": "rationale", "baseline_passed": true}, + {"artifact_id": "model.description", "criterion_id": "no-redundant", "baseline_passed": true}, + + {"artifact_id": "model.rationale", "criterion_id": "clarity", "baseline_passed": true}, + {"artifact_id": "model.rationale", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "model.rationale", "criterion_id": "rationale", "baseline_passed": true}, + {"artifact_id": "model.rationale", "criterion_id": "no-redundant", "baseline_passed": true}, + + {"artifact_id": "test.column.customer_id.not_null", "criterion_id": "clarity", "baseline_passed": true}, + {"artifact_id": "test.column.customer_id.not_null", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "test.column.customer_id.not_null", "criterion_id": "rationale", "baseline_passed": true}, + {"artifact_id": "test.column.customer_id.not_null", "criterion_id": "no-redundant", "baseline_passed": true}, + + {"artifact_id": "test.column.customer_id.unique", "criterion_id": "clarity", "baseline_passed": true}, + {"artifact_id": "test.column.customer_id.unique", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "test.column.customer_id.unique", "criterion_id": "rationale", "baseline_passed": true}, + {"artifact_id": "test.column.customer_id.unique", "criterion_id": "no-redundant", "baseline_passed": true}, + + {"artifact_id": "test.column.status.accepted_values", "criterion_id": "clarity", "baseline_passed": true}, + {"artifact_id": "test.column.status.accepted_values", "criterion_id": "consistency", "baseline_passed": true}, + {"artifact_id": "test.column.status.accepted_values", "criterion_id": "rationale", "baseline_passed": true}, + {"artifact_id": "test.column.status.accepted_values", "criterion_id": "no-redundant", "baseline_passed": true} + ] +} diff --git a/tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py b/tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py new file mode 100644 index 00000000..5693d582 --- /dev/null +++ b/tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py @@ -0,0 +1,197 @@ +"""Gemini @ 1024-token cap does NOT truncate-degrade (#187 US-005 / DEC-004). + +DEC-004 of the #187 plan defends ``max_output_tokens=1024`` as the new +grade-config default partly on the claim that **1024 is enough headroom +to prevent Gemini truncation** on a verbose artifact (Gemini's judge +responses run longer than Anthropic's for the same rubric). This gated +check verifies that claim empirically: grade a deliberately verbose +artifact with ``provider="gemini", model="gemini-2.5-flash"`` under the +new ``max_output_tokens=1024`` default and assert the result is **not +degraded** — i.e. the judge produced a clean, parseable verdict +(``score is not None``) rather than a ``None`` from a ``max_tokens`` +truncation / parse failure. + +Gating — belt-and-suspenders, mirroring +:file:`tests/grade/test_gemini_grade_live.py`: + +* ``pytestmark = pytest.mark.gemini`` — the existing ``gemini`` marker, + excluded from the default ``pytest`` run via :file:`pyproject.toml`'s + ``addopts``. Default CI never collects this test. +* A runtime ``pytest.skip(...)`` when ``SF_RUN_GEMINI != "1"`` OR + ``GOOGLE_API_KEY`` is unset/blank. + +Run:: + + SF_RUN_GEMINI=1 GOOGLE_API_KEY=... pytest -m gemini --no-cov \\ + tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +import signalforge as _sf +from signalforge.draft.models import CandidateColumn, CandidateSchema +from signalforge.grade import Criterion, GradingReport, grade_artifacts +from signalforge.grade.config import GradeConfig +from signalforge.manifest.models import Column, Model +from signalforge.prune.models import PruneResult + +pytestmark = pytest.mark.gemini + + +def _skip_reason() -> str | None: + """Return a clear skip-reason string when env vars are missing.""" + if os.environ.get("SF_RUN_GEMINI") != "1": + return "SF_RUN_GEMINI=1 not set" + if not os.environ.get("GOOGLE_API_KEY", "").strip(): + return "GOOGLE_API_KEY env var not set" + return None + + +def test_gemini_grade_at_1024_tokens_is_not_truncation_degraded(tmp_path: Path) -> None: + """Grade a verbose artifact on Gemini @ 1024 tokens; assert no degrade. + + Uses a deliberately long, dense column description + rationale (the + shape most likely to elicit a long judge response) and a + single-criterion rubric. Asserts every returned + :class:`GradingResult` has a non-``None`` score and + ``aggregate_complete is True`` — proving 1024 output tokens left the + Gemini judge enough headroom to finish cleanly. A truncation at the + cap would surface as ``score=None`` (the DEC-015 degraded path) and + fail this assertion loud. + """ + reason = _skip_reason() + if reason: + pytest.skip(reason) + + model = Model( + unique_id="model.sf_calib.dim_customers", + name="dim_customers", + resource_type="model", + package_name="sf_calib", + original_file_path="models/marts/dim_customers.sql", + path="marts/dim_customers.sql", + database="sf-calib-proj", + schema="main", # type: ignore[call-arg] + columns={"customer_id": Column(name="customer_id")}, + raw_code="select 1 as customer_id", + ) + + # A verbose artifact: long description + long rationale. The judge's + # reasoning + evidence for a dense artifact is the worst case for the + # output-token budget. + verbose_description = ( + "Surrogate primary key uniquely identifying each customer record " + "in the conformed customer dimension. Generated deterministically " + "from the upstream source system's composite natural key " + "(source_system_code, source_customer_id) via " + "dbt_utils.generate_surrogate_key so that the same logical " + "customer always hashes to the same surrogate value across full " + "refreshes and incremental loads. Downstream fact tables " + "(fct_orders, fct_subscriptions, fct_support_tickets) join on this " + "column exclusively; the natural key is intentionally not exposed " + "to BI to prevent leakage of source-system implementation detail " + "into the semantic layer. Stability of this surrogate across loads " + "is a hard contract — a change in the hashing inputs would silently " + "fan out as duplicate-or-orphaned rows across every downstream mart." + ) + verbose_rationale = ( + "Documented at this length because the surrogate-key contract is " + "the single most load-bearing invariant in the customer dimension: " + "every downstream join, every slowly-changing-dimension lineage " + "trace, and every data-quality reconciliation depends on it. A " + "future maintainer tempted to swap the hashing inputs, change the " + "salt, or fall back to the raw natural key needs the full rationale " + "in one place so the blast radius is obvious before the change " + "ships rather than discovered in a downstream incident review." + ) + + candidate = CandidateSchema( + name="dim_customers", + description="Curated one-row-per-customer dimension table for analytics.", + rationale="Conformed customer dimension consumed by every downstream fact table.", + columns=( + CandidateColumn( + name="customer_id", + description=verbose_description, + rationale=verbose_rationale, + tests=(), + ), + ), + tests=(), + ) + + rubric = ( + Criterion( + id="clarity", + criterion=( + "Is the column description clear, specific, and actionable? " + "Explain in detail what is strong or weak about it, citing " + "specific phrases, before giving your verdict." + ), + ), + ) + + prune_result = PruneResult( + model_unique_id=model.unique_id, + decisions=(), + elapsed_ms=0, + signalforge_version=_sf.__version__, + ) + + # The contract under test: provider=gemini, the 1024-token DEFAULT. + config = GradeConfig( + provider="gemini", + model="gemini-2.5-flash", + # max_output_tokens left at the new 1024 default deliberately — + # this is the value DEC-004 claims prevents Gemini truncation. + max_retries_429=0, + max_retries_5xx=0, + max_retries_conn=0, + total_budget_seconds=120, + ) + assert config.max_output_tokens == 1024 + + audit_path = tmp_path / "grade.jsonl" + sidecar_path = tmp_path / "grade.json" + + report = grade_artifacts( + model, + candidate, + prune_result, + rubric=rubric, + config=config, + client=None, + audit_path=audit_path, + sidecar_path=sidecar_path, + project_dir=tmp_path, + ) + + assert isinstance(report, GradingReport) + # 5 artifacts × 1 criterion = 5 results (column desc/rationale, model + # desc/rationale — empty model rationale still grades — and no tests). + # Actually: 1 column desc + 1 column rationale + model desc + model + # rationale = 4 artifacts (no tests). Assert the no-degrade contract + # over whatever the engine emits. + assert report.results, "expected at least one grading result" + + # The load-bearing assertion: NO result degraded to score=None. A + # 1024-token truncation on the verbose artifact would flip one or + # more to None and trip this. + degraded = [r for r in report.results if r.score is None] + assert not degraded, ( + "Gemini grade at max_output_tokens=1024 produced degraded " + f"(score=None) results on a verbose artifact: " + f"{[(r.artifact_id, r.criterion_id) for r in degraded]}. " + "DEC-004's claim that 1024 prevents Gemini truncation does NOT " + "hold for this artifact shape." + ) + assert report.aggregate_complete is True + + # Sidecar round-trips through the typed model. + assert sidecar_path.exists() + GradingReport.model_validate_json(sidecar_path.read_text(encoding="utf-8")) diff --git a/tests/research/187-haiku-calibration/test_haiku_calibration.py b/tests/research/187-haiku-calibration/test_haiku_calibration.py new file mode 100644 index 00000000..fcb76f57 --- /dev/null +++ b/tests/research/187-haiku-calibration/test_haiku_calibration.py @@ -0,0 +1,182 @@ +"""Maintainer-run Haiku-vs-Sonnet grade-concordance gate (#187 US-005). + +The #187 plan ships ``claude-haiku-4-5`` as the new grade-default SKU +(US-001..US-003) behind an empirical gate: **does Haiku grade rubric +artifacts concordantly with the prior Sonnet baseline?** The decision +rule is **≥ 85% per-criterion pass/fail agreement** over the pinned +sample (DEC of the #187 plan; mirrors the concordance bar the epic set). + +This module BUILDS that gate. A maintainer RUNS it later with a live +``ANTHROPIC_API_KEY``:: + + pytest -m anthropic --no-cov \\ + tests/research/187-haiku-calibration/test_haiku_calibration.py + +Then transcribes the printed agreement rate into the "Result +(maintainer-filled)" section of +:file:`docs/research/187-haiku-calibration.md`. + +Gating — belt-and-suspenders, mirroring +:file:`tests/grade/test_smoke_real_api.py`: + +* ``pytestmark = pytest.mark.anthropic`` — the existing ``anthropic`` + marker, excluded from the default ``pytest`` run via + :file:`pyproject.toml`'s + ``addopts = "... -m 'not anthropic ...'"``. Default CI never collects + this test. +* A runtime ``pytest.skip(...)`` when ``ANTHROPIC_API_KEY`` is unset (or + blank) — so a maintainer who runs ``pytest -m anthropic`` without a + key sees a clean skip-with-reason, not a noisy auth failure. + +The substrate (pinned candidate + curated Sonnet baseline) lives in +:mod:`tests.research._substrate`; see +:file:`docs/research/187-haiku-calibration.md` for provenance. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest + +# Research-tier sibling import: the harness lives outside the importable +# package tree, so add this directory to ``sys.path`` for ``_substrate``. +sys.path.insert(0, str(Path(__file__).parent)) + +from _substrate import ( # noqa: E402 (path insert must precede import) + build_candidate, + build_model, + empty_prune_result, + expected_artifact_ids, + load_baseline, +) + +from signalforge.grade import grade_artifacts # noqa: E402 +from signalforge.grade.config import GradeConfig # noqa: E402 + +pytestmark = pytest.mark.anthropic + +# The decision rule locked by the #187 plan: Haiku must agree with the +# Sonnet baseline on at least this fraction of per-criterion pass/fail +# verdicts for the default to ship. +_CONCORDANCE_THRESHOLD = 0.85 + + +def _skip_reason() -> str | None: + """Return a clear skip-reason string when the live key is missing.""" + if not os.environ.get("ANTHROPIC_API_KEY", "").strip(): + return "ANTHROPIC_API_KEY not set" + return None + + +def test_haiku_grade_concordance_vs_sonnet_baseline(tmp_path: Path) -> None: + """Re-grade the pinned sample with the Haiku default; assert ≥ 85% concordance. + + Builds the resolved Haiku-default :class:`GradeConfig` (``model`` + resolves to ``claude-haiku-4-5`` via the #187 US-002 provider + fast-model resolver; ``max_output_tokens=1024``), grades the pinned + candidate over the default four-criterion rubric, joins each + :class:`GradingResult` to the committed Sonnet baseline by + ``(artifact_id, criterion_id)``, computes per-criterion pass/fail + agreement, prints the breakdown for the maintainer writeup, and + asserts the rate clears :data:`_CONCORDANCE_THRESHOLD`. + + Degraded results (``score is None`` — DEC-015 of #7) are excluded + from the agreement denominator and reported separately: a degraded + pair could not be positively evaluated, so it is neither a + concordance nor a discordance. + """ + reason = _skip_reason() + if reason: + pytest.skip(reason) + + model = build_model() + candidate = build_candidate() + prune_result = empty_prune_result(model) + baseline = load_baseline() + + # Resolved Haiku default — model=None resolves to claude-haiku-4-5; + # max_output_tokens defaults to 1024. Explicit construction with the + # defaults documents the contract under test. + config = GradeConfig() + assert config.model == "claude-haiku-4-5", ( + "this gate measures the Haiku default; the resolver should have " + f"produced claude-haiku-4-5, got {config.model!r}" + ) + assert config.max_output_tokens == 1024 + + # Sanity: the committed baseline must cover every artifact_id the + # engine will grade (engineered determinism — no silent gaps). + engine_ids = set(expected_artifact_ids(candidate)) + baseline_ids = {artifact_id for (artifact_id, _crit) in baseline} + missing = engine_ids - baseline_ids + assert not missing, f"baseline sample is missing artifact_ids: {sorted(missing)}" + + audit_path = tmp_path / "grade.jsonl" + sidecar_path = tmp_path / "grade.json" + + report = grade_artifacts( + model, + candidate, + prune_result, + config=config, + audit_path=audit_path, + sidecar_path=sidecar_path, + project_dir=tmp_path, + ) + + agreements = 0 + comparable = 0 + degraded = 0 + discordances: list[tuple[str, str, bool, bool]] = [] + + for result in report.results: + key = (result.artifact_id, result.criterion_id) + if key not in baseline: + # Should not happen given the coverage assertion above, but + # never silently fold an unmatched verdict into the rate. + continue + if result.score is None: + degraded += 1 + continue + comparable += 1 + baseline_passed = baseline[key] + if result.passed == baseline_passed: + agreements += 1 + else: + discordances.append( + (result.artifact_id, result.criterion_id, baseline_passed, result.passed) + ) + + rate = agreements / comparable if comparable else 0.0 + + # Human-readable breakdown for the maintainer to paste into the + # writeup's "Result (maintainer-filled)" section. Printed regardless + # of pass/fail so a sub-threshold run still surfaces the discordances. + print("\n=== #187 Haiku-vs-Sonnet grade concordance ===") + print(f"grade model : {config.model}") + print(f"max_output_tokens : {config.max_output_tokens}") + print(f"comparable verdicts : {comparable}") + print(f"agreements : {agreements}") + print(f"degraded (score=None) : {degraded}") + print(f"agreement rate : {rate:.1%}") + print(f"decision threshold : {_CONCORDANCE_THRESHOLD:.0%}") + if discordances: + print("discordances (artifact, criterion, sonnet_passed, haiku_passed):") + for artifact_id, crit, sonnet_p, haiku_p in discordances: + print(f" - {artifact_id} / {crit}: sonnet={sonnet_p} haiku={haiku_p}") + print("================================================") + + # The aggregate must be complete enough to be a meaningful gate: if + # most pairs degraded, the run did not actually measure concordance. + assert comparable >= 1, "no comparable verdicts — every pair degraded" + + assert rate >= _CONCORDANCE_THRESHOLD, ( + f"Haiku concordance {rate:.1%} below the {_CONCORDANCE_THRESHOLD:.0%} " + f"decision rule ({agreements}/{comparable} agreements). " + "The Haiku default does NOT grade concordantly with the Sonnet " + "baseline on this sample; record the discordances in " + "docs/research/187-haiku-calibration.md and reconsider the default." + ) From 5e6ed89d8717211c7f726ec0f929955b7f985cb5 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 12:36:35 -0400 Subject: [PATCH 09/15] =?UTF-8?q?SignalForge-dpy.6:=20US-006=20=E2=80=94?= =?UTF-8?q?=20docs=20+=20CHANGELOG=20+=20rule=20lockstep=20for=20Haiku=20g?= =?UTF-8?q?rade=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update every non-code surface to agree with the #187 shipped behaviour: grade default model resolves per-provider (anthropic -> claude-haiku-4-5, openai -> gpt-4o-mini, gemini -> gemini-2.5-flash) via PROVIDER_FAST_MODELS; grade max_output_tokens default 256 -> 1024; model<->provider compat validation via PROVIDER_SKU_PREFIXES. - docs/grade-ops.md: model + max_output_tokens defaults; per-provider fast-default cost table (haiku/gpt-4o-mini/gemini-2.5-flash) with pricing.py figures; config example shows model auto-resolve. - docs/llm-providers-ops.md: PROVIDER_FAST_MODELS table + gemini-2.5-flash 1024-cap truncation note; default grader-model matrix row. - docs/draft-ops.md: cheap_model shown as bare SKU claude-haiku-4-5. - .claude/rules/grade-layer.md: locked-defaults section (per-provider sentinel resolution + compat validator). - .claude/rules/llm-drafter.md: PROVIDER_FAST_MODELS / PROVIDER_SKU_PREFIXES source note (reusable by a future draft --cheap). - CHANGELOG.md: Unreleased "Changed" entry. README/CLAUDE.md unchanged (neither enumerates the grade default model; the slim CLAUDE.md delegates API surfaces to CHANGELOG/docs by design). Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/rules/grade-layer.md | 11 +++++++- .claude/rules/llm-drafter.md | 1 + CHANGELOG.md | 1 + docs/draft-ops.md | 5 ++-- docs/grade-ops.md | 50 ++++++++++++++++++++++++++---------- docs/llm-providers-ops.md | 43 ++++++++++++++++++++++++++++--- 6 files changed, 90 insertions(+), 21 deletions(-) diff --git a/.claude/rules/grade-layer.md b/.claude/rules/grade-layer.md index c1fcd61b..c696d08a 100644 --- a/.claude/rules/grade-layer.md +++ b/.claude/rules/grade-layer.md @@ -65,7 +65,7 @@ For 4 default criteria × ~12 artifacts per typical model = ~48 calls per `grade The cached prompt block is the rubric criterion list (constant per run); the dynamic block is the per-pair `...` envelope. Anthropic prompt-cache TTL defaults to `"1h"` for the grader (vs. drafter's `"5m"`). -**Tolerant JSON extraction (issue #144).** `parse_grade_response` routes the response through `signalforge._common.json_payload.extract_json_payload` (after `_strip_code_fence`) so a judge that narrates a prose preamble before the `{` still parses. The judge model (`claude-sonnet-4-6`) does NOT support an assistant-turn prefill (API 400), so the parser is the only JSON-only guardrail. Same decode rule as the drafter — decode at the first structural char (`{` or `[`) only, return unchanged on failure — see `llm-drafter.md` § "Tolerant JSON extraction"; a no-JSON response still routes to `GradeOutputError(violation_type="json_parse")` and the conservative degrade. +**Tolerant JSON extraction (issue #144).** `parse_grade_response` routes the response through `signalforge._common.json_payload.extract_json_payload` (after `_strip_code_fence`) so a judge that narrates a prose preamble before the `{` still parses. The Anthropic judge models (the `claude-haiku-4-5` default per #187, or an explicit `claude-sonnet-4-6`) do NOT support an assistant-turn prefill (API 400), so the parser is the only JSON-only guardrail. Same decode rule as the drafter — decode at the first structural char (`{` or `[`) only, return unchanged on failure — see `llm-drafter.md` § "Tolerant JSON extraction"; a no-JSON response still routes to `GradeOutputError(violation_type="json_parse")` and the conservative degrade. ## Reproducibility hash fields on every GradeEvent (DEC-010, DEC-019) @@ -132,6 +132,15 @@ Every `extra="ignore"` production model — `GradingResult`, `GradingReport`, `G The grade-stage block is `{ grade: { model, cache_ttl, max_output_tokens, max_retries_*, total_budget_seconds, min_pass_rate, min_mean_score, fail_on_below_threshold, rubric? } }`. Sibling top-level keys are reserved and silently ignored by the grade loader. `GradeConfig` uses `extra="forbid"`; `_GradeConfigFile` uses `extra="ignore"` at the top level. Mirrors the other layers' top-level-namespace pattern verbatim. +## Locked defaults: per-provider fast model + 1024 output cap (DEC-026, #187) + +`GradeConfig`'s locked defaults (DEC-023..DEC-027) carry two #187 changes: + +- **`model` default is now a per-provider sentinel.** The field defaults to `None`; a `mode="before"` model-validator (`_resolve_model_default`) resolves the sentinel at config-load to the calling provider's fast model from `signalforge.llm.providers.PROVIDER_FAST_MODELS` — `anthropic` → `claude-haiku-4-5`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. (Pre-#187 the default was the bare `claude-sonnet-4-6` literal regardless of provider.) An explicit `model:` is honoured verbatim; after construction the field is always a concrete non-empty string, never `None`. The resolver runs `before` because `GradeConfig` is `frozen=True` and a `mode="after"` mutation would raise. A provider NOT in the fast-model table is left un-injected so the `provider` field-validator surfaces `UnknownProviderError` rather than a masking `KeyError`. +- **`max_output_tokens` default raised 256 → 1024** so a verbose one-line `gemini-2.5-flash` grade JSON is not truncated (a truncation would surface as the wrong typed degrade — `GradeOutputError` instead of `GradeLLMError`). Still a cap, not a target; the expected JSON is ~150 tokens, so the larger ceiling costs nothing on the happy path. + +**Model↔provider compat validator (DEC-006 of #187).** A `mode="after"` validator (`_validate_model_provider_compat`) reads `signalforge.llm.providers.PROVIDER_SKU_PREFIXES` (`anthropic` → `claude-`, `openai` → `gpt-`, `gemini` → `gemini-`) and fails loud at config-load when `provider` is a known-prefix provider AND the resolved/explicit `model` carries a *different* known provider's SKU prefix (e.g. `provider: openai` with a `claude-` model). Two cases are deliberately left alone: a model whose prefix matches no known provider (forward-compat for future SKUs) and a registry-valid provider outside the prefix table (custom/plugin providers may use any model name). Both `PROVIDER_FAST_MODELS` and `PROVIDER_SKU_PREFIXES` are the single source of truth — no hardcoded SKUs or prefixes in the grade config module. Every fast-model value is an exact key in `signalforge.llm.pricing.PRICES`, so the `--estimate` path never raises on the resolved default. + ## Schema-version surfaces Two exported names ship but are not consumed. **Both re-verified still-reserved on 2026-05-22 (issue #62)** — the v0.1 designs each anticipated remain intact, so neither was promoted: diff --git a/.claude/rules/llm-drafter.md b/.claude/rules/llm-drafter.md index 4d625dfc..bc18bbdd 100644 --- a/.claude/rules/llm-drafter.md +++ b/.claude/rules/llm-drafter.md @@ -18,6 +18,7 @@ Every `# pyright: ignore[...]` and `# type: ignore[...]` comment for the Anthrop - **Neutral value objects:** `UsageMetrics` + the `ExceptionCategory` enum (`AUTH`, `RATE_LIMIT`, `SERVER_ERROR`, `CONNECTION`, `NO_RETRY`) keep the orchestrator off vendor-shaped dicts. - **Capability-gated behaviour (DEC-008):** `supports_prompt_caching=False` ⇒ no `cache_control` marker, no `extended-cache-ttl` beta header, 0 cache tokens, no dual-zero anomaly WARNING. `supports_token_count=False` ⇒ skip the pre-send count gate (no pre-send `LLMCacheTooLargeError`). Anthropic sets both `True`, so its emitted bytes/control flow are unchanged — the byte-identity gate (fixtures + prompt-cache snapshot + drift detectors) is the regression guard. - **`provider` config field (DEC-007):** `DraftConfig.provider` (`llm:` block) and `GradeConfig.provider` (`grade:` block), both registry-validated `str` defaulting to `"anthropic"` — **deliberately NOT a `Literal`** (a registry is a plugin point that grows; #136/#137 register a provider instead of editing a Literal in two configs). The validator raises `UnknownProviderError` (an `LLMError`, so Pydantic v2 does NOT wrap it into `ValidationError` — it propagates raw with the available-keys remediation). +- **Provider→string mappings (#187 US-001):** `PROVIDER_FAST_MODELS` and `PROVIDER_SKU_PREFIXES` also live in `signalforge.llm.providers` and are the single source of truth for two cross-cutting per-provider facts. `PROVIDER_FAST_MODELS` (`anthropic` → `claude-haiku-4-5`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`) is the cheap/fast judge SKU per provider — every value is an exact key in `signalforge.llm.pricing.PRICES`. It drives the `GradeConfig.model` per-provider default (the `grade.model:`-unset sentinel resolves to it at config-load; see `grade-layer.md`). `PROVIDER_SKU_PREFIXES` (`claude-` / `gpt-` / `gemini-`) drives the cost-rollup's prefix dispatch AND the grade model↔provider compat validator. Both tables are **reusable by a future draft `--cheap` flow** — when the drafter graduates an automatic `cheap_model` swap, resolve it from `PROVIDER_FAST_MODELS` keyed on `DraftConfig.provider` rather than hardcoding `claude-haiku-4-5` (today `DraftConfig.cheap_model` defaults to the bare `claude-haiku-4-5` SKU, which matches the `anthropic` entry). **Gate the cache marker on BOTH capability flags, not just `supports_prompt_caching` (#135 QG lesson).** `call_llm` sets `cache_marker_active = supports_prompt_caching AND supports_token_count`. The pre-send count gate is what enforces the sub-minimum drop + the 8000-token oversize cap; attaching a `cache_control` marker without that gate having run would send an *unvalidated* marker (a sub-minimum block silently no-ops the marker — paying the input premium with no discount; an oversize block bypasses `LLMCacheTooLargeError`). Anthropic is `True/True` so the default path is unaffected, but a future provider that caches yet has no token-count API (`True/False`) must degrade to no-caching rather than send an unguarded marker. A new provider's capability flags are load-bearing — set them honestly, and don't assume "supports caching" alone is sufficient to attach a marker. diff --git a/CHANGELOG.md b/CHANGELOG.md index 13720126..ce59977c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ All notable changes to SignalForge are documented here. The format is loosely ba ### Changed +- **Grade default judge model switched to per-provider fast models (#187).** When `grade.model:` is **omitted**, the grade-config loader now resolves it at config-load to the calling provider's fast model from the new `signalforge.llm.providers.PROVIDER_FAST_MODELS` table — `anthropic` → `claude-haiku-4-5` (was the bare `claude-sonnet-4-6` literal regardless of provider), `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. An explicit `grade.model:` is still honoured verbatim. Every fast-model value is an exact key in `signalforge.llm.pricing.PRICES`, so the `--estimate` path never raises on the resolved default. Switching the Anthropic grade judge from Sonnet to the Haiku default cuts the per-token grade cost ~3.75× ($0.80/$4.00 vs. $3.00/$15.00 per MTok input/output). `GradeConfig.max_output_tokens` default raised `256 → 1024` so a verbose one-line `gemini-2.5-flash` grade JSON is not truncated (a truncation would surface as the wrong typed degrade); still a cap, not a target — the expected JSON is ~150 tokens, so the larger ceiling costs nothing on the happy path. New model↔provider compatibility validation: a SKU-prefix/provider mismatch (e.g. `grade.provider: openai` with a `claude-` model) now fails loud at config-load, driven by the new `signalforge.llm.providers.PROVIDER_SKU_PREFIXES` table. See `docs/grade-ops.md` and `docs/llm-providers-ops.md` § Per-provider fast-grade defaults. - **Grade-layer audit JSONL ordering becomes arrival-order under concurrent dispatch (#186).** Record shape is unchanged — `audit_schema_version` stays `Literal[1]` (only on-disk sequence differs). Operators or external tooling that depend on stable ordering should sort post-load by `(artifact_id, criterion_id)` (the SignalForge test suite uses `tests/grade/_helpers.py::_sort_grade_events(...)`). Setting `grade.max_concurrent_calls: 1` in `signalforge.yml` recovers the v0.1 `(criterion, artifact)` iteration order bit-for-bit. - **Anthropic prompt-cache cost penalty under concurrent grade dispatch (#186).** Calls `1..max_concurrent_calls` dispatch in parallel before any response returns, so each pays the cache-write premium (~1.25× input cost on the cached rubric block) instead of the cache-read discount (~0.10×). For the default cap of 10 and the ~430-token rubric block: ~4 450 extra input-token-equivalents per typical model run (~$0.003–$0.005 absolute). Operators cost-sensitive enough to care can set `grade.max_concurrent_calls: 1` to recover the v0.1 cost profile (trading off ~5–6× wall-clock reduction). OpenAI and Gemini do not support prompt caching, so no penalty applies on those providers. See `docs/grade-ops.md` § "Concurrency (asyncio orchestrator)". - **Pytest downgrade `9.x → 8.x` (#186).** Adding `pytest-asyncio>=0.23,<1` to dev-deps pins `pytest<9` (no `pytest-asyncio` release supports pytest 9 yet). Emits a `PytestConfigWarning: Unknown config option: strict_markers` for the `strict_markers = true` ini key (pytest 9-specific per `testing-signal.md`'s "pytest 9 quirk"); the warning is informational and does not fail validation. The `strict_markers = true` ini key is retained — `testing-signal.md` says BOTH `--strict-markers` addopts AND `strict_markers = true` are required on pytest 9; stripping the key would silently regress when pytest-asyncio publishes a 9-compatible release and the pin can be lifted. diff --git a/docs/draft-ops.md b/docs/draft-ops.md index e54973db..642b494d 100644 --- a/docs/draft-ops.md +++ b/docs/draft-ops.md @@ -926,7 +926,7 @@ other stages and silently ignored by the draft loader. llm: provider: anthropic # registry-validated; "anthropic" + "openai" + "gemini" are registered (see provider sections below) model: claude-sonnet-4-6 - cheap_model: claude-haiku-4-5-20251001 + cheap_model: claude-haiku-4-5 max_output_tokens: 4096 cache_ttl: 5m # one of "5m" | "1h" max_retries_429: 3 @@ -949,7 +949,8 @@ Field-by-field: Default `claude-sonnet-4-6`. Any string the SDK accepts is allowed. - **`cheap_model`** — informational; not selected automatically. The CLI (#9) flips on `--cheap` to swap `model` for this value. - Default `claude-haiku-4-5-20251001`. + Default `claude-haiku-4-5` (bare SKU — matches the `anthropic` entry + in `signalforge.llm.providers.PROVIDER_FAST_MODELS`). - **`max_output_tokens`** — Anthropic `max_tokens` ceiling. Must be positive (validator). - **`cache_ttl`** — `Literal["5m", "1h"]`. `"1h"` opts into the diff --git a/docs/grade-ops.md b/docs/grade-ops.md index 525d4a01..197074e0 100644 --- a/docs/grade-ops.md +++ b/docs/grade-ops.md @@ -107,18 +107,19 @@ DEC-020 — every pipeline stage gets one top-level key). Sibling keys (`safety:`, `llm:`, `prune:`, future `diff:` …) are reserved for other stages and silently ignored by the grade loader. -The full schema (every knob, every default, all v0.1 types), mirroring -`tests/fixtures/grade/example_config.yml` (exercised by -`test_load_grade_config_doc_example_round_trips` so the example and the -loader cannot drift): +The full schema (every knob, every default, all v0.1 types). The +companion fixture `tests/fixtures/grade/example_config.yml` (exercised by +`test_load_grade_config_doc_example_round_trips`) pins that the loader +accepts a representative `grade:` block; both this example and the fixture +load cleanly through `load_grade_config`: ```yaml -# signalforge.yml — grade stage configuration (v0.1) +# signalforge.yml — grade stage configuration grade: provider: anthropic # registry-validated; "anthropic" + "openai" + "gemini" are registered (see provider sections below) - model: claude-sonnet-4-6 # model id (default) + # model: claude-haiku-4-5 # omit to auto-resolve to the provider's fast model (anthropic -> claude-haiku-4-5); set explicitly to override cache_ttl: 1h # Prompt-cache TTL ('5m' or '1h') - max_output_tokens: 256 # Per-criterion JSON response cap + max_output_tokens: 1024 # Per-criterion JSON response cap (default 1024) max_retries_429: 3 # Rate-limit retry budget max_retries_5xx: 1 max_retries_conn: 1 @@ -159,9 +160,9 @@ grade: Field-by-field: - **`provider`** — The LLM provider strategy name (issue #135 DEC-007), resolved against the `signalforge.llm.providers` registry and threaded into `call_llm` from the per-criterion judge call, independently of the drafter's `DraftConfig.provider`. Default `"anthropic"`. An unknown value fails loud at config-load, listing the registered provider names. Deliberately a registry-validated `str`, not a `Literal` — the provider registry is a forward-looking plugin point. Today `anthropic`, `openai`, and `gemini` are registered; see [OpenAI provider](#openai-provider) and [Gemini provider](#gemini-provider) below for the non-default options. -- **`model`** — The model id used by every per-pair judge call. Default `claude-sonnet-4-6`. Mirrors `DraftConfig.model` default. Haiku 4.5 is documented as a v0.2 cost-conscious option but not exposed in v0.1. +- **`model`** — The model id used by every per-pair judge call. **Default resolves per-provider at config-load** (#187): when `model:` is omitted, the loader injects the calling provider's fast model from `signalforge.llm.providers.PROVIDER_FAST_MODELS` — `anthropic` → `claude-haiku-4-5`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. An explicit `model:` is honoured verbatim. A SKU-prefix/provider mismatch (e.g. `provider: openai` with a `claude-` model) fails loud at config-load via the model↔provider compat validator (reusing `signalforge.llm.providers.PROVIDER_SKU_PREFIXES`). - **`cache_ttl`** — `Literal["5m", "1h"]`. Default `"1h"` (vs. the drafter's `"5m"`) because 60 sequential per-criterion calls under retry backoff can stretch beyond a 5-minute window; `"1h"` gives margin at no extra cost (cache writes are one-shot regardless of TTL). -- **`max_output_tokens`** — Per-criterion judge response cap. Default `256`. The expected JSON response is ~150 tokens; 256 gives 2× safety. Independent of `DraftConfig.max_output_tokens`. +- **`max_output_tokens`** — Per-criterion judge response cap. Default `1024` (#187 — raised from 256 so a verbose one-line `gemini-2.5-flash` grade JSON is not truncated; the expected JSON response is still ~150 tokens, so the larger ceiling costs nothing on the happy path while removing truncation risk). Independent of `DraftConfig.max_output_tokens`. - **`max_retries_429` / `max_retries_5xx` / `max_retries_conn`** — Per-call retry budgets at the centralised, provider-neutral `signalforge.llm.call_llm` seam (#5 DEC-012; #135 DEC-005). Defaults `3 / 1 / 1` mirror `DraftConfig`; dial down for batch CLI mode where one retry-exhaustion is preferable to dozens of stalled calls. - **`total_budget_seconds`** — Whole-run wall-clock budget. Default `300` (5 minutes — historically ~3× safety on 60 sequential calls × 1s p50; ~10× headroom under concurrent dispatch). Mirrors `PruneConfig.total_budget_seconds` semantics: when the budget trips, every remaining `(artefact, criterion)` pair lands as a degraded `GradingResult(score=None)` rather than silently dropped. Under the asyncio orchestrator (issue #186) the budget is enforced via `asyncio.timeout(...)` wrapping the `TaskGroup`; on trip, un-completed pairs are filled in by a synthesis pass with `reasoning="grade budget exceeded ({N}s) before evaluation"`. Tests inject deterministic timing via the module-level `_async_sleep` alias (mirrors the `_sleep` injection pattern from `llm-drafter.md` DEC-004). - **`max_concurrent_calls`** — Number of in-flight `(artifact × criterion)` LLM calls allowed concurrently (issue #186). Default `10` matches the typical Anthropic-tier throughput sweet-spot; bounded `[1, 100]` with `@field_validator` rejecting `< 1` or `> 100` at config-load. Setting `1` yields v0.1 sequential behaviour bit-for-bit (semaphore-of-1 serialises in dispatch order, preserving `(criterion, artifact)` JSONL ordering). Under concurrent dispatch the audit JSONL lands in **arrival order** (`audit_schema_version` unchanged at `Literal[1]`); the `tests/grade/_helpers.py::_sort_grade_events(lines)` helper restores deterministic ordering for tests that snapshot the file. CLI does not expose a `--max-concurrent-calls` flag (mirrors `min_pass_rate` / `min_mean_score` config-file-only convention). @@ -592,6 +593,25 @@ specifically. See § "Measured baseline (2026-05-29)" for the full-suite rollup ($1.38/run across the three providers). +**Per-provider fast-default judge models (#187).** When `grade.model:` is +omitted the loader resolves to the calling provider's *fast* model — the +cheapest registered SKU per provider (`signalforge.llm.providers.PROVIDER_FAST_MODELS`). +The rows below pair each fast default with its per-MTok USD list price from +`signalforge.llm.pricing` (pricing-table version `2026-05-28`) and an +*estimated* per-model grade cost, scaled from the Sonnet baseline above by +the input/output price ratio (estimate, not a measured run): + +| Provider × fast default | Input $/MTok | Output $/MTok | Est. per-model grade cost | Notes | +|----------------------------------|--------------|---------------|---------------------------|----------------------------------------------------------------------------------------| +| Anthropic `claude-haiku-4-5` | $0.80 | $4.00 | ~$0.10 | The new default grade judge; ~3.75× cheaper than `claude-sonnet-4-6` per token. | +| OpenAI `gpt-4o-mini` | $0.15 | $0.60 | ~$0.013 | ~16.7× cheaper than `gpt-4o` per token; the fast default when `provider: openai`. | +| Gemini `gemini-2.5-flash` | $0.30 | $2.50 | ~$0.045 | Already the documented mid-tier default; the measured figure above is this same SKU. | + +For completeness, the registered Anthropic SKUs span `claude-haiku-4-5` +($0.80 / $4.00 per MTok), `claude-sonnet-4-6` ($3.00 / $15.00), and +`claude-opus-4-7` ($15.00 / $75.00) — switching the grade judge from +Sonnet to the Haiku default cuts the per-token grade cost ~3.75×. + **Fan-out comparison vs the batched alternative:** - The per-criterion fan-out (one LLM call per `(criterion × artefact)`) @@ -627,11 +647,13 @@ default fan-out is too expensive for their use case: `GradeBudgetExceededError` only fires if the budget trips before ANY criterion runs (a hard "the run did nothing" failure); a partial run completes with `aggregate_complete: false`. -- **`max_output_tokens`** (default `256`) — Per-call output cap. The - expected JSON response is ~150 tokens; tightening to 192 trims ~25% - off the output-token bill at marginal risk of truncated JSON - (handled by `GradeOutputError(violation_type="json_parse")` and the - degraded path). +- **`max_output_tokens`** (default `1024`) — Per-call output cap. The + expected JSON response is ~150 tokens, so the cap is a truncation + guard, not a target; the default was raised from 256 to 1024 in #187 + so a verbose one-line `gemini-2.5-flash` grade JSON doesn't truncate. + Tightening it trims the output-token bill at the cost of truncation + risk (handled by `GradeOutputError(violation_type="json_parse")` and + the degraded path); see the per-provider floors below before lowering it. - **`cache_ttl: "1h"`** (default) — Cache-read economics. Prompt caching is a **provider capability** (issue #135): the `cache_control` marker, the extended-cache-ttl beta header, and the pre-send diff --git a/docs/llm-providers-ops.md b/docs/llm-providers-ops.md index 2063421f..64bb215d 100644 --- a/docs/llm-providers-ops.md +++ b/docs/llm-providers-ops.md @@ -104,7 +104,8 @@ not a `Literal`). See [Adding a provider](#adding-a-provider) below. | **Server-side JSON mode** | n/a (Anthropic parser tolerant) | ✅ `response_format={"type":"json_object"}` | ✅ `response_mime_type="application/json"` | | **Pre-send `count_tokens` gate** | ✅ | ❌ (no SDK token-count API) | ❌ (deferred — Gemini has the API but we don't gate on it for cache parity) | | **`cache_ttl` config** | honoured (`"5m"` / `"1h"`) | silently ignored | silently ignored | -| **Default model** | `claude-sonnet-4-6` (drafter + grader) | `gpt-4o` | drafter unset; grader `gemini-2.5-flash` | +| **Default drafter model** | `claude-sonnet-4-6` | `gpt-4o` | unset | +| **Default grader model** | `claude-haiku-4-5` (fast default, #187) | `gpt-4o-mini` (fast default) | `gemini-2.5-flash` (fast default) | | **Live smoke marker** | `@pytest.mark.anthropic` | `@pytest.mark.openai` | `@pytest.mark.gemini` | | **Live smoke env** | `ANTHROPIC_API_KEY` | `SF_RUN_OPENAI=1` + `OPENAI_API_KEY` | `SF_RUN_GEMINI=1` + `GOOGLE_API_KEY` | @@ -114,6 +115,38 @@ without a read discount. For a one-call-per-`generate` drafter this is modest; for the multi-call grader, budget per-call input-token spend at full rates. +### Per-provider fast-grade defaults + +When `grade.model:` is **omitted**, the grade-config loader resolves it +at config-load to the calling provider's *fast* model — the cheapest +registered SKU per provider. The single source of truth is the +`PROVIDER_FAST_MODELS` table in `signalforge.llm.providers` (#187 US-001): + +| Grade provider | Fast default SKU (`PROVIDER_FAST_MODELS`) | +|---|---| +| `anthropic` | `claude-haiku-4-5` | +| `openai` | `gpt-4o-mini` | +| `gemini` | `gemini-2.5-flash` | + +Every value is an exact key in `signalforge.llm.pricing.PRICES`, so the +`--estimate` cost-preview path and `pricing.lookup(model)` never raise on +the resolved default. An explicit `grade.model:` is always honoured +verbatim; the per-provider fast model only fills in when the field is +unset. A model↔provider mismatch (e.g. `grade.provider: openai` with a +`claude-` model) **fails loud at config-load** — the SKU-prefix table +`PROVIDER_SKU_PREFIXES` (also in `signalforge.llm.providers`) drives the +compat check. + +> **Gemini grade truncation note.** Operators grading with +> `gemini-2.5-flash` rely on the `grade.max_output_tokens` default of +> `1024` (#187 — raised from 256) to avoid mid-string truncation of the +> grade JSON. Gemini's verbose reasoning style needs more headroom than +> the older 256 cap allowed; see +> [`docs/grade-ops.md` § Per-provider `max_output_tokens` recommended +> floors](grade-ops.md#per-provider-max_output_tokens-recommended-floors) +> for the fixture-scale floor guidance (Gemini may want `4096` on wider +> models). + ## Supported providers ### Anthropic (default) @@ -121,8 +154,10 @@ spend at full rates. The shipped default for both stages. No extra install; set `ANTHROPIC_API_KEY` and SignalForge runs out of the box. -- **Default models:** `claude-sonnet-4-6` (drafter + grader), - `claude-haiku-4-5-20251001` (drafter `cheap_model`). +- **Default models:** `claude-sonnet-4-6` (drafter), `claude-haiku-4-5` + (grader fast default per #187 — see [Per-provider fast-grade + defaults](#per-provider-fast-grade-defaults) below), `claude-haiku-4-5` + (drafter `cheap_model`). - **Prompt caching:** active. Drafter caches the manifest summary block; grader caches the rubric criterion list. `cache_ttl: 1h` opts into the `extended-cache-ttl-2025-04-11` beta header. The @@ -153,7 +188,7 @@ llm: max_output_tokens: 4096 grade: provider: openai - model: gpt-4o + model: gpt-4o # explicit override; omit to auto-resolve to the fast default gpt-4o-mini ``` - **Install:** `pip install signalforge-dbt[openai]` — pulls From aaf0be3af7ff7faffbdf7a29db4218fdfe8bd199 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 12:54:34 -0400 Subject: [PATCH 10/15] =?UTF-8?q?SignalForge-dpy.7:=20Quality=20gate=20?= =?UTF-8?q?=E2=80=94=20fix=20bugs=20from=204=20code-review=20passes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real bug (passes 1 & 2): GradeConfig.model could stay None for a registered-but-not-in-PROVIDER_FAST_MODELS provider (the custom/plugin growth path), contradicting the engine/_estimate `assert model is not None`. Now fails loud at config-load requiring an explicit grade.model, restoring the post-construction invariant. Updated the #135 neutrality test contract + the divergent-providers estimate test to supply an explicit model. Accuracy fixes (passes 3 & 4): - Soften the "gemini-2.5-flash JSON is not truncated at 1024" claim across config docstring, grade-ops.md, CHANGELOG — 1024 reduces but doesn't eliminate Gemini truncation at full-fixture scale (floors table wants 4096). - grade-ops.md: gpt-4o example reframed as explicit override (default resolves to gpt-4o-mini). - hand-calc estimate test: pin the resolved literal claude-haiku-4-5 so a resolver regression fails there instead of drifting into a tautology. - calibration gate: reject degraded-dominated runs (comparable >= degraded). - round-trip test docstring + example_config.yml comments corrected (fixture pins explicit overrides, not the new defaults). CodeRabbit: unavailable in this environment (skipped). Full validation green. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 +- docs/grade-ops.md | 9 ++-- src/signalforge/grade/config.py | 44 ++++++++++++++----- tests/cli/test_estimate_engine.py | 6 ++- tests/cli/test_generate_estimate.py | 7 ++- tests/fixtures/grade/example_config.yml | 4 +- tests/grade/test_config.py | 8 +++- tests/grade/test_provider_neutrality.py | 14 +++++- .../test_haiku_calibration.py | 13 +++++- 9 files changed, 82 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce59977c..435ea334 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ All notable changes to SignalForge are documented here. The format is loosely ba ### Changed -- **Grade default judge model switched to per-provider fast models (#187).** When `grade.model:` is **omitted**, the grade-config loader now resolves it at config-load to the calling provider's fast model from the new `signalforge.llm.providers.PROVIDER_FAST_MODELS` table — `anthropic` → `claude-haiku-4-5` (was the bare `claude-sonnet-4-6` literal regardless of provider), `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. An explicit `grade.model:` is still honoured verbatim. Every fast-model value is an exact key in `signalforge.llm.pricing.PRICES`, so the `--estimate` path never raises on the resolved default. Switching the Anthropic grade judge from Sonnet to the Haiku default cuts the per-token grade cost ~3.75× ($0.80/$4.00 vs. $3.00/$15.00 per MTok input/output). `GradeConfig.max_output_tokens` default raised `256 → 1024` so a verbose one-line `gemini-2.5-flash` grade JSON is not truncated (a truncation would surface as the wrong typed degrade); still a cap, not a target — the expected JSON is ~150 tokens, so the larger ceiling costs nothing on the happy path. New model↔provider compatibility validation: a SKU-prefix/provider mismatch (e.g. `grade.provider: openai` with a `claude-` model) now fails loud at config-load, driven by the new `signalforge.llm.providers.PROVIDER_SKU_PREFIXES` table. See `docs/grade-ops.md` and `docs/llm-providers-ops.md` § Per-provider fast-grade defaults. +- **Grade default judge model switched to per-provider fast models (#187).** When `grade.model:` is **omitted**, the grade-config loader now resolves it at config-load to the calling provider's fast model from the new `signalforge.llm.providers.PROVIDER_FAST_MODELS` table — `anthropic` → `claude-haiku-4-5` (was the bare `claude-sonnet-4-6` literal regardless of provider), `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. An explicit `grade.model:` is still honoured verbatim. Every fast-model value is an exact key in `signalforge.llm.pricing.PRICES`, so the `--estimate` path never raises on the resolved default. Switching the Anthropic grade judge from Sonnet to the Haiku default cuts the per-token grade cost ~3.75× ($0.80/$4.00 vs. $3.00/$15.00 per MTok input/output). `GradeConfig.max_output_tokens` default raised `256 → 1024` to substantially reduce truncation of a verbose one-line `gemini-2.5-flash` grade JSON (a truncation would surface as the wrong typed degrade); still a cap, not a target — the expected JSON is ~150 tokens, so the larger ceiling costs nothing on the happy path. 1024 reduces but does not fully eliminate Gemini truncation at the full-fixture scale; `docs/grade-ops.md` § per-provider floors recommends `4096` for Gemini-heavy runs. New model↔provider compatibility validation: a SKU-prefix/provider mismatch (e.g. `grade.provider: openai` with a `claude-` model) now fails loud at config-load, driven by the new `signalforge.llm.providers.PROVIDER_SKU_PREFIXES` table. See `docs/grade-ops.md` and `docs/llm-providers-ops.md` § Per-provider fast-grade defaults. - **Grade-layer audit JSONL ordering becomes arrival-order under concurrent dispatch (#186).** Record shape is unchanged — `audit_schema_version` stays `Literal[1]` (only on-disk sequence differs). Operators or external tooling that depend on stable ordering should sort post-load by `(artifact_id, criterion_id)` (the SignalForge test suite uses `tests/grade/_helpers.py::_sort_grade_events(...)`). Setting `grade.max_concurrent_calls: 1` in `signalforge.yml` recovers the v0.1 `(criterion, artifact)` iteration order bit-for-bit. - **Anthropic prompt-cache cost penalty under concurrent grade dispatch (#186).** Calls `1..max_concurrent_calls` dispatch in parallel before any response returns, so each pays the cache-write premium (~1.25× input cost on the cached rubric block) instead of the cache-read discount (~0.10×). For the default cap of 10 and the ~430-token rubric block: ~4 450 extra input-token-equivalents per typical model run (~$0.003–$0.005 absolute). Operators cost-sensitive enough to care can set `grade.max_concurrent_calls: 1` to recover the v0.1 cost profile (trading off ~5–6× wall-clock reduction). OpenAI and Gemini do not support prompt caching, so no penalty applies on those providers. See `docs/grade-ops.md` § "Concurrency (asyncio orchestrator)". - **Pytest downgrade `9.x → 8.x` (#186).** Adding `pytest-asyncio>=0.23,<1` to dev-deps pins `pytest<9` (no `pytest-asyncio` release supports pytest 9 yet). Emits a `PytestConfigWarning: Unknown config option: strict_markers` for the `strict_markers = true` ini key (pytest 9-specific per `testing-signal.md`'s "pytest 9 quirk"); the warning is informational and does not fail validation. The `strict_markers = true` ini key is retained — `testing-signal.md` says BOTH `--strict-markers` addopts AND `strict_markers = true` are required on pytest 9; stripping the key would silently regress when pytest-asyncio publishes a 9-compatible release and the pin can be lifted. diff --git a/docs/grade-ops.md b/docs/grade-ops.md index 197074e0..874ec0a0 100644 --- a/docs/grade-ops.md +++ b/docs/grade-ops.md @@ -162,7 +162,7 @@ Field-by-field: - **`provider`** — The LLM provider strategy name (issue #135 DEC-007), resolved against the `signalforge.llm.providers` registry and threaded into `call_llm` from the per-criterion judge call, independently of the drafter's `DraftConfig.provider`. Default `"anthropic"`. An unknown value fails loud at config-load, listing the registered provider names. Deliberately a registry-validated `str`, not a `Literal` — the provider registry is a forward-looking plugin point. Today `anthropic`, `openai`, and `gemini` are registered; see [OpenAI provider](#openai-provider) and [Gemini provider](#gemini-provider) below for the non-default options. - **`model`** — The model id used by every per-pair judge call. **Default resolves per-provider at config-load** (#187): when `model:` is omitted, the loader injects the calling provider's fast model from `signalforge.llm.providers.PROVIDER_FAST_MODELS` — `anthropic` → `claude-haiku-4-5`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. An explicit `model:` is honoured verbatim. A SKU-prefix/provider mismatch (e.g. `provider: openai` with a `claude-` model) fails loud at config-load via the model↔provider compat validator (reusing `signalforge.llm.providers.PROVIDER_SKU_PREFIXES`). - **`cache_ttl`** — `Literal["5m", "1h"]`. Default `"1h"` (vs. the drafter's `"5m"`) because 60 sequential per-criterion calls under retry backoff can stretch beyond a 5-minute window; `"1h"` gives margin at no extra cost (cache writes are one-shot regardless of TTL). -- **`max_output_tokens`** — Per-criterion judge response cap. Default `1024` (#187 — raised from 256 so a verbose one-line `gemini-2.5-flash` grade JSON is not truncated; the expected JSON response is still ~150 tokens, so the larger ceiling costs nothing on the happy path while removing truncation risk). Independent of `DraftConfig.max_output_tokens`. +- **`max_output_tokens`** — Per-criterion judge response cap. Default `1024` (#187 — raised from 256 to substantially reduce truncation risk for a verbose one-line `gemini-2.5-flash` grade JSON; the expected JSON response is still ~150 tokens, so the larger ceiling costs nothing on the happy path). 1024 reduces but does not fully eliminate Gemini truncation at scale — see the per-provider floors below; Gemini-heavy runs may want `4096`. Independent of `DraftConfig.max_output_tokens`. - **`max_retries_429` / `max_retries_5xx` / `max_retries_conn`** — Per-call retry budgets at the centralised, provider-neutral `signalforge.llm.call_llm` seam (#5 DEC-012; #135 DEC-005). Defaults `3 / 1 / 1` mirror `DraftConfig`; dial down for batch CLI mode where one retry-exhaustion is preferable to dozens of stalled calls. - **`total_budget_seconds`** — Whole-run wall-clock budget. Default `300` (5 minutes — historically ~3× safety on 60 sequential calls × 1s p50; ~10× headroom under concurrent dispatch). Mirrors `PruneConfig.total_budget_seconds` semantics: when the budget trips, every remaining `(artefact, criterion)` pair lands as a degraded `GradingResult(score=None)` rather than silently dropped. Under the asyncio orchestrator (issue #186) the budget is enforced via `asyncio.timeout(...)` wrapping the `TaskGroup`; on trip, un-completed pairs are filled in by a synthesis pass with `reasoning="grade budget exceeded ({N}s) before evaluation"`. Tests inject deterministic timing via the module-level `_async_sleep` alias (mirrors the `_sleep` injection pattern from `llm-drafter.md` DEC-004). - **`max_concurrent_calls`** — Number of in-flight `(artifact × criterion)` LLM calls allowed concurrently (issue #186). Default `10` matches the typical Anthropic-tier throughput sweet-spot; bounded `[1, 100]` with `@field_validator` rejecting `< 1` or `> 100` at config-load. Setting `1` yields v0.1 sequential behaviour bit-for-bit (semaphore-of-1 serialises in dispatch order, preserving `(criterion, artifact)` JSONL ordering). Under concurrent dispatch the audit JSONL lands in **arrival order** (`audit_schema_version` unchanged at `Literal[1]`); the `tests/grade/_helpers.py::_sort_grade_events(lines)` helper restores deterministic ordering for tests that snapshot the file. CLI does not expose a `--max-concurrent-calls` flag (mirrors `min_pass_rate` / `min_mean_score` config-file-only convention). @@ -650,7 +650,10 @@ default fan-out is too expensive for their use case: - **`max_output_tokens`** (default `1024`) — Per-call output cap. The expected JSON response is ~150 tokens, so the cap is a truncation guard, not a target; the default was raised from 256 to 1024 in #187 - so a verbose one-line `gemini-2.5-flash` grade JSON doesn't truncate. + to substantially reduce truncation of a verbose one-line + `gemini-2.5-flash` grade JSON (1024 reduces but does not fully + eliminate it at the full-fixture scale — the per-provider floors below + recommend `4096` for Gemini-heavy runs). Tightening it trims the output-token bill at the cost of truncation risk (handled by `GradeOutputError(violation_type="json_parse")` and the degraded path); see the per-provider floors below before lowering it. @@ -716,7 +719,7 @@ Issue #136 registered `OpenAIProvider` as the second ```yaml grade: provider: openai - model: gpt-4o # default judge model for the OpenAI provider; any model id the SDK accepts is allowed + model: gpt-4o # explicit override; omit `model:` to auto-resolve to the OpenAI fast default `gpt-4o-mini` (#187). Any model id the SDK accepts is allowed. # cache_ttl, max_retries_*, total_budget_seconds, thresholds — same shape as the anthropic provider ``` diff --git a/src/signalforge/grade/config.py b/src/signalforge/grade/config.py index 03aac9f3..971fb5f3 100644 --- a/src/signalforge/grade/config.py +++ b/src/signalforge/grade/config.py @@ -135,7 +135,11 @@ class GradeConfig(BaseModel): cheaper/faster model can exceed 256 tokens, and a truncated response surfaces as the wrong typed degrade. This is a **cap**, not a target; the expected JSON is still ~150 tokens, so the larger ceiling costs - nothing on the happy path while removing the truncation risk. + nothing on the happy path while substantially reducing truncation + risk. Note 1024 reduces but does not eliminate Gemini truncation at + scale — ``docs/grade-ops.md`` § per-provider floors records that + ``gemini-2.5-flash`` may still degrade on a minority of pairs at the + full-fixture scale (#158) and recommends 4096 for Gemini-heavy runs. Independent of :attr:`signalforge.draft.DraftConfig.max_output_tokens`.""" max_retries_429: int = 3 @@ -256,9 +260,14 @@ def _resolve_model_default(cls, data: Any) -> Any: :data:`signalforge.llm.providers.PROVIDER_FAST_MODELS` keyed on the requested ``provider`` (defaulting to ``"anthropic"`` to match the field default). A provider NOT in the fast-model table - is left alone — no injection — so the existing ``provider`` - field-validator raises :class:`UnknownProviderError` rather than - this masking it with a ``KeyError`` (#187 US-002 / DEC-004). + is left alone — no injection — via ``.get()`` so this never masks + an error with a ``KeyError`` (#187 US-002 / DEC-004). Two such + cases follow downstream: an *unregistered* provider is rejected by + the ``provider`` field-validator (:class:`UnknownProviderError`); + a *registered* provider absent from the fast-model table with no + explicit model is rejected by + :meth:`_validate_model_provider_compat` (which requires the + operator to set ``grade.model`` explicitly). """ if not isinstance(data, dict): return data @@ -372,18 +381,31 @@ def _validate_model_provider_compat(self) -> GradeConfig: a future SKU the table doesn't yet enumerate must not be rejected as a mismatch). * A registry-valid provider that is NOT in the prefix table - (a custom/plugin provider). Such a provider may use any model - name — the cross-vendor mismatch concept only applies among the - three known-prefix vendors, so the check does not fire when - :attr:`provider` is outside the table. + (a custom/plugin provider) *with an explicit model*. Such a + provider may use any model name — the cross-vendor mismatch + concept only applies among the three known-prefix vendors, so + the check does not fire when :attr:`provider` is outside the + table. + + A registry-valid provider absent from + :data:`signalforge.llm.providers.PROVIDER_FAST_MODELS` AND given + no explicit ``model`` reaches here with ``model is None`` (the + before-validator had no fast model to inject; the ``provider`` + field-validator passed because the provider IS registered). We + cannot guess a custom provider's model, so this fails loud rather + than letting ``None`` flow into the engine — which keeps the + post-construction "``model`` is never ``None``" invariant the + consumers assert on genuinely true (#187 QG). This is a read-only check — no mutation — so it is safe on the frozen instance. """ - # ``model`` is concrete by this point on every reachable path. model = self.model - if model is None: # pragma: no cover - defensive; resolution + provider guard cover it - return self + if model is None: + raise ValueError( + f"provider {self.provider!r} has no built-in default model; " + f"set 'grade.model' explicitly in signalforge.yml" + ) # Only the known-prefix providers participate in the mismatch check. if self.provider not in PROVIDER_SKU_PREFIXES: return self diff --git a/tests/cli/test_estimate_engine.py b/tests/cli/test_estimate_engine.py index cda21c54..cee1b8a4 100644 --- a/tests/cli/test_estimate_engine.py +++ b/tests/cli/test_estimate_engine.py @@ -289,7 +289,11 @@ def test_estimate_total_llm_usd_matches_hand_calculation( # The drafter and grader key on separate price rows post #187 US-002. draft_pricing = pricing_lookup(draft_config.model) - assert grade_config.model is not None # resolved to the fast model at config-load + # Pin the resolved fast default explicitly so a resolver regression (grade + # silently falling back to Sonnet) fails HERE rather than drifting the + # engine and the expected USD together into a passing tautology. The + # `is not None` also narrows `str | None` -> `str` for pricing_lookup. + assert grade_config.model is not None and grade_config.model == "claude-haiku-4-5" grade_pricing = pricing_lookup(grade_config.model) expected_draft = (1_000_000 / 1_000_000.0) * draft_pricing.input_per_mtok + ( 4096 / 1_000_000.0 diff --git a/tests/cli/test_generate_estimate.py b/tests/cli/test_generate_estimate.py index cf2487a7..272baccf 100644 --- a/tests/cli/test_generate_estimate.py +++ b/tests/cli/test_generate_estimate.py @@ -493,8 +493,13 @@ def test_generate_estimate_divergent_providers_fails_fast( providers_mod.register_provider(FakeNoCacheProvider()) try: project_dir = make_fake_dbt_project(tmp_path) + # An explicit model is required for a custom provider (it's not in + # PROVIDER_FAST_MODELS, so #187 won't guess its fast default); supply one + # so config LOADS and the test exercises the divergent-provider check + # (grade=fake-nocache vs draft=anthropic → tier-2 exit 2) rather than + # incidentally tripping the missing-model config-load error (tier 1). (project_dir / "signalforge.yml").write_text( - "grade:\n provider: fake-nocache\n", encoding="utf-8" + "grade:\n provider: fake-nocache\n model: fake-judge\n", encoding="utf-8" ) monkeypatch.chdir(project_dir) _install_estimate_patches(monkeypatch) diff --git a/tests/fixtures/grade/example_config.yml b/tests/fixtures/grade/example_config.yml index 699658a7..bec2fddb 100644 --- a/tests/fixtures/grade/example_config.yml +++ b/tests/fixtures/grade/example_config.yml @@ -1,8 +1,8 @@ # signalforge.yml — grade stage configuration (v0.1) grade: - model: claude-sonnet-4-6 # Anthropic model id (default) + model: claude-sonnet-4-6 # explicit override (omit to auto-resolve to the provider fast default, #187 — anthropic→claude-haiku-4-5) cache_ttl: 1h # Prompt-cache TTL ('5m' or '1h') - max_output_tokens: 256 # Per-criterion JSON response cap + max_output_tokens: 256 # explicit override (default is 1024 since #187) max_retries_429: 3 # Rate-limit retry budget max_retries_5xx: 1 max_retries_conn: 1 diff --git a/tests/grade/test_config.py b/tests/grade/test_config.py index de81dbc3..569983af 100644 --- a/tests/grade/test_config.py +++ b/tests/grade/test_config.py @@ -594,8 +594,12 @@ def test_load_grade_config_explicit_path_takes_precedence(tmp_path: Path) -> Non def test_load_grade_config_doc_example_round_trips(tmp_path: Path) -> None: - """The example YAML in docs/grade-ops.md round-trips through - load_grade_config without errors.""" + """The committed example fixture (tests/fixtures/grade/example_config.yml) + round-trips through load_grade_config without errors. The fixture pins + EXPLICIT model/token values (not the #187 resolved defaults) to exercise + the explicit-override path — `claude-sonnet-4-6` under the default + `anthropic` provider is accepted by the model↔provider compat validator + (matching `claude-` prefix).""" fixture = Path(__file__).parent.parent / "fixtures" / "grade" / "example_config.yml" target = tmp_path / "signalforge.yml" target.write_text(fixture.read_text(encoding="utf-8"), encoding="utf-8") diff --git a/tests/grade/test_provider_neutrality.py b/tests/grade/test_provider_neutrality.py index 5ede9ab0..5563e9d8 100644 --- a/tests/grade/test_provider_neutrality.py +++ b/tests/grade/test_provider_neutrality.py @@ -152,10 +152,20 @@ def test_registering_provider_is_the_only_wiring_needed(_isolate_registry: None) # Registry resolves the freshly-registered provider by name. assert provider_for(FAKE_NOCACHE_PROVIDER_NAME) is provider - # The registry-validated config str accepts it (and rejects an unknown name). - config = GradeConfig(provider=FAKE_NOCACHE_PROVIDER_NAME) + # The registry-validated config str accepts it. A custom provider is not in + # PROVIDER_FAST_MODELS, so #187 requires an explicit model (we can't guess a + # plugin provider's fast model) rather than silently defaulting it. + config = GradeConfig(provider=FAKE_NOCACHE_PROVIDER_NAME, model="fake-nocache-judge") assert config.provider == FAKE_NOCACHE_PROVIDER_NAME + # ...and a custom provider WITHOUT an explicit model fails loud at config-load + # (#187 QG — keeps the "model is never None post-construction" invariant the + # grade engine asserts on genuinely true for the plugin-provider growth path). + from pydantic import ValidationError + + with pytest.raises(ValidationError, match="no built-in default model"): + GradeConfig(provider=FAKE_NOCACHE_PROVIDER_NAME) + from signalforge.llm.errors import UnknownProviderError with pytest.raises(UnknownProviderError): diff --git a/tests/research/187-haiku-calibration/test_haiku_calibration.py b/tests/research/187-haiku-calibration/test_haiku_calibration.py index fcb76f57..522eaf31 100644 --- a/tests/research/187-haiku-calibration/test_haiku_calibration.py +++ b/tests/research/187-haiku-calibration/test_haiku_calibration.py @@ -169,9 +169,18 @@ def test_haiku_grade_concordance_vs_sonnet_baseline(tmp_path: Path) -> None: print(f" - {artifact_id} / {crit}: sonnet={sonnet_p} haiku={haiku_p}") print("================================================") - # The aggregate must be complete enough to be a meaningful gate: if - # most pairs degraded, the run did not actually measure concordance. + # The aggregate must be complete enough to be a meaningful gate. Excluding + # degraded (score=None) pairs from the denominator is correct (a pair that + # could not be evaluated is neither concordance nor discordance), but a run + # where degraded pairs DOMINATE has too small/biased a comparable set to + # trust the percentage — a high rate over a handful of survivors is not a + # real ≥85% signal. Require the comparable set to be the majority. assert comparable >= 1, "no comparable verdicts — every pair degraded" + assert comparable >= degraded, ( + f"too many degraded verdicts ({degraded}) vs comparable ({comparable}); " + "the sample/run is too noisy to trust the concordance number — " + "raise max_output_tokens or investigate the degradations before judging the gate" + ) assert rate >= _CONCORDANCE_THRESHOLD, ( f"Haiku concordance {rate:.1%} below the {_CONCORDANCE_THRESHOLD:.0%} " From 2e5bcb5cb3e7a206bbd381869f93b7c1d52ca800 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 12:58:46 -0400 Subject: [PATCH 11/15] =?UTF-8?q?SignalForge-dpy.8:=20Patterns=20&=20Memor?= =?UTF-8?q?y=20=E2=80=94=20capture=20sentinel-resolver=20+=20registry-tabl?= =?UTF-8?q?e-invariant=20+=20calibration-gate=20patterns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Distil the durable, generalisable conventions surfaced by epic #187 into the rule files so future work benefits: - .claude/rules/grade-layer.md — new "Reusable conventions distilled from #187" subsection: (1) frozen-config default-from-sibling-field resolves in @model_validator(mode="before") not mode="after" (frozen=True forbids the after-mutation); (2) the load-bearing lesson — a default looked up in a table keyed by a registry-growable field must FAIL LOUD on a registered-but-absent key rather than leak a None/sentinel, citing the #187 QG bug as the cautionary example, generalised for any per-X default table. - .claude/rules/testing-signal.md — two additions cross-referencing the e2e-gated conventions: gated calibration/concordance harness as a research-test pattern (tests/research/187-haiku-calibration/ — pinned baseline + marker + runtime skip + maintainer-run decision gate); and concordance-gate denominator hygiene (exclude degraded score=None pairs but fail loud when degraded dominates: comparable >= degraded). Docs-only; no src/ or tests/ behaviour change. Full suite green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/rules/grade-layer.md | 16 ++++++++++++++++ .claude/rules/testing-signal.md | 21 ++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/.claude/rules/grade-layer.md b/.claude/rules/grade-layer.md index c696d08a..b0b50b7d 100644 --- a/.claude/rules/grade-layer.md +++ b/.claude/rules/grade-layer.md @@ -141,6 +141,22 @@ The grade-stage block is `{ grade: { model, cache_ttl, max_output_tokens, max_re **Model↔provider compat validator (DEC-006 of #187).** A `mode="after"` validator (`_validate_model_provider_compat`) reads `signalforge.llm.providers.PROVIDER_SKU_PREFIXES` (`anthropic` → `claude-`, `openai` → `gpt-`, `gemini` → `gemini-`) and fails loud at config-load when `provider` is a known-prefix provider AND the resolved/explicit `model` carries a *different* known provider's SKU prefix (e.g. `provider: openai` with a `claude-` model). Two cases are deliberately left alone: a model whose prefix matches no known provider (forward-compat for future SKUs) and a registry-valid provider outside the prefix table (custom/plugin providers may use any model name). Both `PROVIDER_FAST_MODELS` and `PROVIDER_SKU_PREFIXES` are the single source of truth — no hardcoded SKUs or prefixes in the grade config module. Every fast-model value is an exact key in `signalforge.llm.pricing.PRICES`, so the `--estimate` path never raises on the resolved default. +## Reusable conventions distilled from #187 + +Two patterns from the #187 sentinel-default work generalise beyond the grade layer. Reach for them whenever a config field's default depends on *another* field, or whenever a default is looked up in a table keyed by a registry-growable value. + +**Frozen-config "default from a sibling field" resolves in `@model_validator(mode="before")`, never `mode="after"`.** When a config field defaults based on another field on the same model (here `model` ← `provider`), the resolution MUST inject the computed value into the raw dict in a `mode="before"` validator — NOT mutate `self.` in a `mode="after"` validator. The pipeline's config models are `frozen=True` (`extra="forbid"`), and a `mode="after"` `self.model = ...` raises (Pydantic forbids attribute assignment on a frozen instance). The before-validator runs ahead of field validation, so the injected value flows through the normal construction path and the field is concrete the moment the frozen instance exists. Copy-on-write the dict (`data = {**data, "model": resolved}`) so a caller-owned mapping is never mutated, and guard the input shape (`if not isinstance(data, dict): return data`) so an already-constructed instance passed to `model_validate` passes through untouched. This is the reusable convention for any future "this knob defaults from that knob" on a frozen `*Config` (e.g. a draft `cheap_model` ← `provider`, a prune `partition_filter` ← `scope`). + +**A default looked up in a table keyed by a registry-growable field must FAIL LOUD on a registered-but-absent key — never leak the sentinel.** This is the load-bearing #187 lesson. `model` defaults from `PROVIDER_FAST_MODELS[provider]`, but `provider` is a *registry-validated `str`, not a `Literal`* (the provider registry is a plugin point designed to grow — see `llm-drafter.md`). So three population states exist for the key field, and each needs a distinct fate: + +1. **Unregistered provider** — the `provider` field-validator already raises `UnknownProviderError`. The before-validator declines to inject (uses `.get()`, not `[]`) so it never masks that with a `KeyError`. +2. **Registered AND in the fast-model table** — the before-validator injects the fast model. Happy path. +3. **Registered BUT absent from the fast-model table** (a custom/plugin provider — the registry-growth path) — the before-validator has no value to inject, so `model` reaches the `mode="after"` validator still `None`. **This case must fail loud**, requiring an explicit `grade.model`, rather than letting `None` flow downstream. + +State 3 is the trap. The #187 Quality-Gate review caught a real bug here: the original compat validator only checked SKU-prefix mismatches and silently returned for a non-prefix provider, so a registered-but-untabled provider left `model=None` — directly contradicting the engine's `assert config.model is not None` invariant (the engine, `GradeEvent.model`, and the cost-rollup's prefix dispatch all assert/depend on a concrete model). The fix makes `_validate_model_provider_compat` raise at config-load when `model is None`, keeping the "model is never `None` post-construction" invariant *genuinely* true rather than merely usually true. + +The general rule for any per-X default table whose key field comes from a growable registry: enumerate the three population states explicitly, and make the "registered-but-absent-from-the-table" state a loud config-load failure that names the remediation (set the field explicitly). A `None`/sentinel that survives construction because the table happened not to cover a key is exactly the silent-no-op failure mode that downstream `assert`/exact-match consumers turn into a confusing far-from-the-cause crash. `PROVIDER_FAST_MODELS` / `PROVIDER_SKU_PREFIXES` live in `signalforge.llm.providers` and are the single source for per-provider fast models / SKU-prefix dispatch (shared with `cost/_rollup.py`, reusable by a future draft `--cheap`). + ## Schema-version surfaces Two exported names ship but are not consumed. **Both re-verified still-reserved on 2026-05-22 (issue #62)** — the v0.1 designs each anticipated remain intact, so neither was promoted: diff --git a/.claude/rules/testing-signal.md b/.claude/rules/testing-signal.md index 265443b1..ac491924 100644 --- a/.claude/rules/testing-signal.md +++ b/.claude/rules/testing-signal.md @@ -206,6 +206,25 @@ grep -c "rate limit" pytest-stderr.log **Markers that STAY serial — do NOT pass `-n` to these.** `cli_subprocess` and `wheel_smoke` invocations shell out to a single installed console-script / build a single wheel into shared `dist/`, so parallel workers would collide on the artefact. Run them as documented in `python-build.md` / `cli-layer.md`: `uv run pytest -m cli_subprocess --no-cov` and `uv run pytest -m wheel_smoke --no-cov`, sequential, no `-n`. +## Gated calibration / concordance harness as a research-test pattern (issue #187) + +When a behaviour change swaps a default whose *correctness* depends on live-model behaviour (the #187 grade-default flip from Sonnet to Haiku), the empirical gate is a **calibration harness**: pin a baseline sample as committed data, re-grade it live with the new default, and assert a concordance threshold. The `tests/research/187-haiku-calibration/` precedent ships: + +- A **pinned baseline + substrate** as committed data (`sonnet_baseline_sample.json` — the prior model's per-`(artifact_id, criterion_id)` pass/fail verdicts; `_substrate.py` — the candidate/model builders). Per § "Engineered determinism for LLM-driven assertions", the *only* live variable is the new model's re-grade; everything it joins against is committed bytes, so the comparison is reproducible. +- **Belt-and-suspenders gating, same as the e2e tests** (§ "Belt-and-suspenders gating"): `pytestmark = pytest.mark.anthropic` (deselected by the default `addopts` `-m 'not anthropic ...'`, so default CI never collects it) PLUS a runtime `_skip_reason()` → `pytest.skip(...)` when the live key is unset. CI can't run live API; the harness is a **maintainer-run decision gate** — the maintainer runs `pytest -m anthropic --no-cov tests/research/187-haiku-calibration/`, reads the printed breakdown, and transcribes the result into the writeup (`docs/research/187-haiku-calibration.md`). +- The harness **prints the breakdown regardless of pass/fail** (the discordance list, the rates) so a sub-threshold run still surfaces *why* it fell short — the gate's value is the diagnostic, not just the boolean. + +When a future ticket swaps a default that only live behaviour can validate (a drafter model flip, a new provider's fast default, a rubric-text change), copy this shape: committed baseline + substrate, dual gating (marker + runtime skip), maintainer-run, printed diagnostics. Mirror the e2e-gated conventions above rather than inventing a parallel gate; reuse an existing live-API marker (`anthropic` / `openai` / `gemini`) rather than minting a new one. + +## Concordance-gate denominator hygiene (issue #187) + +A concordance/agreement metric over an LLM eval computes `agreements / comparable`. Two denominator traps, both pinned in `tests/research/187-haiku-calibration/test_haiku_calibration.py`: + +1. **Exclude degraded (`score=None`) pairs from the denominator — but do NOT let a degraded-dominated run masquerade as signal.** A degraded pair (`GradingResult.score is None` — the DEC-015 conservative degrade) could not be positively evaluated, so it is neither a concordance nor a discordance; folding it into the denominator would understate agreement. *However*, a run where degraded pairs DOMINATE has too small/biased a comparable set to trust — a high rate over a handful of survivors is not a real signal. The gate therefore asserts both `comparable >= 1` (something was actually compared) AND `comparable >= degraded` (the comparable set is the majority); a run that fails the second assertion fails loud with a remediation (raise `max_output_tokens` or investigate the degradations) rather than reporting a flattering percentage over noise. +2. **Coverage assertion before the rate.** The harness asserts every `artifact_id` the engine grades is present in the committed baseline (`engine_ids - baseline_ids == set()`) BEFORE computing concordance, and never silently folds an unmatched verdict into the rate (`if key not in baseline: continue`). A silent coverage gap would otherwise inflate or deflate the denominator invisibly. + +Generalise this to any agreement/precision/recall gate computed over an LLM eval where some pairs can degrade to "not evaluated": exclude the un-evaluated pairs from the denominator, but require the evaluated subset to be the majority (or some explicit floor) so the metric is computed over enough signal to mean anything — and assert baseline coverage up-front so the denominator is the set you think it is. + ## Reference -`plans/super/1-project-scaffolding.md` — DEC-010. `plans/super/2-manifest-loader.md` — DEC-005, DEC-009, DEC-012, DEC-017. `plans/super/27-codecov-coverage.md` — DEC-001, DEC-004, DEC-009. `plans/super/10-e2e-bigquery-smoke.md` — DEC-001, DEC-002, DEC-004, DEC-008, DEC-010, DEC-022. `plans/super/157-e2e-cost-and-parallel.md` — DEC-001 … DEC-010 (parallel-safe e2e, `signalforge.llm.cost.rollup_audit_dir`, pricing-table-version parity gate). `tests/test_smoke.py`, `tests/manifest/`, `tests/fixtures/regenerate.sh`, `tests/cli/_e2e_helpers.py`, `tests/cli/test_e2e_bigquery_smoke.py`, `tests/test_contributing_e2e_enumeration_parity.py`, `src/signalforge/llm/cost/`. +`plans/super/1-project-scaffolding.md` — DEC-010. `plans/super/2-manifest-loader.md` — DEC-005, DEC-009, DEC-012, DEC-017. `plans/super/27-codecov-coverage.md` — DEC-001, DEC-004, DEC-009. `plans/super/10-e2e-bigquery-smoke.md` — DEC-001, DEC-002, DEC-004, DEC-008, DEC-010, DEC-022. `plans/super/157-e2e-cost-and-parallel.md` — DEC-001 … DEC-010 (parallel-safe e2e, `signalforge.llm.cost.rollup_audit_dir`, pricing-table-version parity gate). `plans/super/187-fast-grade-defaults.md` — DEC-005 (gated calibration/concordance harness, maintainer-run decision gate). `tests/test_smoke.py`, `tests/manifest/`, `tests/fixtures/regenerate.sh`, `tests/cli/_e2e_helpers.py`, `tests/cli/test_e2e_bigquery_smoke.py`, `tests/test_contributing_e2e_enumeration_parity.py`, `tests/research/187-haiku-calibration/`, `src/signalforge/llm/cost/`. From d5667bac7748fbb898822b82ebcb27b1c3859683 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 19:44:27 -0400 Subject: [PATCH 12/15] #187 calibration: capture a REAL Sonnet baseline from intuit_airflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the hand-authored US-005 calibration baseline with a genuine one, per request: - capture_sonnet_baseline.py drafts artifacts for the real intuit_airflow model calendar_hour (schema-only, no warehouse), freezes them to real_candidate.json, and grades them with claude-sonnet-4-6. - _substrate.py: build_model() is now the real model (deterministic); build_candidate() loads the frozen drafted artifacts. - sonnet_baseline_sample.json: 80 live Sonnet verdicts (4 degraded excluded). RESULT — the gate FAILS: Haiku concordance 81.8% / 77.0% on two runs, below the 85% DEC-005 bar. Divergence is systematic (Haiku stricter than Sonnet, concentrated on no-redundant + clarity). Per DEC-005 this points to shipping Haiku as opt-in, NOT the default. Full analysis in docs/research/187-haiku-calibration.md § Result / Disposition. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/research/187-haiku-calibration.md | 362 ++++++------ .../187-haiku-calibration/_substrate.py | 220 +++---- .../capture_sonnet_baseline.py | 175 ++++++ .../187-haiku-calibration/real_candidate.json | 116 ++++ .../sonnet_baseline_sample.json | 547 ++++++++++++++++-- .../test_haiku_calibration.py | 7 +- 6 files changed, 1062 insertions(+), 365 deletions(-) create mode 100644 tests/research/187-haiku-calibration/capture_sonnet_baseline.py create mode 100644 tests/research/187-haiku-calibration/real_candidate.json diff --git a/docs/research/187-haiku-calibration.md b/docs/research/187-haiku-calibration.md index 80285950..6143cea9 100644 --- a/docs/research/187-haiku-calibration.md +++ b/docs/research/187-haiku-calibration.md @@ -1,20 +1,24 @@ # Issue #187 — Haiku grade-default calibration gate -**Status:** harness BUILT (2026-06-02); maintainer RUN pending. The #187 -plan ships `claude-haiku-4-5` as the new grade-default SKU (US-001..US-003 — -the provider fast-model resolver now resolves `model=None` to -`claude-haiku-4-5` on the Anthropic provider, and `max_output_tokens` -defaults to `1024`). The default ships behind this empirical gate: a -maintainer runs the gated harness below with a live `ANTHROPIC_API_KEY` -and transcribes the result into the **"Result (maintainer-filled)"** -section. Until then this writeup records the substrate and method only. +**Status:** RUN COMPLETE (2026-06-02). The #187 plan ships `claude-haiku-4-5` +as the new grade-default SKU behind this empirical gate (DEC-005). The gate has +now been run against a **real Sonnet baseline drafted from a real +`intuit_airflow` model** — and **Haiku does NOT clear the ≥ 85% concordance +bar** (81.8% and 77.0% on two independent runs). Per the DEC-005 decision rule, +this points to **shipping Haiku as an opt-in fast mode, not the default**. See +§ "Result" and § "Disposition". **Companion artefacts:** -- `tests/research/187-haiku-calibration/_substrate.py` — pinned `Model` - + `CandidateSchema` + Sonnet-baseline loader. -- `tests/research/187-haiku-calibration/sonnet_baseline_sample.json` — - the curated baseline verdict sample. +- `tests/research/187-haiku-calibration/capture_sonnet_baseline.py` — one-shot + capture: drafts artifacts for the real model (schema-only) and grades them + with `claude-sonnet-4-6` to produce the baseline. Maintainer-run with a key. +- `tests/research/187-haiku-calibration/_substrate.py` — constructs the real + `Model` deterministically + loads the frozen drafted candidate + the baseline. +- `tests/research/187-haiku-calibration/real_candidate.json` — the frozen + `CandidateSchema` the production drafter (Sonnet) emitted for the model. +- `tests/research/187-haiku-calibration/sonnet_baseline_sample.json` — **live + `claude-sonnet-4-6` grades** of the frozen artifacts (NOT hand-authored). - `tests/research/187-haiku-calibration/test_haiku_calibration.py` — the `@pytest.mark.anthropic`-gated concordance gate. - `tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py` — @@ -22,140 +26,108 @@ section. Until then this writeup records the substrate and method only. ## tl;dr -- **The question:** does `claude-haiku-4-5` grade rubric artifacts - concordantly with the prior `claude-sonnet-4-6` baseline? The decision - rule is **≥ 85% per-criterion pass/fail agreement** over a pinned - sample. -- **The harness:** re-grades a hand-authored, calibration-spanning - candidate (strong / adequate / vague artifacts) with the resolved - Haiku default over the locked four-criterion `DEFAULT_RUBRIC`, joins - each `GradingResult` to a curated Sonnet baseline by - `(artifact_id, criterion_id)`, and asserts the agreement rate clears - 85%. Degraded (`score=None`) pairs are excluded from the denominator - and reported separately. +- **The question:** does `claude-haiku-4-5` grade rubric artifacts concordantly + with `claude-sonnet-4-6`? Decision rule: **≥ 85% per-criterion pass/fail + agreement** over a pinned sample (DEC-005). +- **Real substrate (recaptured on request):** the artifacts are no longer + synthetic. The production drafter drafted a `CandidateSchema` for the real + `intuit_airflow` model `plugins/dbt/models/analytical/calendar_hour.sql` + (schema-only — no warehouse), frozen to `real_candidate.json`. The baseline is + **live `claude-sonnet-4-6` grades** of those frozen artifacts. Only the Haiku + re-grade is the live variable. +- **Result — the gate FAILS:** Haiku agreement was **81.8%** (run 1) and + **77.0%** (run 2), both **below 85%**. The divergence is **systematic, not + noise**: ~80% of discordances are `sonnet=pass → haiku=fail` — **Haiku grades + the rubric stricter than Sonnet**, concentrated on the **`no-redundant`** and + **`clarity`** criteria. Haiku-as-judge would flag column rationales / + descriptions that Sonnet passes. - **A second gated check** verifies DEC-004's claim that the new - `max_output_tokens=1024` default leaves Gemini enough headroom: it - grades a deliberately verbose artifact on `gemini-2.5-flash` @ 1024 - tokens and asserts no `GradingResult` degraded to `score=None` from a - truncation. -- **Default CI is untouched:** both checks are deselected by the - existing `anthropic` / `gemini` markers in `pyproject.toml`'s - `addopts`, and skip-with-reason at runtime if collected without keys. - No live API call happens during normal validation. + `max_output_tokens=1024` default leaves Gemini enough headroom (separate; see + § "Gemini check"). +- **Default CI is untouched:** both checks are deselected by the `anthropic` / + `gemini` markers in `pyproject.toml`'s `addopts` and skip-with-reason without + keys. No live API call happens during normal validation. The concordance gate + asserting ≥ 85% now **fails when run** — that failure IS the recorded signal + that the default is mis-calibrated. ## Substrate -### Pinned candidate (the artifacts under grade) - -`_substrate.build_candidate()` returns a `CandidateSchema` for a fictional -`dim_customers` model, hand-authored to span the rubric's calibration -space (engineered determinism per `.claude/rules/testing-signal.md` -§ "Engineered determinism over snapshot normalisation"): - -| Artifact | Shape | Intended baseline signal | -|---|---|---| -| `customer_id` description | Strong, specific, sourced | passes every criterion | -| `customer_id` rationale | Strong, names downstream consumers | passes every criterion | -| `email` description | Adequate, concrete | passes clarity / consistency | -| `email` rationale | Thin ("Contact channel.") | fails clarity / rationale | -| `status` description | Deliberately vague ("A status field…") | fails clarity / rationale | -| `status` rationale | Restates the description | fails clarity / rationale / no-redundant | -| `model` description / rationale | Strong, conformed-dimension framing | passes every criterion | -| `customer_id` `not_null` / `unique` tests | Well-justified | passes every criterion | -| `status` `accepted_values` test | Well-justified closed set | passes every criterion | - -The engine's `_stable_artifact_pairs(candidate)` derives **11 artifacts** -from this shape (3 column descriptions + 3 column rationales + model -description + model rationale + 3 test rationales). The harness derives -the `artifact_id` set from the engine itself (via -`_substrate.expected_artifact_ids`) rather than hand-listing it, so the -sample can never silently drift from the formatter -(`.claude/rules/grade-layer.md` § "`_artifact_id_for` … hoist"). - -Over the locked four-criterion `DEFAULT_RUBRIC` (`clarity`, -`consistency`, `rationale`, `no-redundant`) this is **11 × 4 = 44 judge -calls** per run — a reasonable maintainer-gate budget on Haiku -(materially cheaper than the Sonnet baseline; cf. the ~$0.005/call Sonnet -figure in `docs/research/179-test-primitive-expansion-retest.md`). - -### Curated Sonnet baseline - -`sonnet_baseline_sample.json` is a **curated sample, NOT the raw #179 -Phase-B `grade.jsonl` dump**. That dump is not committed anywhere in this -repo (`find . -name grade.jsonl` finds only the drift-detector fixture at -`tests/fixtures/grade/grade_event_v1.jsonl`), and the #179 retest was run -against a private `intuit_airflow` fixture with transient `/tmp/phaseB/` -sidecars (see `docs/research/179-test-primitive-expansion-retest.md` -§ "Reproducing this retest"). Rather than depend on an un-committed dump, -the baseline here is a small representative sample of **44 hand-assigned -plausible `claude-sonnet-4-6` pass/fail verdicts** — one per -`(artifact_id, criterion_id)` pair — whose distribution tracks the -engineered candidate shape above (strong artifacts pass; vague / thin / -redundant artifacts fail on the relevant criteria). - -This makes the comparison reproducible with the **only live variable -being the Haiku re-grade**: the candidate is pinned bytes, the rubric is -locked, the baseline is committed. A concordant Haiku run reproduces the -same verdict distribution; a discordant one surfaces the specific -`(artifact, criterion)` pairs where Haiku and the baseline disagree. +### Pinned candidate (the artifacts under grade) — REAL, drafted from intuit_airflow + +`_substrate.build_model()` constructs the real `calendar_hour` hour-grain time +dimension deterministically (its SQL + four business columns — `date_id`, +`hour_of_day`, `date_hour`, `prior_year_date_hour` — are inlined so the capture +reproduces without the `intuit_airflow` repo checked out). +`_substrate.build_candidate()` loads `real_candidate.json`: the artifacts the +**production drafter** (`claude-sonnet-4-6`, schema-only) emitted for that model, +frozen by `capture_sonnet_baseline.py`. Freezing the LLM draft makes the +artifacts deterministic. + +The engine's `_stable_artifact_pairs(candidate)` derives **21 artifacts** from +the drafted candidate (4 column descriptions + 4 column rationales + model +description + model rationale + the drafted tests' rationales). The harness +derives the `artifact_id` set from the engine itself (via +`_substrate.expected_artifact_ids`) rather than hand-listing it, so the sample +can never drift from the formatter (`.claude/rules/grade-layer.md` § +"`_artifact_id_for` … hoist"). + +Over the locked four-criterion `DEFAULT_RUBRIC` (`clarity`, `consistency`, +`rationale`, `no-redundant`) this is **21 × 4 = 84 judge calls** per run. + +### Real Sonnet baseline + +`sonnet_baseline_sample.json` is now a **live `claude-sonnet-4-6` grade** of the +frozen artifacts (replacing the original hand-authored sample). Capture: +`capture_sonnet_baseline.py` drafts → freezes → grades with Sonnet → writes the +per-`(artifact_id, criterion_id)` pass/fail verdicts. Of the 84 pairs, **80 are +genuine Sonnet verdicts (63 pass / 17 fail)**; **4 pairs that Sonnet could not +grade** (`score=None`, retry exhaustion under rate limiting) are **excluded** +(see `degraded_count`) — a pair with no verdict is not a baseline. + +This makes the comparison reproducible with the **only live variable being the +Haiku re-grade**: the model is deterministic bytes, the candidate is frozen +bytes, the rubric is locked, the Sonnet baseline is committed. ### Config under test -`GradeConfig()` with all defaults — after US-002 this resolves to: - -- `model` → `claude-haiku-4-5` (provider fast-model resolver, - `provider="anthropic"`), -- `max_output_tokens` → `1024`, -- `provider` → `anthropic`. - -The harness asserts both resolved values before grading, so a regression -in the resolver fails the gate loud rather than silently measuring the -wrong SKU. +`GradeConfig()` with all defaults — after US-002 this resolves to `model → +claude-haiku-4-5`, `max_output_tokens → 1024`, `provider → anthropic`. The +harness asserts both resolved values before grading, so a resolver regression +fails the gate loud rather than silently measuring the wrong SKU. ## Method — the ≥ 85% concordance rule -1. Build the resolved Haiku-default `GradeConfig()`; assert - `model == "claude-haiku-4-5"` and `max_output_tokens == 1024`. -2. Assert the committed baseline covers every `artifact_id` the engine - will grade (no silent gaps). -3. Run `grade_artifacts(model, candidate, prune_result, config=...)` — - 44 live Haiku judge calls. -4. For each returned `GradingResult`, join to the baseline by - `(artifact_id, criterion_id)`: - - `score is None` (degraded, DEC-015 of #7) → counted as **degraded**, - excluded from the agreement denominator (neither concordant nor - discordant — the pair could not be positively evaluated). - - otherwise → **comparable**; `agreement` iff - `result.passed == baseline_passed`. -5. `agreement_rate = agreements / comparable`. Assert - `agreement_rate >= 0.85`. The harness prints the full breakdown - (model, comparable count, agreements, degraded count, rate, and each - discordance) regardless of pass/fail so a sub-threshold run still - surfaces the disagreements for the writeup. - -**Decision:** if the rate clears 85%, the Haiku default ships as planned. -If it falls short, the printed discordances name the specific -`(artifact, criterion)` shapes where Haiku diverges — those become the -follow-on (prompt-engineering, rubric-tuning, or reconsidering the -default), not silent acceptance. (Same disposition as the #179 epic's -"name the shapes that fell through" acceptance criterion.) +1. Build the resolved Haiku-default `GradeConfig()`; assert `model == + "claude-haiku-4-5"` and `max_output_tokens == 1024`. +2. Assert the committed baseline covers every `artifact_id` the engine will + grade (no silent gaps). +3. Run `grade_artifacts(...)` — 84 live Haiku judge calls. +4. For each `GradingResult`, join to the baseline by `(artifact_id, + criterion_id)`: + - `score is None` (degraded, DEC-015 of #7) → **degraded**, excluded from the + denominator (neither concordant nor discordant). + - otherwise → **comparable**; `agreement` iff `result.passed == + baseline_passed`. +5. `agreement_rate = agreements / comparable`. Assert `>= 0.85`. A guard + (`comparable >= degraded`) rejects a degraded-dominated run as too noisy to + trust. The harness prints the full breakdown (each discordance) regardless of + pass/fail. ### Running the gate (maintainer) ```bash -# From the repo root, with a live key: -ANTHROPIC_API_KEY=sk-... \ - uv run pytest -m anthropic --no-cov -s \ +# 1) (Re)capture the real Sonnet baseline — drafts + grades with Sonnet: +set -a && source /.env && set +a # provides ANTHROPIC_API_KEY +uv run python tests/research/187-haiku-calibration/capture_sonnet_baseline.py + +# 2) Run the Haiku concordance gate against that baseline: +uv run pytest -m anthropic --no-cov -s \ tests/research/187-haiku-calibration/test_haiku_calibration.py ``` -`--no-cov` is required because the gated path exercises only a fraction -of the codebase and would trip the 80% coverage floor in `addopts` -(mirrors the `pytest -m bigquery --no-cov` precedent in -`.claude/rules/testing-signal.md`). `-s` surfaces the printed concordance -breakdown. - -For the Gemini 1024-token check: +`--no-cov` is required (the gated path exercises a fraction of the codebase and +would trip the 80% coverage floor in `addopts`). `-s` surfaces the printed +breakdown. For the Gemini 1024-token check: ```bash SF_RUN_GEMINI=1 GOOGLE_API_KEY=... \ @@ -163,57 +135,101 @@ SF_RUN_GEMINI=1 GOOGLE_API_KEY=... \ tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py ``` -## Result (maintainer-filled) - -> **TODO (maintainer):** run `pytest -m anthropic --no-cov -s -> tests/research/187-haiku-calibration/test_haiku_calibration.py` with a -> live `ANTHROPIC_API_KEY` and fill in the table + verdict below from the -> printed breakdown. Then run the Gemini check and record its outcome. +## Result -**Run metadata** +**Run metadata** — Date: 2026-06-02 · SignalForge `0.6.0.dev0` · grade model +resolved `claude-haiku-4-5` · `max_output_tokens=1024` · baseline = 80 live +`claude-sonnet-4-6` verdicts (63 pass / 17 fail; 4 Sonnet-degraded excluded) of +the frozen `calendar_hour` artifacts. -- Date run: `TODO` -- SignalForge version: `TODO` (e.g. `0.x.y.dev0`) -- Grade model resolved: `claude-haiku-4-5` (assert in-test) -- `max_output_tokens`: `1024` +**Haiku-vs-Sonnet concordance** (two independent runs — Haiku grading is itself +non-deterministic): -**Haiku-vs-Sonnet concordance** - -| Metric | Value | -|---|---| -| Comparable verdicts | `TODO / 44` | -| Agreements | `TODO` | -| Degraded (`score=None`) | `TODO` | -| **Agreement rate** | `TODO %` | -| Decision threshold | 85% | -| **Verdict** | `TODO` PASS / FAIL | - -**Discordances** (if any — `(artifact_id, criterion, sonnet_passed, haiku_passed)`): - -- `TODO` (or "none — full concordance") - -**Gemini 1024-token no-truncation check** - -- Outcome: `TODO` (PASS = no `score=None` degrade / FAIL = truncation observed) -- Notes: `TODO` - -**Disposition:** `TODO` — ship the Haiku default as planned, OR name the -follow-on if concordance fell short. +| Metric | Run 1 | Run 2 | +|---|---|---| +| Comparable verdicts | 77 | 74 | +| Agreements | 63 | 57 | +| Degraded (`score=None`, Haiku side) | 3 | 6 | +| **Agreement rate** | **81.8%** | **77.0%** | +| Decision threshold | 85% | 85% | +| **Verdict** | **FAIL** | **FAIL** | + +**Both runs fall short of 85%**, and the gap is not a single-run fluke: across +two runs Haiku sits in the ~77–82% band. + +**Discordances are systematic — Haiku grades stricter than Sonnet.** Of the 17 +discordances in run 2, **14 are `sonnet=pass → haiku=fail`** (Haiku fails what +Sonnet passes) and only 3 are the reverse. They cluster by criterion: + +- **`no-redundant` (8 discordances, all sonnet=pass → haiku=fail):** + `column.date_id.rationale`, `column.hour_of_day.{description,rationale}`, + `column.date_id.description`, `column.prior_year_date_hour.rationale`, + `model.description`, `test.column.prior_year_date_hour.custom_sql`. Haiku + reads column rationales/descriptions as redundant with each other where Sonnet + tolerates them. +- **`clarity` (4, all sonnet=pass → haiku=fail):** `column.hour_of_day.rationale` + and three test rationales (`hour_of_day.custom_sql`, + `prior_year_date_hour.custom_sql`, `model.row_count_anomaly_by_period`). +- **`rationale` (2, sonnet=pass → haiku=fail):** `column.date_id.rationale`, + `column.hour_of_day.rationale`. +- **3 reverse (`sonnet=fail → haiku=pass`), all `consistency`/`no-redundant` on + test artifacts** (`column.hour_of_day.description` consistency; + `test.model.row_count_between` consistency + no-redundant) — Haiku is *more* + lenient on a couple of test-rationale shapes. + +**Interpretation.** Haiku is a stricter rubric judge than Sonnet, especially on +redundancy and clarity of short column rationales. This is a real behavioural +difference, not sampling noise — it reproduces across runs and concentrates on +two specific criteria. For SignalForge that means a Haiku default would flag +more artifacts (lower kept-rate on the grade side) than the Sonnet baseline an +operator calibrated against. + +### Gemini check + +The `gemini-2.5-flash` @ 1024-token no-truncation check +(`test_gemini_1024_no_truncation.py`) was **not run** in this session (no +`GOOGLE_API_KEY` available). It remains a separate maintainer step; DEC-004's +softened claim (1024 reduces but does not eliminate Gemini truncation at +full-fixture scale; the per-provider floors recommend 4096) already accounts for +the uncertainty. + +## Disposition + +Per the DEC-005 decision rule (**< 85% → opt-in knob, not default**), the real +calibration says **do not ship `claude-haiku-4-5` as the resolved grade +default**. Options, in order of fidelity to the data: + +1. **Recommended — make Haiku opt-in, keep Sonnet the grade default.** Revert + the Anthropic entry in `PROVIDER_FAST_MODELS` (or the grade resolution) so + `provider: anthropic` resolves to `claude-sonnet-4-6`, and document + `grade.model: claude-haiku-4-5` as the operator-opt-in fast mode. The + per-provider resolver, compat validator, and 1024 cap (US-001..US-006) all + stand — only the Anthropic *default target* changes. OpenAI/Gemini fast + defaults are unaffected by this Anthropic-specific finding (they were never + calibrated against a Sonnet baseline; they're explicit operator choices). +2. **Accept Haiku at ~80% with eyes open** — only if the maintainer judges the + ~3× speed / ~3.75× cost win worth a stricter judge that flags ~1 in 5 rubric + verdicts differently. This contradicts the gate's own rule; if taken, lower + `_CONCORDANCE_THRESHOLD` deliberately and document why here. +3. **Re-calibrate the rubric/prompt for Haiku** (v0.x follow-up) — the + discordances are concentrated on `no-redundant`/`clarity`, so a Haiku-tuned + criterion prompt might close the gap. Larger scope than #187. + +This is a maintainer product decision; the harness + this writeup record the +evidence. The gated test deliberately still asserts ≥ 85% (it fails on Haiku) so +the signal can't be silently lost. ## References - Issue **#187** — the epic this writeup gates (Haiku grade default). - US-001..US-003 ship the resolver + 1024 cap; US-005 (this) ships the - gated harness + writeup; US-006 owns the docs/rules/CHANGELOG updates. - `docs/research/179-test-primitive-expansion-retest.md` — the prior - empirical-retest writeup whose structure this mirrors; source of the - "name the shapes that fell through" disposition and the cost reference. -- `tests/grade/test_smoke_real_api.py` — the `anthropic`-gated grade - smoke whose marker + env-skip pattern the concordance gate reuses. -- `tests/grade/test_gemini_grade_live.py` — the `gemini`-gated grade - smoke whose `SF_RUN_GEMINI` + `GOOGLE_API_KEY` env gating the - truncation check reuses. -- `.claude/rules/testing-signal.md` § "End-to-end gated tests" + - § "Engineered determinism" — the gating + determinism conventions. + empirical-retest writeup whose structure this mirrors; the `intuit_airflow` + dbt project is the same source repo. +- `tests/grade/test_smoke_real_api.py` — the `anthropic`-gated grade smoke whose + marker + env-skip pattern the concordance gate reuses. +- `tests/grade/test_gemini_grade_live.py` — the `gemini`-gated smoke whose + `SF_RUN_GEMINI` + `GOOGLE_API_KEY` gating the truncation check reuses. +- `.claude/rules/testing-signal.md` § "End-to-end gated tests" + § "Engineered + determinism" + § "Gated calibration/concordance harness" — the conventions. - `.claude/rules/grade-layer.md` — the grade-layer contract (artifact-id formatter, DEC-015 degraded path, four-criterion `DEFAULT_RUBRIC`). diff --git a/tests/research/187-haiku-calibration/_substrate.py b/tests/research/187-haiku-calibration/_substrate.py index dc5d30b6..0263862f 100644 --- a/tests/research/187-haiku-calibration/_substrate.py +++ b/tests/research/187-haiku-calibration/_substrate.py @@ -1,26 +1,29 @@ """Shared substrate for the #187 Haiku-calibration harness (US-005). -Builds the pinned :class:`Model` + :class:`CandidateSchema` whose -artifacts the calibration harness re-grades, and the loader for the -committed Sonnet-baseline verdict sample. - -The substrate is *engineered-deterministic* per -:file:`.claude/rules/testing-signal.md` § "Engineered determinism over -snapshot normalisation": the candidate is hand-authored so the engine's -:func:`signalforge.grade.engine._stable_artifact_pairs` emits a fixed, -known set of ``artifact_id`` strings. The committed baseline JSON keys on -exactly those ``(artifact_id, criterion_id)`` pairs, so the only live -variable when the maintainer runs the gate is the Haiku re-grade verdict. - -The Sonnet baseline is a **curated sample**, not the raw #179 Phase-B -``grade.jsonl`` dump (which is not committed anywhere in this repo — -``find . -name grade.jsonl`` finds only the drift-detector fixture). -The ``passed`` verdicts in :data:`BASELINE_PATH` are hand-assigned -plausible Sonnet outcomes that span the rubric's calibration space -(clear/strong artifacts pass; vague/weak/redundant artifacts fail), so a -concordant Haiku run reproduces the same verdict distribution. See -:file:`docs/research/187-haiku-calibration.md` for the full provenance -note. +Provides the pinned :class:`Model` + :class:`CandidateSchema` whose artifacts +the calibration harness re-grades, and the loader for the committed Sonnet +baseline verdicts. + +**Real artifacts, real Sonnet baseline.** The original US-005 harness shipped a +hand-authored candidate + hand-assigned Sonnet verdicts. This version uses a +real model from the ``intuit_airflow`` repo +(``plugins/dbt/models/analytical/calendar_hour.sql``): + +* :func:`build_model` constructs that model **deterministically** (its SQL + + columns are inlined here, so the capture is reproducible without that repo). +* :func:`build_candidate` loads ``real_candidate.json`` — the artifacts the + production drafter (``claude-sonnet-4-6``, schema-only) emitted for the model, + frozen by :mod:`capture_sonnet_baseline`. Freezing makes the LLM-drafted + artifacts deterministic so the only live variable when the maintainer runs the + gate is the Haiku re-grade verdict. +* ``sonnet_baseline_sample.json`` holds **live** ``claude-sonnet-4-6`` grades of + those frozen artifacts (NOT hand-authored). Regenerate both files via + ``capture_sonnet_baseline.py``. + +The engine's :func:`signalforge.grade.engine._stable_artifact_pairs` derives the +``artifact_id`` set from the frozen candidate; the committed baseline keys on +exactly those ``(artifact_id, criterion_id)`` pairs. See +:file:`docs/research/187-haiku-calibration.md` for the full provenance note. """ from __future__ import annotations @@ -29,134 +32,85 @@ from pathlib import Path import signalforge as _sf -from signalforge.draft.models import ( - CandidateColumn, - CandidateSchema, - CandidateTestAcceptedValues, - CandidateTestNotNull, - CandidateTestUnique, -) +from signalforge.draft.models import CandidateSchema from signalforge.manifest.models import Column, Model from signalforge.prune.models import PruneResult -# The committed Sonnet-baseline verdict sample lives next to this module. +# The frozen drafted candidate + the committed Sonnet-baseline verdicts live +# next to this module (written by capture_sonnet_baseline.py). +CANDIDATE_PATH = Path(__file__).with_name("real_candidate.json") BASELINE_PATH = Path(__file__).with_name("sonnet_baseline_sample.json") +# Inlined verbatim from intuit_airflow plugins/dbt/models/analytical/calendar_hour.sql +# (HEAD at capture time) so the capture reproduces without that repo present. +_CALENDAR_HOUR_SQL = """with final as ( + select + cd.date_id as date_id, + h.hour_of_day, + timestampadd(hour, h.hour_of_day, cd.date_id) as date_hour, + dateadd(hour, h.hour_of_day, date(cd.prior_year_cal_dt, 'yyyymmdd')) as prior_year_date_hour + from {{ ref('calendar_date') }} cd + cross join ( + select + seq4() as hour_of_day + from table(generator(rowcount=>24))) h + where cd.date_id > '2018-01-31' +) + +{{ audit_columns('final') }}""" + def build_model() -> Model: - """Return the pinned manifest :class:`Model` the harness grades. + """Return the pinned manifest :class:`Model` — the real intuit_airflow + ``calendar_hour`` hour-grain time dimension. - Carries exactly the columns referenced by :func:`build_candidate` - so the candidate's tests resolve against real columns. + Constructed deterministically (no LLM, no warehouse) so the drafter and + grader have a stable target. Columns are the four business columns the + model's final SELECT projects (the ``audit_columns`` macro injects audit + columns downstream; those are out of scope for calibration). """ return Model( - unique_id="model.sf_calib.dim_customers", - name="dim_customers", + unique_id="model.bi.calendar_hour", + name="calendar_hour", resource_type="model", - package_name="sf_calib", - original_file_path="models/marts/dim_customers.sql", - path="marts/dim_customers.sql", - database="sf-calib-proj", - schema="main", # type: ignore[call-arg] + package_name="bi", + original_file_path="models/analytical/calendar_hour.sql", + path="analytical/calendar_hour.sql", + database="intuit-bi", + schema="analytical", # type: ignore[call-arg] columns={ - "customer_id": Column(name="customer_id"), - "email": Column(name="email"), - "status": Column(name="status"), + "date_id": Column(name="date_id", data_type="DATE"), + "hour_of_day": Column(name="hour_of_day", data_type="NUMBER"), + "date_hour": Column(name="date_hour", data_type="TIMESTAMP"), + "prior_year_date_hour": Column(name="prior_year_date_hour", data_type="TIMESTAMP"), }, - raw_code=("select customer_id, email, status from {{ ref('stg_customers') }}"), + raw_code=_CALENDAR_HOUR_SQL, ) def build_candidate() -> CandidateSchema: - """Return the pinned :class:`CandidateSchema` the harness grades. + """Return the frozen drafted :class:`CandidateSchema` the harness grades. - Hand-authored to span the rubric's calibration space: - - * ``customer_id`` — strong, specific description + rationale - (expected baseline ``passed=True`` on every criterion). - * ``email`` — adequate description, thin rationale (mixed). - * ``status`` — deliberately vague description ("a status field") - and a redundant rationale that restates the description - (expected baseline ``passed=False`` on clarity / rationale / - no-redundant). - - The engine's :func:`_stable_artifact_pairs` derives the - ``artifact_id`` set from this shape; the committed baseline keys on - exactly those ids. See :func:`expected_artifact_ids`. + Loads ``real_candidate.json`` — the artifacts the production drafter + (``claude-sonnet-4-6``, schema-only) emitted for :func:`build_model`, + frozen by :mod:`capture_sonnet_baseline`. The load is lazy (inside the + function) so importing this module never requires the file; the only caller + is the gated harness, which is deselected from the default suite. """ - return CandidateSchema( - name="dim_customers", - description=( - "Curated one-row-per-customer dimension joining stg_customers " - "with stg_customer_status to expose the current lifecycle state " - "of every customer for analytics." - ), - rationale=( - "Materialises the conformed customer dimension consumed by the " - "orders and subscriptions fact tables; resolves status at load " - "time so downstream marts never re-derive lifecycle logic." - ), - columns=( - CandidateColumn( - name="customer_id", - description=( - "Surrogate primary key uniquely identifying each " - "customer. Generated from the source system's natural " - "key via dbt_utils.generate_surrogate_key." - ), - rationale=( - "Used as the join key by every downstream fact table; " - "stability across loads is contractually required." - ), - tests=( - CandidateTestNotNull( - column="customer_id", - rationale="Primary keys must never be null.", - ), - CandidateTestUnique( - column="customer_id", - rationale=( - "One row per customer is the table's declared " - "grain; duplicates indicate a broken join." - ), - ), - ), - ), - CandidateColumn( - name="email", - description=( - "Customer's primary contact email address, lower-cased " - "and trimmed at load time." - ), - rationale="Contact channel.", - tests=(), - ), - CandidateColumn( - name="status", - description="A status field for the customer.", - rationale="Stores the status of the customer.", - tests=( - CandidateTestAcceptedValues( - column="status", - values=("active", "churned", "trialing"), - rationale=( - "The customer lifecycle is a closed set of " - "three states; any other value is a data error." - ), - ), - ), - ), - ), - tests=(), - ) + if not CANDIDATE_PATH.exists(): + raise FileNotFoundError( + f"{CANDIDATE_PATH.name} not found — run capture_sonnet_baseline.py " + "with an ANTHROPIC_API_KEY to draft + freeze the real artifacts first." + ) + return CandidateSchema.model_validate_json(CANDIDATE_PATH.read_text(encoding="utf-8")) def empty_prune_result(model: Model) -> PruneResult: """Return an empty :class:`PruneResult` linked to ``model``. - The no-redundant criterion is the only consumer of dropped tests; - the curated baseline grades the artifacts standalone, so an empty - decision tuple is correct here. + The no-redundant criterion is the only consumer of dropped tests; the + calibration grades the artifacts standalone, so an empty decision tuple is + correct here. """ return PruneResult( model_unique_id=model.unique_id, @@ -170,13 +124,12 @@ def expected_artifact_ids(candidate: CandidateSchema) -> list[str]: """Return the engine's canonical artifact_id set for ``candidate``. Thin wrapper over the engine's own - :func:`signalforge.grade.engine._stable_artifact_pairs` so the - harness never hand-enumerates ids (which would drift the moment the - formatter changes). Importing the private helper is acceptable here: - this is research-tier test code, and the alternative — duplicating - the dotted-path grammar — is exactly the drift risk - :file:`.claude/rules/grade-layer.md` § "_artifact_id_for ... hoist" - warns against. + :func:`signalforge.grade.engine._stable_artifact_pairs` so the harness never + hand-enumerates ids (which would drift the moment the formatter changes). + Importing the private helper is acceptable here: this is research-tier test + code, and the alternative — duplicating the dotted-path grammar — is exactly + the drift risk :file:`.claude/rules/grade-layer.md` § "_artifact_id_for ... + hoist" warns against. """ from signalforge.grade.engine import _stable_artifact_pairs @@ -187,7 +140,8 @@ def load_baseline() -> dict[tuple[str, str], bool]: """Load the committed Sonnet baseline as ``{(artifact_id, crit): passed}``. The on-disk shape is a JSON object with a ``"verdicts"`` array of - ``{"artifact_id", "criterion_id", "baseline_passed"}`` records. + ``{"artifact_id", "criterion_id", "baseline_passed"}`` records (extra keys + such as ``baseline_score`` are ignored). """ raw = json.loads(BASELINE_PATH.read_text(encoding="utf-8")) out: dict[tuple[str, str], bool] = {} diff --git a/tests/research/187-haiku-calibration/capture_sonnet_baseline.py b/tests/research/187-haiku-calibration/capture_sonnet_baseline.py new file mode 100644 index 00000000..57aa4297 --- /dev/null +++ b/tests/research/187-haiku-calibration/capture_sonnet_baseline.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +"""One-shot capture of a REAL ``claude-sonnet-4-6`` baseline for the #187 +Haiku-calibration gate, using a real model from the ``intuit_airflow`` repo. + +This replaces the original *hand-authored* Sonnet baseline (US-005) with a +genuine one: + +1. **Draft** candidate artifacts for the real intuit_airflow model + ``plugins/dbt/models/analytical/calendar_hour.sql`` with the production + drafter (default ``claude-sonnet-4-6``). The drafter runs **schema-only**, + so NO warehouse is contacted (DEC-012(c) of the safety layer) — the + ``intuit_airflow`` project is Snowflake, but calibration never queries it. +2. **Freeze** the drafted :class:`CandidateSchema` to ``real_candidate.json`` + so the artifacts are deterministic from here on (the LLM draft is the only + non-deterministic step; the model itself is constructed deterministically by + :func:`_substrate.build_model`). +3. **Grade** the frozen candidate with ``claude-sonnet-4-6`` and write the + per-``(artifact_id, criterion_id)`` pass/fail verdicts to + ``sonnet_baseline_sample.json``. + +The gated harness :mod:`test_haiku_calibration` then re-grades the SAME frozen +artifacts with the resolved Haiku default and measures concordance against this +real Sonnet baseline. + +Maintainer-run (needs ``ANTHROPIC_API_KEY``; ~1 draft + a few dozen Sonnet grade +calls, well under $1):: + + set -a && source /.env && set +a + uv run python tests/research/187-haiku-calibration/capture_sonnet_baseline.py + +Re-running overwrites ``real_candidate.json`` and ``sonnet_baseline_sample.json``. +The model source is ``intuit_airflow`` HEAD at capture time; the SQL is inlined +in :func:`_substrate.build_model` so the capture is reproducible without that +repo checked out. +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[2] +# The harness keeps ``_substrate`` next to this script (not an importable +# package), and ``tests`` is a namespace package (no ``__init__.py`` per the +# src-layout convention) — put both on the path. +sys.path.insert(0, str(_HERE)) +sys.path.insert(0, str(_REPO_ROOT)) + +from _substrate import ( # noqa: E402 (path insert must precede import) + BASELINE_PATH, + CANDIDATE_PATH, + build_model, + empty_prune_result, +) + +import signalforge as _sf # noqa: E402 +from signalforge.draft import draft_schema # noqa: E402 +from signalforge.draft.config import DraftConfig # noqa: E402 +from signalforge.grade import grade_artifacts # noqa: E402 +from signalforge.grade.config import GradeConfig # noqa: E402 +from signalforge.manifest.models import Manifest # noqa: E402 +from signalforge.safety.policy import SafetyPolicy # noqa: E402 + +_BASELINE_MODEL = "claude-sonnet-4-6" + + +def main() -> int: + if not os.environ.get("ANTHROPIC_API_KEY"): + print( + "ERROR: ANTHROPIC_API_KEY not set. Source your .env first:\n" + f" set -a && source {_REPO_ROOT}/.env && set +a", + file=sys.stderr, + ) + return 2 + + model = build_model() + manifest = Manifest( + metadata={"dbt_schema_version": "v12", "project_name": "bi"}, + nodes={model.unique_id: model}, + ) + + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + # Schema-only policy → the FakeAdapter is never invoked. + from tests.safety._fake_adapter import FakeAdapter + + policy = SafetyPolicy(audit_path=tmp / "audit.jsonl") + adapter = FakeAdapter() + + print(f"Drafting candidate artifacts for {model.unique_id} (schema-only, Sonnet)…") + outcome = draft_schema(model, adapter, policy, manifest, config=DraftConfig()) + candidate = outcome.candidate + + # Freeze the drafted artifacts BEFORE grading so the baseline is a + # grade of exactly what gets committed. + CANDIDATE_PATH.write_text(candidate.model_dump_json(indent=2) + "\n", encoding="utf-8") + print(f" froze {len(candidate.columns)} columns -> {CANDIDATE_PATH.name}") + + print(f"Grading the frozen candidate with {_BASELINE_MODEL}…") + report = grade_artifacts( + model, + candidate, + empty_prune_result(model), + config=GradeConfig(model=_BASELINE_MODEL), + audit_path=tmp / "grade.jsonl", + sidecar_path=tmp / "grade.json", + project_dir=tmp, + ) + + # A degraded (score=None) Sonnet grade is "could not evaluate" (retry + # exhaustion / parse failure), NOT a real "Sonnet says fail". Excluding it + # from the baseline is the honest treatment — the concordance gate then + # compares Haiku only against pairs where we actually HAVE a Sonnet verdict. + # (Each excluded pair's artifact retains its other criteria, so coverage of + # every artifact_id is preserved.) + verdicts: list[dict[str, object]] = [] + degraded = 0 + for r in report.results: + if r.score is None: + degraded += 1 + continue + verdicts.append( + { + "artifact_id": r.artifact_id, + "criterion_id": r.criterion_id, + "baseline_passed": bool(r.passed), + # Informational only; load_baseline() reads baseline_passed. + "baseline_score": r.score, + } + ) + + baseline = { + "_provenance": ( + "REAL claude-sonnet-4-6 baseline for the #187 Haiku-calibration gate " + "(US-005, recaptured on request). Artifacts were drafted by the " + "production drafter (schema-only, no warehouse) from the real " + "intuit_airflow model plugins/dbt/models/analytical/calendar_hour.sql " + "and frozen to real_candidate.json; the model itself is constructed " + "deterministically by _substrate.build_model. Verdicts below are live " + "claude-sonnet-4-6 grades of those frozen artifacts, captured via " + "capture_sonnet_baseline.py. The gated harness re-grades the SAME " + "frozen artifacts with the resolved Haiku default and measures " + "per-(artifact_id, criterion_id) pass/fail concordance against these " + "verdicts. Pairs Sonnet could not grade (score=None, e.g. retry " + "exhaustion under rate limiting) are EXCLUDED — see degraded_count. " + "This is NOT hand-authored — re-run the capture script to regenerate." + ), + "baseline_model": _BASELINE_MODEL, + "source_model": model.unique_id, + "source_repo_path": "intuit_airflow/plugins/dbt/models/analytical/calendar_hour.sql", + "signalforge_version": _sf.__version__, + "degraded_count": degraded, + "rubric_criteria": ["clarity", "consistency", "rationale", "no-redundant"], + "verdicts": verdicts, + } + BASELINE_PATH.write_text(json.dumps(baseline, indent=2) + "\n", encoding="utf-8") + + passed = sum(1 for v in verdicts if v["baseline_passed"]) + print( + f"\nCaptured {len(verdicts)} Sonnet verdicts " + f"({passed} passed / {len(verdicts) - passed} failed; " + f"{degraded} degraded pairs excluded) -> {BASELINE_PATH.name}" + ) + print(f"Frozen artifacts -> {CANDIDATE_PATH.name}") + print("Next: run the gated concordance gate with a key:") + print(" uv run pytest -m anthropic --no-cov tests/research/187-haiku-calibration/") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/research/187-haiku-calibration/real_candidate.json b/tests/research/187-haiku-calibration/real_candidate.json new file mode 100644 index 00000000..e90f82c9 --- /dev/null +++ b/tests/research/187-haiku-calibration/real_candidate.json @@ -0,0 +1,116 @@ +{ + "schema_version": 1, + "name": "calendar_hour", + "description": "A calendar dimension at hourly granularity, produced by cross-joining calendar dates with 24 sequential hours (0–23). Each row represents one hour within a given date, along with the corresponding timestamp from the prior year.", + "rationale": "Tests focus on structural integrity of this bounded dimension: uniqueness of the composite grain, completeness of all 24 hours per date, and non-nullability of every derived timestamp column.", + "columns": [ + { + "name": "date_hour", + "description": "The exact timestamp for the start of the hour, derived by adding hour_of_day hours to the date_id. This is the primary key of the model, uniquely identifying each hour slot.", + "rationale": "date_hour is the natural primary key of this hourly calendar and must be unique and non-null.", + "tests": [ + { + "type": "not_null", + "column": "date_hour", + "rationale": "Every row must have a valid timestamp; a null here would indicate a broken join or bad input date." + }, + { + "type": "unique", + "column": "date_hour", + "rationale": "Each calendar hour must appear exactly once; duplicates would indicate a cross-join or date-dimension defect." + } + ], + "meta": null + }, + { + "name": "date_id", + "description": "The calendar date to which this hour belongs, sourced from the upstream calendar_date model. Acts as the date-level grouping key.", + "rationale": "date_id must be non-null and should always produce exactly 24 hour rows per date, making it a key grain-validation anchor.", + "tests": [ + { + "type": "not_null", + "column": "date_id", + "rationale": "A null date_id would mean the hour cannot be attributed to any calendar day, breaking downstream joins." + } + ], + "meta": null + }, + { + "name": "hour_of_day", + "description": "An integer (0–23) representing the hour within the day, generated via a Snowflake sequence. Each date should have exactly one row for each of the 24 hours.", + "rationale": "hour_of_day drives the hourly fan-out and must be non-null; its 0–23 range and exactly-24-per-date invariant are the core structural checks.", + "tests": [ + { + "type": "not_null", + "column": "hour_of_day", + "rationale": "A null hour_of_day would indicate a generator failure and produce an unattributable row." + }, + { + "type": "custom_sql", + "sql": "SELECT date_id FROM {{ this }} GROUP BY date_id HAVING COUNT(*) <> 24", + "column": "hour_of_day", + "rationale": "Every date must have exactly 24 hour rows; any deviation signals a cross-join or generator defect." + }, + { + "type": "custom_sql", + "sql": "SELECT * FROM {{ this }} WHERE hour_of_day < 0 OR hour_of_day > 23", + "column": "hour_of_day", + "rationale": "hour_of_day must be within the valid 0–23 range; values outside this window indicate a broken sequence generator." + } + ], + "meta": null + }, + { + "name": "prior_year_date_hour", + "description": "The timestamp corresponding to the same hour in the prior calendar year, used for year-over-year comparisons. Derived by offsetting the prior year calendar date by hour_of_day hours.", + "rationale": "prior_year_date_hour must be non-null and should always fall exactly one year before date_hour, making structural drift detectable via a simple gap check.", + "tests": [ + { + "type": "not_null", + "column": "prior_year_date_hour", + "rationale": "A null prior-year timestamp would silently break any year-over-year aggregation that joins on this column." + }, + { + "type": "custom_sql", + "sql": "SELECT * FROM {{ this }} WHERE DATEDIFF('day', prior_year_date_hour, date_hour) NOT BETWEEN 364 AND 366", + "column": "prior_year_date_hour", + "rationale": "The prior_year_date_hour should always be approximately 365 days before date_hour; values outside this band indicate a mapping error in the upstream calendar_date model." + } + ], + "meta": null + } + ], + "tests": [ + { + "type": "unique_combination", + "column": null, + "columns": [ + "date_id", + "hour_of_day" + ], + "where": null, + "rationale": "The grain of this model is one row per (date, hour) combination; duplicate pairs would indicate a defect in the cross-join or upstream date fan-out." + }, + { + "type": "row_count_between", + "column": null, + "minimum": 24, + "maximum": null, + "where": null, + "rationale": "The model covers all dates after 2018-01-31 with 24 rows each, so there must be at least 24 rows; an empty result would indicate a total pipeline failure." + }, + { + "type": "row_count_anomaly_by_period", + "column": null, + "date_column": "date_hour", + "period": "day", + "lookback_periods": 28, + "method": "mad", + "seasonality": "none", + "threshold": 3.0, + "min_samples_per_bucket": 3, + "where": null, + "rationale": "This incremental-style hourly dimension should load exactly 24 rows per calendar day; anomaly detection on date_hour will catch gaps or duplications introduced by upstream calendar_date changes." + } + ] +} diff --git a/tests/research/187-haiku-calibration/sonnet_baseline_sample.json b/tests/research/187-haiku-calibration/sonnet_baseline_sample.json index 68e05dfe..8ceee4d6 100644 --- a/tests/research/187-haiku-calibration/sonnet_baseline_sample.json +++ b/tests/research/187-haiku-calibration/sonnet_baseline_sample.json @@ -1,61 +1,496 @@ { - "_provenance": "Curated Sonnet-baseline verdict sample for the #187 Haiku-calibration gate (US-005). NOT the raw #179 Phase-B grade.jsonl dump (which is not committed in this repo). Verdicts are hand-assigned plausible claude-sonnet-4-6 outcomes spanning the four DEFAULT_RUBRIC criteria (clarity, consistency, rationale, no-redundant). Engineered determinism per testing-signal.md: strong/specific artifacts pass; vague/thin/redundant artifacts fail. The artifact_id set is derived from tests/research/187-haiku-calibration/_substrate.build_candidate() via the grade engine's _stable_artifact_pairs; see docs/research/187-haiku-calibration.md for full provenance.", + "_provenance": "REAL claude-sonnet-4-6 baseline for the #187 Haiku-calibration gate (US-005, recaptured on request). Artifacts were drafted by the production drafter (schema-only, no warehouse) from the real intuit_airflow model plugins/dbt/models/analytical/calendar_hour.sql and frozen to real_candidate.json; the model itself is constructed deterministically by _substrate.build_model. Verdicts below are live claude-sonnet-4-6 grades of those frozen artifacts, captured via capture_sonnet_baseline.py. The gated harness re-grades the SAME frozen artifacts with the resolved Haiku default and measures per-(artifact_id, criterion_id) pass/fail concordance against these verdicts. Pairs Sonnet could not grade (score=None, e.g. retry exhaustion under rate limiting) are EXCLUDED \u2014 see degraded_count. This is NOT hand-authored \u2014 re-run the capture script to regenerate.", "baseline_model": "claude-sonnet-4-6", - "rubric_criteria": ["clarity", "consistency", "rationale", "no-redundant"], + "source_model": "model.bi.calendar_hour", + "source_repo_path": "intuit_airflow/plugins/dbt/models/analytical/calendar_hour.sql", + "signalforge_version": "0.6.0.dev0", + "degraded_count": 4, + "rubric_criteria": [ + "clarity", + "consistency", + "rationale", + "no-redundant" + ], "verdicts": [ - {"artifact_id": "column.customer_id.description", "criterion_id": "clarity", "baseline_passed": true}, - {"artifact_id": "column.customer_id.description", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "column.customer_id.description", "criterion_id": "rationale", "baseline_passed": true}, - {"artifact_id": "column.customer_id.description", "criterion_id": "no-redundant", "baseline_passed": true}, - - {"artifact_id": "column.email.description", "criterion_id": "clarity", "baseline_passed": true}, - {"artifact_id": "column.email.description", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "column.email.description", "criterion_id": "rationale", "baseline_passed": true}, - {"artifact_id": "column.email.description", "criterion_id": "no-redundant", "baseline_passed": true}, - - {"artifact_id": "column.status.description", "criterion_id": "clarity", "baseline_passed": false}, - {"artifact_id": "column.status.description", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "column.status.description", "criterion_id": "rationale", "baseline_passed": false}, - {"artifact_id": "column.status.description", "criterion_id": "no-redundant", "baseline_passed": true}, - - {"artifact_id": "column.customer_id.rationale", "criterion_id": "clarity", "baseline_passed": true}, - {"artifact_id": "column.customer_id.rationale", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "column.customer_id.rationale", "criterion_id": "rationale", "baseline_passed": true}, - {"artifact_id": "column.customer_id.rationale", "criterion_id": "no-redundant", "baseline_passed": true}, - - {"artifact_id": "column.email.rationale", "criterion_id": "clarity", "baseline_passed": false}, - {"artifact_id": "column.email.rationale", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "column.email.rationale", "criterion_id": "rationale", "baseline_passed": false}, - {"artifact_id": "column.email.rationale", "criterion_id": "no-redundant", "baseline_passed": true}, - - {"artifact_id": "column.status.rationale", "criterion_id": "clarity", "baseline_passed": false}, - {"artifact_id": "column.status.rationale", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "column.status.rationale", "criterion_id": "rationale", "baseline_passed": false}, - {"artifact_id": "column.status.rationale", "criterion_id": "no-redundant", "baseline_passed": false}, - - {"artifact_id": "model.description", "criterion_id": "clarity", "baseline_passed": true}, - {"artifact_id": "model.description", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "model.description", "criterion_id": "rationale", "baseline_passed": true}, - {"artifact_id": "model.description", "criterion_id": "no-redundant", "baseline_passed": true}, - - {"artifact_id": "model.rationale", "criterion_id": "clarity", "baseline_passed": true}, - {"artifact_id": "model.rationale", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "model.rationale", "criterion_id": "rationale", "baseline_passed": true}, - {"artifact_id": "model.rationale", "criterion_id": "no-redundant", "baseline_passed": true}, - - {"artifact_id": "test.column.customer_id.not_null", "criterion_id": "clarity", "baseline_passed": true}, - {"artifact_id": "test.column.customer_id.not_null", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "test.column.customer_id.not_null", "criterion_id": "rationale", "baseline_passed": true}, - {"artifact_id": "test.column.customer_id.not_null", "criterion_id": "no-redundant", "baseline_passed": true}, - - {"artifact_id": "test.column.customer_id.unique", "criterion_id": "clarity", "baseline_passed": true}, - {"artifact_id": "test.column.customer_id.unique", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "test.column.customer_id.unique", "criterion_id": "rationale", "baseline_passed": true}, - {"artifact_id": "test.column.customer_id.unique", "criterion_id": "no-redundant", "baseline_passed": true}, - - {"artifact_id": "test.column.status.accepted_values", "criterion_id": "clarity", "baseline_passed": true}, - {"artifact_id": "test.column.status.accepted_values", "criterion_id": "consistency", "baseline_passed": true}, - {"artifact_id": "test.column.status.accepted_values", "criterion_id": "rationale", "baseline_passed": true}, - {"artifact_id": "test.column.status.accepted_values", "criterion_id": "no-redundant", "baseline_passed": true} + { + "artifact_id": "column.date_hour.description", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.72 + }, + { + "artifact_id": "column.date_id.description", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.62 + }, + { + "artifact_id": "column.hour_of_day.description", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.72 + }, + { + "artifact_id": "column.prior_year_date_hour.description", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.78 + }, + { + "artifact_id": "column.date_hour.rationale", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.55 + }, + { + "artifact_id": "column.date_id.rationale", + "criterion_id": "clarity", + "baseline_passed": false, + "baseline_score": 0.45 + }, + { + "artifact_id": "column.hour_of_day.rationale", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.62 + }, + { + "artifact_id": "column.prior_year_date_hour.rationale", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.72 + }, + { + "artifact_id": "model.description", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.72 + }, + { + "artifact_id": "model.rationale", + "criterion_id": "clarity", + "baseline_passed": false, + "baseline_score": 0.35 + }, + { + "artifact_id": "test.column.date_hour.not_null", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.55 + }, + { + "artifact_id": "test.column.date_hour.unique", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "test.column.date_id.not_null", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.72 + }, + { + "artifact_id": "test.column.hour_of_day.not_null", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.72 + }, + { + "artifact_id": "test.column.hour_of_day.custom_sql.22db0141", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.72 + }, + { + "artifact_id": "test.column.hour_of_day.custom_sql.e7f5b321", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "test.column.prior_year_date_hour.not_null", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.55 + }, + { + "artifact_id": "test.column.prior_year_date_hour.custom_sql", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.72 + }, + { + "artifact_id": "test.model.unique_combination", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "test.model.row_count_between", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.55 + }, + { + "artifact_id": "test.model.row_count_anomaly_by_period", + "criterion_id": "clarity", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "column.date_hour.description", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.55 + }, + { + "artifact_id": "column.date_id.description", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.6 + }, + { + "artifact_id": "column.hour_of_day.description", + "criterion_id": "consistency", + "baseline_passed": false, + "baseline_score": 0.55 + }, + { + "artifact_id": "column.prior_year_date_hour.description", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.7 + }, + { + "artifact_id": "column.date_hour.rationale", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.8 + }, + { + "artifact_id": "column.date_id.rationale", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.6 + }, + { + "artifact_id": "column.hour_of_day.rationale", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.75 + }, + { + "artifact_id": "column.prior_year_date_hour.rationale", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "model.description", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.5 + }, + { + "artifact_id": "model.rationale", + "criterion_id": "consistency", + "baseline_passed": false, + "baseline_score": 0.3 + }, + { + "artifact_id": "test.column.date_hour.not_null", + "criterion_id": "consistency", + "baseline_passed": false, + "baseline_score": 0.55 + }, + { + "artifact_id": "test.column.date_hour.unique", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.75 + }, + { + "artifact_id": "test.column.date_id.not_null", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.8 + }, + { + "artifact_id": "test.column.hour_of_day.not_null", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.7 + }, + { + "artifact_id": "test.column.hour_of_day.custom_sql.22db0141", + "criterion_id": "consistency", + "baseline_passed": false, + "baseline_score": 0.4 + }, + { + "artifact_id": "test.column.hour_of_day.custom_sql.e7f5b321", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.7 + }, + { + "artifact_id": "test.column.prior_year_date_hour.not_null", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.55 + }, + { + "artifact_id": "test.column.prior_year_date_hour.custom_sql", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.model.unique_combination", + "criterion_id": "consistency", + "baseline_passed": true, + "baseline_score": 0.55 + }, + { + "artifact_id": "test.model.row_count_between", + "criterion_id": "consistency", + "baseline_passed": false, + "baseline_score": 0.4 + }, + { + "artifact_id": "test.model.row_count_anomaly_by_period", + "criterion_id": "consistency", + "baseline_passed": false, + "baseline_score": 0.55 + }, + { + "artifact_id": "column.date_hour.description", + "criterion_id": "rationale", + "baseline_passed": false, + "baseline_score": 0.1 + }, + { + "artifact_id": "column.date_id.description", + "criterion_id": "rationale", + "baseline_passed": false, + "baseline_score": 0.1 + }, + { + "artifact_id": "column.hour_of_day.description", + "criterion_id": "rationale", + "baseline_passed": false, + "baseline_score": 0.1 + }, + { + "artifact_id": "column.prior_year_date_hour.description", + "criterion_id": "rationale", + "baseline_passed": false, + "baseline_score": 0.1 + }, + { + "artifact_id": "column.date_hour.rationale", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.55 + }, + { + "artifact_id": "column.date_id.rationale", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.72 + }, + { + "artifact_id": "column.hour_of_day.rationale", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "column.prior_year_date_hour.rationale", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "model.description", + "criterion_id": "rationale", + "baseline_passed": false, + "baseline_score": 0.1 + }, + { + "artifact_id": "model.rationale", + "criterion_id": "rationale", + "baseline_passed": false, + "baseline_score": 0.2 + }, + { + "artifact_id": "test.column.date_hour.not_null", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "test.column.date_hour.unique", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.column.date_id.not_null", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.column.hour_of_day.not_null", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.column.hour_of_day.custom_sql.22db0141", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.column.hour_of_day.custom_sql.e7f5b321", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.column.prior_year_date_hour.not_null", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.column.prior_year_date_hour.custom_sql", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "test.model.unique_combination", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "test.model.row_count_anomaly_by_period", + "criterion_id": "rationale", + "baseline_passed": true, + "baseline_score": 0.82 + }, + { + "artifact_id": "column.date_id.description", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.5 + }, + { + "artifact_id": "column.hour_of_day.description", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.5 + }, + { + "artifact_id": "column.date_hour.rationale", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.75 + }, + { + "artifact_id": "column.date_id.rationale", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.5 + }, + { + "artifact_id": "column.hour_of_day.rationale", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.6 + }, + { + "artifact_id": "column.prior_year_date_hour.rationale", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.55 + }, + { + "artifact_id": "model.description", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.5 + }, + { + "artifact_id": "model.rationale", + "criterion_id": "no-redundant", + "baseline_passed": false, + "baseline_score": 0.4 + }, + { + "artifact_id": "test.column.date_hour.not_null", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.75 + }, + { + "artifact_id": "test.column.date_hour.unique", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.8 + }, + { + "artifact_id": "test.column.date_id.not_null", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.column.hour_of_day.custom_sql.22db0141", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.column.hour_of_day.custom_sql.e7f5b321", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.75 + }, + { + "artifact_id": "test.column.prior_year_date_hour.not_null", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.column.prior_year_date_hour.custom_sql", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.6 + }, + { + "artifact_id": "test.model.unique_combination", + "criterion_id": "no-redundant", + "baseline_passed": true, + "baseline_score": 0.85 + }, + { + "artifact_id": "test.model.row_count_between", + "criterion_id": "no-redundant", + "baseline_passed": false, + "baseline_score": 0.35 + }, + { + "artifact_id": "test.model.row_count_anomaly_by_period", + "criterion_id": "no-redundant", + "baseline_passed": false, + "baseline_score": 0.45 + } ] } diff --git a/tests/research/187-haiku-calibration/test_haiku_calibration.py b/tests/research/187-haiku-calibration/test_haiku_calibration.py index 522eaf31..b100dc7d 100644 --- a/tests/research/187-haiku-calibration/test_haiku_calibration.py +++ b/tests/research/187-haiku-calibration/test_haiku_calibration.py @@ -28,9 +28,10 @@ blank) — so a maintainer who runs ``pytest -m anthropic`` without a key sees a clean skip-with-reason, not a noisy auth failure. -The substrate (pinned candidate + curated Sonnet baseline) lives in -:mod:`tests.research._substrate`; see -:file:`docs/research/187-haiku-calibration.md` for provenance. +The substrate (real model + frozen drafted candidate + live Sonnet baseline) +lives in :mod:`tests.research._substrate`; see +:file:`docs/research/187-haiku-calibration.md` for provenance and the recorded +result (Haiku fell short of the 85% bar — 81.8% / 77.0% on two runs). """ from __future__ import annotations From 7af6df8815722fde93683ca670eac4cb538ccab6 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 20:18:25 -0400 Subject: [PATCH 13/15] #187: keep Sonnet the grade default, Haiku opt-in (calibration result) The empirical calibration (real Sonnet baseline from intuit_airflow) found claude-haiku-4-5 grades stricter than Sonnet (~77-82% concordance, below the 85% DEC-005 bar). Per the decision rule, Haiku does NOT ship as the default. - Rename PROVIDER_FAST_MODELS -> PROVIDER_DEFAULT_MODELS (Sonnet isn't "fast"); anthropic -> claude-sonnet-4-6 (was claude-haiku-4-5). OpenAI/Gemini fast defaults unchanged (gpt-4o-mini / gemini-2.5-flash). - GradeConfig() now resolves to claude-sonnet-4-6; claude-haiku-4-5 is the documented opt-in (grade.model: claude-haiku-4-5). - Revert estimate goldens + grade_event fixture + hand-calc to Sonnet (estimate is independent of max_output_tokens, so origin/dev values apply). - Kept: the per-provider resolver, model<->provider compat validator, the 1024 max_output_tokens cap (still justified by the gemini-2.5-flash default). - Calibration gate now selects Haiku explicitly and still asserts >=85% (fails) as the durable record that Haiku is the stricter opt-in. - Docs/rules/CHANGELOG/writeup updated across all surfaces. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/rules/grade-layer.md | 10 +-- .claude/rules/llm-drafter.md | 2 +- CHANGELOG.md | 2 +- docs/draft-ops.md | 5 +- docs/grade-ops.md | 33 +++++----- docs/llm-providers-ops.md | 35 ++++++----- docs/research/187-haiku-calibration.md | 53 +++++++++------- src/signalforge/grade/config.py | 48 ++++++++------- src/signalforge/llm/providers.py | 18 ++++-- tests/cli/test_estimate_engine.py | 61 +++++++------------ tests/cli/test_generate_estimate.py | 2 +- .../anthropic_byte_identity_golden.txt | 14 ++--- tests/fixtures/grade/example_config.yml | 2 +- tests/fixtures/grade/grade_event_v1.jsonl | 2 +- tests/grade/test_config.py | 15 +++-- tests/grade/test_provider_neutrality.py | 2 +- tests/llm/test_providers.py | 38 ++++++------ .../test_haiku_calibration.py | 17 +++--- 18 files changed, 188 insertions(+), 171 deletions(-) diff --git a/.claude/rules/grade-layer.md b/.claude/rules/grade-layer.md index b0b50b7d..620e9450 100644 --- a/.claude/rules/grade-layer.md +++ b/.claude/rules/grade-layer.md @@ -65,7 +65,7 @@ For 4 default criteria × ~12 artifacts per typical model = ~48 calls per `grade The cached prompt block is the rubric criterion list (constant per run); the dynamic block is the per-pair `...` envelope. Anthropic prompt-cache TTL defaults to `"1h"` for the grader (vs. drafter's `"5m"`). -**Tolerant JSON extraction (issue #144).** `parse_grade_response` routes the response through `signalforge._common.json_payload.extract_json_payload` (after `_strip_code_fence`) so a judge that narrates a prose preamble before the `{` still parses. The Anthropic judge models (the `claude-haiku-4-5` default per #187, or an explicit `claude-sonnet-4-6`) do NOT support an assistant-turn prefill (API 400), so the parser is the only JSON-only guardrail. Same decode rule as the drafter — decode at the first structural char (`{` or `[`) only, return unchanged on failure — see `llm-drafter.md` § "Tolerant JSON extraction"; a no-JSON response still routes to `GradeOutputError(violation_type="json_parse")` and the conservative degrade. +**Tolerant JSON extraction (issue #144).** `parse_grade_response` routes the response through `signalforge._common.json_payload.extract_json_payload` (after `_strip_code_fence`) so a judge that narrates a prose preamble before the `{` still parses. The Anthropic judge models (the `claude-sonnet-4-6` default, or the `claude-haiku-4-5` opt-in per #187) do NOT support an assistant-turn prefill (API 400), so the parser is the only JSON-only guardrail. Same decode rule as the drafter — decode at the first structural char (`{` or `[`) only, return unchanged on failure — see `llm-drafter.md` § "Tolerant JSON extraction"; a no-JSON response still routes to `GradeOutputError(violation_type="json_parse")` and the conservative degrade. ## Reproducibility hash fields on every GradeEvent (DEC-010, DEC-019) @@ -136,10 +136,10 @@ The grade-stage block is `{ grade: { model, cache_ttl, max_output_tokens, max_re `GradeConfig`'s locked defaults (DEC-023..DEC-027) carry two #187 changes: -- **`model` default is now a per-provider sentinel.** The field defaults to `None`; a `mode="before"` model-validator (`_resolve_model_default`) resolves the sentinel at config-load to the calling provider's fast model from `signalforge.llm.providers.PROVIDER_FAST_MODELS` — `anthropic` → `claude-haiku-4-5`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. (Pre-#187 the default was the bare `claude-sonnet-4-6` literal regardless of provider.) An explicit `model:` is honoured verbatim; after construction the field is always a concrete non-empty string, never `None`. The resolver runs `before` because `GradeConfig` is `frozen=True` and a `mode="after"` mutation would raise. A provider NOT in the fast-model table is left un-injected so the `provider` field-validator surfaces `UnknownProviderError` rather than a masking `KeyError`. +- **`model` default is now a per-provider sentinel.** The field defaults to `None`; a `mode="before"` model-validator (`_resolve_model_default`) resolves the sentinel at config-load to the calling provider's default judge model from `signalforge.llm.providers.PROVIDER_DEFAULT_MODELS` — `anthropic` → `claude-sonnet-4-6`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. **Anthropic stays on Sonnet:** the #187 calibration gate (`tests/research/187-haiku-calibration/`) found `claude-haiku-4-5` grades the rubric stricter than Sonnet (~77–82% concordance on a real `intuit_airflow`-drafted sample, below the 85% DEC-005 bar), so Haiku is an explicit opt-in (`grade.model: claude-haiku-4-5`), not the default. OpenAI/Gemini default to their fast judges (explicit operator choices, never calibrated). (Pre-#187 the default was the bare `claude-sonnet-4-6` literal regardless of provider; the per-provider resolver is the #187 improvement — `provider: openai`/`gemini` now Just Works without forcing an explicit model.) An explicit `model:` is honoured verbatim; after construction the field is always a concrete non-empty string, never `None`. The resolver runs `before` because `GradeConfig` is `frozen=True` and a `mode="after"` mutation would raise. A provider registered but absent from the default-model table with no explicit model fails loud at config-load (the compat validator requires an explicit `grade.model`) — a custom/plugin provider's model can't be guessed, and letting `None` reach the engine's `assert model is not None` would be a latent crash. - **`max_output_tokens` default raised 256 → 1024** so a verbose one-line `gemini-2.5-flash` grade JSON is not truncated (a truncation would surface as the wrong typed degrade — `GradeOutputError` instead of `GradeLLMError`). Still a cap, not a target; the expected JSON is ~150 tokens, so the larger ceiling costs nothing on the happy path. -**Model↔provider compat validator (DEC-006 of #187).** A `mode="after"` validator (`_validate_model_provider_compat`) reads `signalforge.llm.providers.PROVIDER_SKU_PREFIXES` (`anthropic` → `claude-`, `openai` → `gpt-`, `gemini` → `gemini-`) and fails loud at config-load when `provider` is a known-prefix provider AND the resolved/explicit `model` carries a *different* known provider's SKU prefix (e.g. `provider: openai` with a `claude-` model). Two cases are deliberately left alone: a model whose prefix matches no known provider (forward-compat for future SKUs) and a registry-valid provider outside the prefix table (custom/plugin providers may use any model name). Both `PROVIDER_FAST_MODELS` and `PROVIDER_SKU_PREFIXES` are the single source of truth — no hardcoded SKUs or prefixes in the grade config module. Every fast-model value is an exact key in `signalforge.llm.pricing.PRICES`, so the `--estimate` path never raises on the resolved default. +**Model↔provider compat validator (DEC-006 of #187).** A `mode="after"` validator (`_validate_model_provider_compat`) reads `signalforge.llm.providers.PROVIDER_SKU_PREFIXES` (`anthropic` → `claude-`, `openai` → `gpt-`, `gemini` → `gemini-`) and fails loud at config-load when `provider` is a known-prefix provider AND the resolved/explicit `model` carries a *different* known provider's SKU prefix (e.g. `provider: openai` with a `claude-` model). Two cases are deliberately left alone: a model whose prefix matches no known provider (forward-compat for future SKUs) and a registry-valid provider outside the prefix table (custom/plugin providers may use any model name). Both `PROVIDER_DEFAULT_MODELS` and `PROVIDER_SKU_PREFIXES` are the single source of truth — no hardcoded SKUs or prefixes in the grade config module. Every fast-model value is an exact key in `signalforge.llm.pricing.PRICES`, so the `--estimate` path never raises on the resolved default. ## Reusable conventions distilled from #187 @@ -147,7 +147,7 @@ Two patterns from the #187 sentinel-default work generalise beyond the grade lay **Frozen-config "default from a sibling field" resolves in `@model_validator(mode="before")`, never `mode="after"`.** When a config field defaults based on another field on the same model (here `model` ← `provider`), the resolution MUST inject the computed value into the raw dict in a `mode="before"` validator — NOT mutate `self.` in a `mode="after"` validator. The pipeline's config models are `frozen=True` (`extra="forbid"`), and a `mode="after"` `self.model = ...` raises (Pydantic forbids attribute assignment on a frozen instance). The before-validator runs ahead of field validation, so the injected value flows through the normal construction path and the field is concrete the moment the frozen instance exists. Copy-on-write the dict (`data = {**data, "model": resolved}`) so a caller-owned mapping is never mutated, and guard the input shape (`if not isinstance(data, dict): return data`) so an already-constructed instance passed to `model_validate` passes through untouched. This is the reusable convention for any future "this knob defaults from that knob" on a frozen `*Config` (e.g. a draft `cheap_model` ← `provider`, a prune `partition_filter` ← `scope`). -**A default looked up in a table keyed by a registry-growable field must FAIL LOUD on a registered-but-absent key — never leak the sentinel.** This is the load-bearing #187 lesson. `model` defaults from `PROVIDER_FAST_MODELS[provider]`, but `provider` is a *registry-validated `str`, not a `Literal`* (the provider registry is a plugin point designed to grow — see `llm-drafter.md`). So three population states exist for the key field, and each needs a distinct fate: +**A default looked up in a table keyed by a registry-growable field must FAIL LOUD on a registered-but-absent key — never leak the sentinel.** This is the load-bearing #187 lesson. `model` defaults from `PROVIDER_DEFAULT_MODELS[provider]`, but `provider` is a *registry-validated `str`, not a `Literal`* (the provider registry is a plugin point designed to grow — see `llm-drafter.md`). So three population states exist for the key field, and each needs a distinct fate: 1. **Unregistered provider** — the `provider` field-validator already raises `UnknownProviderError`. The before-validator declines to inject (uses `.get()`, not `[]`) so it never masks that with a `KeyError`. 2. **Registered AND in the fast-model table** — the before-validator injects the fast model. Happy path. @@ -155,7 +155,7 @@ Two patterns from the #187 sentinel-default work generalise beyond the grade lay State 3 is the trap. The #187 Quality-Gate review caught a real bug here: the original compat validator only checked SKU-prefix mismatches and silently returned for a non-prefix provider, so a registered-but-untabled provider left `model=None` — directly contradicting the engine's `assert config.model is not None` invariant (the engine, `GradeEvent.model`, and the cost-rollup's prefix dispatch all assert/depend on a concrete model). The fix makes `_validate_model_provider_compat` raise at config-load when `model is None`, keeping the "model is never `None` post-construction" invariant *genuinely* true rather than merely usually true. -The general rule for any per-X default table whose key field comes from a growable registry: enumerate the three population states explicitly, and make the "registered-but-absent-from-the-table" state a loud config-load failure that names the remediation (set the field explicitly). A `None`/sentinel that survives construction because the table happened not to cover a key is exactly the silent-no-op failure mode that downstream `assert`/exact-match consumers turn into a confusing far-from-the-cause crash. `PROVIDER_FAST_MODELS` / `PROVIDER_SKU_PREFIXES` live in `signalforge.llm.providers` and are the single source for per-provider fast models / SKU-prefix dispatch (shared with `cost/_rollup.py`, reusable by a future draft `--cheap`). +The general rule for any per-X default table whose key field comes from a growable registry: enumerate the three population states explicitly, and make the "registered-but-absent-from-the-table" state a loud config-load failure that names the remediation (set the field explicitly). A `None`/sentinel that survives construction because the table happened not to cover a key is exactly the silent-no-op failure mode that downstream `assert`/exact-match consumers turn into a confusing far-from-the-cause crash. `PROVIDER_DEFAULT_MODELS` / `PROVIDER_SKU_PREFIXES` live in `signalforge.llm.providers` and are the single source for per-provider fast models / SKU-prefix dispatch (shared with `cost/_rollup.py`, reusable by a future draft `--cheap`). ## Schema-version surfaces diff --git a/.claude/rules/llm-drafter.md b/.claude/rules/llm-drafter.md index bc18bbdd..525ce995 100644 --- a/.claude/rules/llm-drafter.md +++ b/.claude/rules/llm-drafter.md @@ -18,7 +18,7 @@ Every `# pyright: ignore[...]` and `# type: ignore[...]` comment for the Anthrop - **Neutral value objects:** `UsageMetrics` + the `ExceptionCategory` enum (`AUTH`, `RATE_LIMIT`, `SERVER_ERROR`, `CONNECTION`, `NO_RETRY`) keep the orchestrator off vendor-shaped dicts. - **Capability-gated behaviour (DEC-008):** `supports_prompt_caching=False` ⇒ no `cache_control` marker, no `extended-cache-ttl` beta header, 0 cache tokens, no dual-zero anomaly WARNING. `supports_token_count=False` ⇒ skip the pre-send count gate (no pre-send `LLMCacheTooLargeError`). Anthropic sets both `True`, so its emitted bytes/control flow are unchanged — the byte-identity gate (fixtures + prompt-cache snapshot + drift detectors) is the regression guard. - **`provider` config field (DEC-007):** `DraftConfig.provider` (`llm:` block) and `GradeConfig.provider` (`grade:` block), both registry-validated `str` defaulting to `"anthropic"` — **deliberately NOT a `Literal`** (a registry is a plugin point that grows; #136/#137 register a provider instead of editing a Literal in two configs). The validator raises `UnknownProviderError` (an `LLMError`, so Pydantic v2 does NOT wrap it into `ValidationError` — it propagates raw with the available-keys remediation). -- **Provider→string mappings (#187 US-001):** `PROVIDER_FAST_MODELS` and `PROVIDER_SKU_PREFIXES` also live in `signalforge.llm.providers` and are the single source of truth for two cross-cutting per-provider facts. `PROVIDER_FAST_MODELS` (`anthropic` → `claude-haiku-4-5`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`) is the cheap/fast judge SKU per provider — every value is an exact key in `signalforge.llm.pricing.PRICES`. It drives the `GradeConfig.model` per-provider default (the `grade.model:`-unset sentinel resolves to it at config-load; see `grade-layer.md`). `PROVIDER_SKU_PREFIXES` (`claude-` / `gpt-` / `gemini-`) drives the cost-rollup's prefix dispatch AND the grade model↔provider compat validator. Both tables are **reusable by a future draft `--cheap` flow** — when the drafter graduates an automatic `cheap_model` swap, resolve it from `PROVIDER_FAST_MODELS` keyed on `DraftConfig.provider` rather than hardcoding `claude-haiku-4-5` (today `DraftConfig.cheap_model` defaults to the bare `claude-haiku-4-5` SKU, which matches the `anthropic` entry). +- **Provider→string mappings (#187):** `PROVIDER_DEFAULT_MODELS` and `PROVIDER_SKU_PREFIXES` also live in `signalforge.llm.providers` and are the single source of truth for two cross-cutting per-provider facts. `PROVIDER_DEFAULT_MODELS` (`anthropic` → `claude-sonnet-4-6`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`) is the **default judge SKU per provider** — every value is an exact key in `signalforge.llm.pricing.PRICES`. It drives the `GradeConfig.model` per-provider default (the `grade.model:`-unset sentinel resolves to it at config-load; see `grade-layer.md`). **Anthropic defaults to Sonnet, NOT its cheapest SKU** — the #187 calibration gate found `claude-haiku-4-5` grades stricter than Sonnet (below the 85% bar), so Haiku stays a grade opt-in; OpenAI/Gemini default to their fast judges (explicit operator choices). `PROVIDER_SKU_PREFIXES` (`claude-` / `gpt-` / `gemini-`) drives the cost-rollup's prefix dispatch AND the grade model↔provider compat validator. The cheap *draft* model is a separate concept: `DraftConfig.cheap_model` defaults to the bare `claude-haiku-4-5` SKU, which is NOT the grade default (Sonnet) — a future draft `--cheap` swap should resolve the cheap SKU from a dedicated source, not conflate it with `PROVIDER_DEFAULT_MODELS`. **Gate the cache marker on BOTH capability flags, not just `supports_prompt_caching` (#135 QG lesson).** `call_llm` sets `cache_marker_active = supports_prompt_caching AND supports_token_count`. The pre-send count gate is what enforces the sub-minimum drop + the 8000-token oversize cap; attaching a `cache_control` marker without that gate having run would send an *unvalidated* marker (a sub-minimum block silently no-ops the marker — paying the input premium with no discount; an oversize block bypasses `LLMCacheTooLargeError`). Anthropic is `True/True` so the default path is unaffected, but a future provider that caches yet has no token-count API (`True/False`) must degrade to no-caching rather than send an unguarded marker. A new provider's capability flags are load-bearing — set them honestly, and don't assume "supports caching" alone is sufficient to attach a marker. diff --git a/CHANGELOG.md b/CHANGELOG.md index 435ea334..62cc0238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ All notable changes to SignalForge are documented here. The format is loosely ba ### Changed -- **Grade default judge model switched to per-provider fast models (#187).** When `grade.model:` is **omitted**, the grade-config loader now resolves it at config-load to the calling provider's fast model from the new `signalforge.llm.providers.PROVIDER_FAST_MODELS` table — `anthropic` → `claude-haiku-4-5` (was the bare `claude-sonnet-4-6` literal regardless of provider), `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. An explicit `grade.model:` is still honoured verbatim. Every fast-model value is an exact key in `signalforge.llm.pricing.PRICES`, so the `--estimate` path never raises on the resolved default. Switching the Anthropic grade judge from Sonnet to the Haiku default cuts the per-token grade cost ~3.75× ($0.80/$4.00 vs. $3.00/$15.00 per MTok input/output). `GradeConfig.max_output_tokens` default raised `256 → 1024` to substantially reduce truncation of a verbose one-line `gemini-2.5-flash` grade JSON (a truncation would surface as the wrong typed degrade); still a cap, not a target — the expected JSON is ~150 tokens, so the larger ceiling costs nothing on the happy path. 1024 reduces but does not fully eliminate Gemini truncation at the full-fixture scale; `docs/grade-ops.md` § per-provider floors recommends `4096` for Gemini-heavy runs. New model↔provider compatibility validation: a SKU-prefix/provider mismatch (e.g. `grade.provider: openai` with a `claude-` model) now fails loud at config-load, driven by the new `signalforge.llm.providers.PROVIDER_SKU_PREFIXES` table. See `docs/grade-ops.md` and `docs/llm-providers-ops.md` § Per-provider fast-grade defaults. +- **Per-provider grade-judge default resolution (#187).** When `grade.model:` is **omitted**, the grade-config loader now resolves it at config-load to the calling provider's default judge from the new `signalforge.llm.providers.PROVIDER_DEFAULT_MODELS` table — `anthropic` → `claude-sonnet-4-6`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. This fixes the prior bug where selecting `provider: openai`/`gemini` without a model leaked the Anthropic default and failed at the API; those providers now Just Work. An explicit `grade.model:` is still honoured verbatim, and every default value is an exact key in `signalforge.llm.pricing.PRICES` so `--estimate` never raises. **Anthropic stays on Sonnet:** an empirical calibration gate (`tests/research/187-haiku-calibration/`, run against a real Sonnet baseline drafted from an `intuit_airflow` model) found `claude-haiku-4-5` grades the rubric **stricter** than Sonnet — ~77–82% per-criterion concordance, below the 85% decision bar — so Haiku is an explicit **opt-in fast judge** (`grade.model: claude-haiku-4-5`, ~3.75× cheaper, stricter), not the default. `GradeConfig.max_output_tokens` default raised `256 → 1024` to substantially reduce truncation of a verbose one-line `gemini-2.5-flash` grade JSON; still a cap, not a target. 1024 reduces but does not fully eliminate Gemini truncation at the full-fixture scale — `docs/grade-ops.md` § per-provider floors recommends `4096` for Gemini-heavy runs. New model↔provider compatibility validation: a SKU-prefix/provider mismatch (e.g. `grade.provider: openai` with a `claude-` model), or a registered custom provider with no explicit model, now fails loud at config-load. See `docs/grade-ops.md` and `docs/llm-providers-ops.md` § Per-provider grade defaults, and `docs/research/187-haiku-calibration.md` for the calibration evidence. - **Grade-layer audit JSONL ordering becomes arrival-order under concurrent dispatch (#186).** Record shape is unchanged — `audit_schema_version` stays `Literal[1]` (only on-disk sequence differs). Operators or external tooling that depend on stable ordering should sort post-load by `(artifact_id, criterion_id)` (the SignalForge test suite uses `tests/grade/_helpers.py::_sort_grade_events(...)`). Setting `grade.max_concurrent_calls: 1` in `signalforge.yml` recovers the v0.1 `(criterion, artifact)` iteration order bit-for-bit. - **Anthropic prompt-cache cost penalty under concurrent grade dispatch (#186).** Calls `1..max_concurrent_calls` dispatch in parallel before any response returns, so each pays the cache-write premium (~1.25× input cost on the cached rubric block) instead of the cache-read discount (~0.10×). For the default cap of 10 and the ~430-token rubric block: ~4 450 extra input-token-equivalents per typical model run (~$0.003–$0.005 absolute). Operators cost-sensitive enough to care can set `grade.max_concurrent_calls: 1` to recover the v0.1 cost profile (trading off ~5–6× wall-clock reduction). OpenAI and Gemini do not support prompt caching, so no penalty applies on those providers. See `docs/grade-ops.md` § "Concurrency (asyncio orchestrator)". - **Pytest downgrade `9.x → 8.x` (#186).** Adding `pytest-asyncio>=0.23,<1` to dev-deps pins `pytest<9` (no `pytest-asyncio` release supports pytest 9 yet). Emits a `PytestConfigWarning: Unknown config option: strict_markers` for the `strict_markers = true` ini key (pytest 9-specific per `testing-signal.md`'s "pytest 9 quirk"); the warning is informational and does not fail validation. The `strict_markers = true` ini key is retained — `testing-signal.md` says BOTH `--strict-markers` addopts AND `strict_markers = true` are required on pytest 9; stripping the key would silently regress when pytest-asyncio publishes a 9-compatible release and the pin can be lifted. diff --git a/docs/draft-ops.md b/docs/draft-ops.md index 642b494d..56251040 100644 --- a/docs/draft-ops.md +++ b/docs/draft-ops.md @@ -949,8 +949,9 @@ Field-by-field: Default `claude-sonnet-4-6`. Any string the SDK accepts is allowed. - **`cheap_model`** — informational; not selected automatically. The CLI (#9) flips on `--cheap` to swap `model` for this value. - Default `claude-haiku-4-5` (bare SKU — matches the `anthropic` entry - in `signalforge.llm.providers.PROVIDER_FAST_MODELS`). + Default `claude-haiku-4-5` (bare SKU — matches a `PRICES` key; it is + also the opt-in fast judge on the grade side per #187, though the grade + *default* stayed `claude-sonnet-4-6` after the #187 calibration gate). - **`max_output_tokens`** — Anthropic `max_tokens` ceiling. Must be positive (validator). - **`cache_ttl`** — `Literal["5m", "1h"]`. `"1h"` opts into the diff --git a/docs/grade-ops.md b/docs/grade-ops.md index 874ec0a0..ee384b7e 100644 --- a/docs/grade-ops.md +++ b/docs/grade-ops.md @@ -117,7 +117,7 @@ load cleanly through `load_grade_config`: # signalforge.yml — grade stage configuration grade: provider: anthropic # registry-validated; "anthropic" + "openai" + "gemini" are registered (see provider sections below) - # model: claude-haiku-4-5 # omit to auto-resolve to the provider's fast model (anthropic -> claude-haiku-4-5); set explicitly to override + # model: claude-haiku-4-5 # omit to auto-resolve to the provider's default judge (anthropic -> claude-sonnet-4-6); set claude-haiku-4-5 to opt into the faster/stricter Haiku judge cache_ttl: 1h # Prompt-cache TTL ('5m' or '1h') max_output_tokens: 1024 # Per-criterion JSON response cap (default 1024) max_retries_429: 3 # Rate-limit retry budget @@ -160,7 +160,7 @@ grade: Field-by-field: - **`provider`** — The LLM provider strategy name (issue #135 DEC-007), resolved against the `signalforge.llm.providers` registry and threaded into `call_llm` from the per-criterion judge call, independently of the drafter's `DraftConfig.provider`. Default `"anthropic"`. An unknown value fails loud at config-load, listing the registered provider names. Deliberately a registry-validated `str`, not a `Literal` — the provider registry is a forward-looking plugin point. Today `anthropic`, `openai`, and `gemini` are registered; see [OpenAI provider](#openai-provider) and [Gemini provider](#gemini-provider) below for the non-default options. -- **`model`** — The model id used by every per-pair judge call. **Default resolves per-provider at config-load** (#187): when `model:` is omitted, the loader injects the calling provider's fast model from `signalforge.llm.providers.PROVIDER_FAST_MODELS` — `anthropic` → `claude-haiku-4-5`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. An explicit `model:` is honoured verbatim. A SKU-prefix/provider mismatch (e.g. `provider: openai` with a `claude-` model) fails loud at config-load via the model↔provider compat validator (reusing `signalforge.llm.providers.PROVIDER_SKU_PREFIXES`). +- **`model`** — The model id used by every per-pair judge call. **Default resolves per-provider at config-load** (#187): when `model:` is omitted, the loader injects the calling provider's default judge model from `signalforge.llm.providers.PROVIDER_DEFAULT_MODELS` — `anthropic` → `claude-sonnet-4-6`, `openai` → `gpt-4o-mini`, `gemini` → `gemini-2.5-flash`. **Anthropic defaults to Sonnet:** the #187 calibration gate found `claude-haiku-4-5` grades the rubric stricter than Sonnet (~77–82% concordance, below the 85% bar — see `docs/research/187-haiku-calibration.md`), so Haiku is an explicit opt-in (`grade.model: claude-haiku-4-5`), not the default. An explicit `model:` is honoured verbatim. A SKU-prefix/provider mismatch (e.g. `provider: openai` with a `claude-` model) fails loud at config-load via the model↔provider compat validator (reusing `signalforge.llm.providers.PROVIDER_SKU_PREFIXES`). - **`cache_ttl`** — `Literal["5m", "1h"]`. Default `"1h"` (vs. the drafter's `"5m"`) because 60 sequential per-criterion calls under retry backoff can stretch beyond a 5-minute window; `"1h"` gives margin at no extra cost (cache writes are one-shot regardless of TTL). - **`max_output_tokens`** — Per-criterion judge response cap. Default `1024` (#187 — raised from 256 to substantially reduce truncation risk for a verbose one-line `gemini-2.5-flash` grade JSON; the expected JSON response is still ~150 tokens, so the larger ceiling costs nothing on the happy path). 1024 reduces but does not fully eliminate Gemini truncation at scale — see the per-provider floors below; Gemini-heavy runs may want `4096`. Independent of `DraftConfig.max_output_tokens`. - **`max_retries_429` / `max_retries_5xx` / `max_retries_conn`** — Per-call retry budgets at the centralised, provider-neutral `signalforge.llm.call_llm` seam (#5 DEC-012; #135 DEC-005). Defaults `3 / 1 / 1` mirror `DraftConfig`; dial down for batch CLI mode where one retry-exhaustion is preferable to dozens of stalled calls. @@ -593,24 +593,27 @@ specifically. See § "Measured baseline (2026-05-29)" for the full-suite rollup ($1.38/run across the three providers). -**Per-provider fast-default judge models (#187).** When `grade.model:` is -omitted the loader resolves to the calling provider's *fast* model — the -cheapest registered SKU per provider (`signalforge.llm.providers.PROVIDER_FAST_MODELS`). -The rows below pair each fast default with its per-MTok USD list price from -`signalforge.llm.pricing` (pricing-table version `2026-05-28`) and an -*estimated* per-model grade cost, scaled from the Sonnet baseline above by -the input/output price ratio (estimate, not a measured run): - -| Provider × fast default | Input $/MTok | Output $/MTok | Est. per-model grade cost | Notes | +**Per-provider default judge models (#187).** When `grade.model:` is omitted +the loader resolves to the calling provider's default judge +(`signalforge.llm.providers.PROVIDER_DEFAULT_MODELS`). Anthropic defaults to +**Sonnet** (the #187 calibration gate kept it the default — Haiku grades +stricter, below the 85% bar); OpenAI/Gemini default to their fast judges +(explicit operator choices of a cheaper provider). The rows below pair each +default with its per-MTok USD list price from `signalforge.llm.pricing` +(pricing-table version `2026-05-28`) and an *estimated* per-model grade cost +(estimate, not a measured run, except where noted): + +| Provider × default judge | Input $/MTok | Output $/MTok | Est. per-model grade cost | Notes | |----------------------------------|--------------|---------------|---------------------------|----------------------------------------------------------------------------------------| -| Anthropic `claude-haiku-4-5` | $0.80 | $4.00 | ~$0.10 | The new default grade judge; ~3.75× cheaper than `claude-sonnet-4-6` per token. | -| OpenAI `gpt-4o-mini` | $0.15 | $0.60 | ~$0.013 | ~16.7× cheaper than `gpt-4o` per token; the fast default when `provider: openai`. | +| Anthropic `claude-sonnet-4-6` | $3.00 | $15.00 | ~$0.38 (measured) | The default grade judge (calibration baseline). `claude-haiku-4-5` ($0.80/$4.00, ~$0.10, ~3.75× cheaper) is the opt-in fast judge — stricter, see the calibration writeup. | +| OpenAI `gpt-4o-mini` | $0.15 | $0.60 | ~$0.013 | ~16.7× cheaper than `gpt-4o` per token; the default when `provider: openai`. | | Gemini `gemini-2.5-flash` | $0.30 | $2.50 | ~$0.045 | Already the documented mid-tier default; the measured figure above is this same SKU. | For completeness, the registered Anthropic SKUs span `claude-haiku-4-5` ($0.80 / $4.00 per MTok), `claude-sonnet-4-6` ($3.00 / $15.00), and -`claude-opus-4-7` ($15.00 / $75.00) — switching the grade judge from -Sonnet to the Haiku default cuts the per-token grade cost ~3.75×. +`claude-opus-4-7` ($15.00 / $75.00) — opting into the Haiku judge +(`grade.model: claude-haiku-4-5`) cuts the per-token grade cost ~3.75× vs the +Sonnet default, at the cost of stricter grading (#187 calibration). **Fan-out comparison vs the batched alternative:** diff --git a/docs/llm-providers-ops.md b/docs/llm-providers-ops.md index 64bb215d..b1377058 100644 --- a/docs/llm-providers-ops.md +++ b/docs/llm-providers-ops.md @@ -105,7 +105,7 @@ not a `Literal`). See [Adding a provider](#adding-a-provider) below. | **Pre-send `count_tokens` gate** | ✅ | ❌ (no SDK token-count API) | ❌ (deferred — Gemini has the API but we don't gate on it for cache parity) | | **`cache_ttl` config** | honoured (`"5m"` / `"1h"`) | silently ignored | silently ignored | | **Default drafter model** | `claude-sonnet-4-6` | `gpt-4o` | unset | -| **Default grader model** | `claude-haiku-4-5` (fast default, #187) | `gpt-4o-mini` (fast default) | `gemini-2.5-flash` (fast default) | +| **Default grader model** | `claude-sonnet-4-6` (#187; Haiku is opt-in) | `gpt-4o-mini` (fast default) | `gemini-2.5-flash` (fast default) | | **Live smoke marker** | `@pytest.mark.anthropic` | `@pytest.mark.openai` | `@pytest.mark.gemini` | | **Live smoke env** | `ANTHROPIC_API_KEY` | `SF_RUN_OPENAI=1` + `OPENAI_API_KEY` | `SF_RUN_GEMINI=1` + `GOOGLE_API_KEY` | @@ -118,15 +118,21 @@ spend at full rates. ### Per-provider fast-grade defaults When `grade.model:` is **omitted**, the grade-config loader resolves it -at config-load to the calling provider's *fast* model — the cheapest -registered SKU per provider. The single source of truth is the -`PROVIDER_FAST_MODELS` table in `signalforge.llm.providers` (#187 US-001): - -| Grade provider | Fast default SKU (`PROVIDER_FAST_MODELS`) | -|---|---| -| `anthropic` | `claude-haiku-4-5` | -| `openai` | `gpt-4o-mini` | -| `gemini` | `gemini-2.5-flash` | +at config-load to the calling provider's default judge model. The single +source of truth is the `PROVIDER_DEFAULT_MODELS` table in +`signalforge.llm.providers` (#187). Anthropic defaults to **Sonnet** — +the #187 calibration gate found `claude-haiku-4-5` grades the rubric +stricter than Sonnet (~77–82% concordance, below the 85% bar; see +`docs/research/187-haiku-calibration.md`), so Haiku is an explicit opt-in +(`grade.model: claude-haiku-4-5`), not the default. OpenAI/Gemini default +to their fast judges (explicit operator choices of a cheaper provider, +never calibrated against the Sonnet baseline): + +| Grade provider | Default judge SKU (`PROVIDER_DEFAULT_MODELS`) | Opt-in fast judge | +|---|---|---| +| `anthropic` | `claude-sonnet-4-6` | `claude-haiku-4-5` (stricter; #187) | +| `openai` | `gpt-4o-mini` | — | +| `gemini` | `gemini-2.5-flash` | — | Every value is an exact key in `signalforge.llm.pricing.PRICES`, so the `--estimate` cost-preview path and `pricing.lookup(model)` never raise on @@ -154,10 +160,11 @@ compat check. The shipped default for both stages. No extra install; set `ANTHROPIC_API_KEY` and SignalForge runs out of the box. -- **Default models:** `claude-sonnet-4-6` (drafter), `claude-haiku-4-5` - (grader fast default per #187 — see [Per-provider fast-grade - defaults](#per-provider-fast-grade-defaults) below), `claude-haiku-4-5` - (drafter `cheap_model`). +- **Default models:** `claude-sonnet-4-6` (drafter AND grader — the + grader default stayed Sonnet per the #187 calibration gate; see + [Per-provider grade defaults](#per-provider-fast-grade-defaults) below), + `claude-haiku-4-5` (the opt-in fast grade judge AND the drafter + `cheap_model`). - **Prompt caching:** active. Drafter caches the manifest summary block; grader caches the rubric criterion list. `cache_ttl: 1h` opts into the `extended-cache-ttl-2025-04-11` beta header. The diff --git a/docs/research/187-haiku-calibration.md b/docs/research/187-haiku-calibration.md index 6143cea9..ab19eb46 100644 --- a/docs/research/187-haiku-calibration.md +++ b/docs/research/187-haiku-calibration.md @@ -1,12 +1,13 @@ # Issue #187 — Haiku grade-default calibration gate -**Status:** RUN COMPLETE (2026-06-02). The #187 plan ships `claude-haiku-4-5` -as the new grade-default SKU behind this empirical gate (DEC-005). The gate has -now been run against a **real Sonnet baseline drafted from a real -`intuit_airflow` model** — and **Haiku does NOT clear the ≥ 85% concordance -bar** (81.8% and 77.0% on two independent runs). Per the DEC-005 decision rule, -this points to **shipping Haiku as an opt-in fast mode, not the default**. See -§ "Result" and § "Disposition". +**Status:** RUN COMPLETE + DECISION IMPLEMENTED (2026-06-02). This gate was run +against a **real Sonnet baseline drafted from a real `intuit_airflow` model** and +**Haiku did NOT clear the ≥ 85% concordance bar** (81.8% and 77.0% on two +independent runs). Per the DEC-005 decision rule (`< 85% → opt-in, not default`), +**the grade default was kept at `claude-sonnet-4-6`** and `claude-haiku-4-5` ships +as an **explicit opt-in** (`grade.model: claude-haiku-4-5`). `PROVIDER_DEFAULT_MODELS` +now maps `anthropic → claude-sonnet-4-6` (OpenAI/Gemini keep their fast defaults). +See § "Result" and § "Disposition". **Companion artefacts:** @@ -90,15 +91,18 @@ bytes, the rubric is locked, the Sonnet baseline is committed. ### Config under test -`GradeConfig()` with all defaults — after US-002 this resolves to `model → -claude-haiku-4-5`, `max_output_tokens → 1024`, `provider → anthropic`. The -harness asserts both resolved values before grading, so a resolver regression -fails the gate loud rather than silently measuring the wrong SKU. +`GradeConfig(model="claude-haiku-4-5")` — the **Haiku opt-in**. After the +decision below, `GradeConfig()` (no model) resolves to the *Sonnet* default, so +the gate selects Haiku explicitly to measure the opt-in. `max_output_tokens → +1024`, `provider → anthropic`. The harness also asserts `GradeConfig().model == +"claude-sonnet-4-6"` (the default is Sonnet, not Haiku) so a resolver regression +fails loud. ## Method — the ≥ 85% concordance rule -1. Build the resolved Haiku-default `GradeConfig()`; assert `model == - "claude-haiku-4-5"` and `max_output_tokens == 1024`. +1. Build the Haiku opt-in `GradeConfig(model="claude-haiku-4-5")`; assert + `max_output_tokens == 1024` and that the *default* `GradeConfig().model` is + `claude-sonnet-4-6`. 2. Assert the committed baseline covers every `artifact_id` the engine will grade (no silent gaps). 3. Run `grade_artifacts(...)` — 84 live Haiku judge calls. @@ -197,16 +201,19 @@ the uncertainty. Per the DEC-005 decision rule (**< 85% → opt-in knob, not default**), the real calibration says **do not ship `claude-haiku-4-5` as the resolved grade -default**. Options, in order of fidelity to the data: - -1. **Recommended — make Haiku opt-in, keep Sonnet the grade default.** Revert - the Anthropic entry in `PROVIDER_FAST_MODELS` (or the grade resolution) so - `provider: anthropic` resolves to `claude-sonnet-4-6`, and document - `grade.model: claude-haiku-4-5` as the operator-opt-in fast mode. The - per-provider resolver, compat validator, and 1024 cap (US-001..US-006) all - stand — only the Anthropic *default target* changes. OpenAI/Gemini fast - defaults are unaffected by this Anthropic-specific finding (they were never - calibrated against a Sonnet baseline; they're explicit operator choices). +default**. **Decision: Option 1 was chosen and implemented** (2026-06-02). + +1. **✅ CHOSEN + IMPLEMENTED — Haiku is opt-in, Sonnet is the grade default.** + `PROVIDER_DEFAULT_MODELS["anthropic"]` now resolves to `claude-sonnet-4-6`; + `grade.model: claude-haiku-4-5` is the documented operator opt-in (faster, + ~3.75× cheaper, but stricter). The per-provider resolver, compat validator, + and 1024 cap all stand — only the Anthropic *default target* changed. + OpenAI/Gemini fast defaults are unaffected by this Anthropic-specific finding + (they were never calibrated against a Sonnet baseline; they're explicit + operator choices). The constant was renamed `PROVIDER_FAST_MODELS` → + `PROVIDER_DEFAULT_MODELS` since Sonnet is not "fast". This gated test now + selects Haiku explicitly and still asserts ≥ 85% (so it fails) — the failure + is the durable record that Haiku is the stricter opt-in, not the default. 2. **Accept Haiku at ~80% with eyes open** — only if the maintainer judges the ~3× speed / ~3.75× cost win worth a stricter judge that flags ~1 in 5 rubric verdicts differently. This contradicts the gate's own rule; if taken, lower diff --git a/src/signalforge/grade/config.py b/src/signalforge/grade/config.py index 971fb5f3..6d66e397 100644 --- a/src/signalforge/grade/config.py +++ b/src/signalforge/grade/config.py @@ -24,12 +24,15 @@ here. The loader takes it as a required argument so the caller is explicit about the resolution base. * **DEC-023..DEC-027** — Locked default values: - ``model=None`` (resolves to the calling provider's fast model at - config-load — ``anthropic`` -> ``claude-haiku-4-5`` per - :data:`signalforge.llm.providers.PROVIDER_FAST_MODELS`; #187 US-002 / - DEC-004), ``cache_ttl="1h"``, ``max_output_tokens=1024`` (#187 DEC-004 - — raised from 256 so a one-line ``gemini-2.5-flash`` grade JSON is not - truncated), ``max_retries_429=3``, ``max_retries_5xx=1``, + ``model=None`` (resolves to the calling provider's default judge model + at config-load — ``anthropic`` -> ``claude-sonnet-4-6`` per + :data:`signalforge.llm.providers.PROVIDER_DEFAULT_MODELS`; #187 US-002 / + DEC-004. The #187 calibration gate found ``claude-haiku-4-5`` grades + the rubric stricter than Sonnet — below the 85% bar — so Haiku stays an + explicit opt-in, not the default), ``cache_ttl="1h"``, + ``max_output_tokens=1024`` (#187 DEC-004 — raised from 256 so a one-line + ``gemini-2.5-flash`` grade JSON is substantially less likely to + truncate), ``max_retries_429=3``, ``max_retries_5xx=1``, ``max_retries_conn=1``, ``total_budget_seconds=300``, ``min_pass_rate=0.7``, ``min_mean_score=0.5``, ``rubric=None``, ``fail_on_below_threshold=False``. @@ -80,7 +83,7 @@ from signalforge.grade.errors import GradeConfigError, GradeRubricError from signalforge.grade.rubric import Rubric, validate_rubric -from signalforge.llm.providers import PROVIDER_FAST_MODELS, PROVIDER_SKU_PREFIXES +from signalforge.llm.providers import PROVIDER_DEFAULT_MODELS, PROVIDER_SKU_PREFIXES _DEFAULT_CONFIG_FILENAME = "signalforge.yml" @@ -106,14 +109,17 @@ class GradeConfig(BaseModel): model: str | None = None """LLM-judge model id (DEC-026; #187 US-002 / DEC-004). - The sentinel default ``None`` means "use the calling provider's fast - model" — resolved at config-load by the + The sentinel default ``None`` means "use the calling provider's + default judge model" — resolved at config-load by the :meth:`_resolve_model_default` before-validator to - :data:`signalforge.llm.providers.PROVIDER_FAST_MODELS` keyed on - :attr:`provider` (``anthropic`` -> ``claude-haiku-4-5``, ``openai`` - -> ``gpt-4o-mini``, ``gemini`` -> ``gemini-2.5-flash``). An explicit - ``model:`` is always honoured verbatim. After construction this - field is always a concrete non-empty string — never ``None``. + :data:`signalforge.llm.providers.PROVIDER_DEFAULT_MODELS` keyed on + :attr:`provider` (``anthropic`` -> ``claude-sonnet-4-6``, ``openai`` + -> ``gpt-4o-mini``, ``gemini`` -> ``gemini-2.5-flash``). Anthropic + defaults to Sonnet because the #187 calibration gate found + ``claude-haiku-4-5`` grades stricter than Sonnet (below the 85% bar); + Haiku is an explicit opt-in (``grade.model: claude-haiku-4-5``). An + explicit ``model:`` is always honoured verbatim. After construction + this field is always a concrete non-empty string — never ``None``. When set explicitly, a SKU-prefix/provider mismatch (e.g. ``provider="openai"`` with a ``claude-`` model) fails loud at @@ -246,7 +252,7 @@ class GradeConfig(BaseModel): @model_validator(mode="before") @classmethod def _resolve_model_default(cls, data: Any) -> Any: - """Resolve the sentinel ``model=None`` to the provider's fast model. + """Resolve the sentinel ``model=None`` to the provider's default judge model. Runs BEFORE field validation (and before the frozen instance exists) so the injected value flows through the normal @@ -257,14 +263,14 @@ def _resolve_model_default(cls, data: Any) -> Any: untouched. When ``model`` is absent or ``None``, inject - :data:`signalforge.llm.providers.PROVIDER_FAST_MODELS` keyed on + :data:`signalforge.llm.providers.PROVIDER_DEFAULT_MODELS` keyed on the requested ``provider`` (defaulting to ``"anthropic"`` to - match the field default). A provider NOT in the fast-model table + match the field default). A provider NOT in the default-model table is left alone — no injection — via ``.get()`` so this never masks an error with a ``KeyError`` (#187 US-002 / DEC-004). Two such cases follow downstream: an *unregistered* provider is rejected by the ``provider`` field-validator (:class:`UnknownProviderError`); - a *registered* provider absent from the fast-model table with no + a *registered* provider absent from the default-model table with no explicit model is rejected by :meth:`_validate_model_provider_compat` (which requires the operator to set ``grade.model`` explicitly). @@ -273,7 +279,7 @@ def _resolve_model_default(cls, data: Any) -> Any: return data if data.get("model") is None: provider = data.get("provider", "anthropic") - resolved = PROVIDER_FAST_MODELS.get(provider) + resolved = PROVIDER_DEFAULT_MODELS.get(provider) if resolved is not None: # Copy-on-write so we don't mutate a caller-owned dict. data = {**data, "model": resolved} @@ -388,9 +394,9 @@ def _validate_model_provider_compat(self) -> GradeConfig: table. A registry-valid provider absent from - :data:`signalforge.llm.providers.PROVIDER_FAST_MODELS` AND given + :data:`signalforge.llm.providers.PROVIDER_DEFAULT_MODELS` AND given no explicit ``model`` reaches here with ``model is None`` (the - before-validator had no fast model to inject; the ``provider`` + before-validator had no default model to inject; the ``provider`` field-validator passed because the provider IS registered). We cannot guess a custom provider's model, so this fails loud rather than letting ``None`` flow into the engine — which keeps the diff --git a/src/signalforge/llm/providers.py b/src/signalforge/llm/providers.py index 046c1381..6f4cb7cf 100644 --- a/src/signalforge/llm/providers.py +++ b/src/signalforge/llm/providers.py @@ -347,7 +347,7 @@ def provider_for(name: str) -> LLMProvider: # source of truth for two cross-cutting facts that previously lived as # duplicated literals scattered across stages: # -# * ``PROVIDER_FAST_MODELS`` — the cheap/fast judge SKU per provider, used by +# * ``PROVIDER_DEFAULT_MODELS`` — the cheap/fast judge SKU per provider, used by # the faster-grade defaults (#187). Every value MUST be an exact key in # :data:`signalforge.llm.pricing.PRICES` so ``pricing.lookup(model)`` and the # ``--estimate`` cost-preview path never raise. @@ -361,10 +361,16 @@ def provider_for(name: str) -> LLMProvider: # lookup tables, the values are immutable strings, and no caller mutates them. # --------------------------------------------------------------------------- -#: Cheap/fast judge SKU per provider (#187 US-001). Every value is an exact -#: key in :data:`signalforge.llm.pricing.PRICES`. -PROVIDER_FAST_MODELS: dict[str, str] = { - "anthropic": "claude-haiku-4-5", +#: Default judge SKU per provider (#187) — used when ``grade.model`` is unset. +#: Anthropic defaults to ``claude-sonnet-4-6``: the #187 calibration gate found +#: ``claude-haiku-4-5`` grades the rubric stricter than Sonnet (~77-82% +#: concordance, below the 85% bar — see ``docs/research/187-haiku-calibration.md``), +#: so Haiku stays an explicit opt-in (`grade.model: claude-haiku-4-5`), not the +#: default. OpenAI/Gemini default to their fast judges (explicit operator choices +#: of a cheaper provider; never calibrated against the Sonnet baseline). Every +#: value is an exact key in :data:`signalforge.llm.pricing.PRICES`. +PROVIDER_DEFAULT_MODELS: dict[str, str] = { + "anthropic": "claude-sonnet-4-6", "openai": "gpt-4o-mini", "gemini": "gemini-2.5-flash", } @@ -1463,7 +1469,7 @@ def estimate_input_tokens( __all__ = ( - "PROVIDER_FAST_MODELS", + "PROVIDER_DEFAULT_MODELS", "PROVIDER_SKU_PREFIXES", "AnthropicProvider", "ExceptionCategory", diff --git a/tests/cli/test_estimate_engine.py b/tests/cli/test_estimate_engine.py index cee1b8a4..18d7f258 100644 --- a/tests/cli/test_estimate_engine.py +++ b/tests/cli/test_estimate_engine.py @@ -239,37 +239,27 @@ def test_estimate_total_llm_usd_matches_hand_calculation( ) -> None: """Pin USD to four decimals against a hand-computed expected. - Hand calculation with the default ``DraftConfig`` / ``GradeConfig`` - and the default rubric (4 criteria). Post #187 US-002 the two stages - use DIFFERENT default models — the drafter stays on - ``claude-sonnet-4-6`` (``$3/MTok`` input, ``$15/MTok`` output) while - the grader resolves to the provider's fast model - ``claude-haiku-4-5`` (``$0.80/MTok`` input, ``$4/MTok`` output). The - draft and grade halves therefore key on separate price rows: - - Draft (sonnet pricing): - Draft input: 1_000_000 tokens (1 MTok) → 1 * 3.00 = $3.00. - Draft output: 4096 tokens (default ``DraftConfig.max_output_tokens``) - → 4096 / 1e6 * 15 ≈ $0.06144. - Draft USD ≈ 3.06144. - - Grade (haiku pricing — 4 criteria): + Hand calculation with the default ``DraftConfig``/``GradeConfig`` + (``claude-sonnet-4-6`` for both, ``$3/MTok`` input, ``$15/MTok`` + output) and the default rubric (4 criteria): + + Draft input: 1_000_000 tokens (1 MTok) → $3.00. + Draft output: 4096 tokens (default ``max_output_tokens``) + → 4096 / 1e6 * 15 ≈ $0.06144. + Draft USD ≈ 3.06144. + + Grade per criterion (4 criteria): artifact_count for our 2-column model: 2*2 (column desc+rationale) + 2 (model desc+rationale) + int(3.5*2) (test rationales) = 4 + 2 + 7 = 13. Input tokens per call (queued) = 500 → 500 * 13 = 6500. - Per-criterion input USD: 6500/1e6 * 0.80 = 0.0052. - Per-criterion output USD: 50 * 13 / 1e6 * 4 = 650/1e6*4 - = 0.0026. - (The grade output-token figure is the fixed - ``_GRADE_OUTPUT_TOKENS_PER_CALL`` of 50, NOT - ``GradeConfig.max_output_tokens`` — the 256→1024 default bump in - #187 US-002 is a response cap, not the estimate's per-call - output projection, so it does not enter this math.) - Per-criterion total: 0.0052 + 0.0026 = 0.0078. - Across 4 criteria: 4 * 0.0078 = 0.0312. - - Grand total: 3.06144 + 0.0312 = 3.09264. + Per-criterion input USD: 6500/1e6 * 3 = 0.0195. + Per-criterion output USD: 50 * 13 / 1e6 * 15 = 650/1e6*15 + = 0.00975. + Per-criterion total: 0.0195 + 0.00975 = 0.02925. + Across 4 criteria: 4 * 0.02925 = 0.117. + + Grand total: 3.06144 + 0.117 = 3.17844. Test pins to 4 decimals. """ @@ -287,20 +277,13 @@ def test_estimate_total_llm_usd_matches_hand_calculation( fake_anthropic, ) - # The drafter and grader key on separate price rows post #187 US-002. - draft_pricing = pricing_lookup(draft_config.model) - # Pin the resolved fast default explicitly so a resolver regression (grade - # silently falling back to Sonnet) fails HERE rather than drifting the - # engine and the expected USD together into a passing tautology. The - # `is not None` also narrows `str | None` -> `str` for pricing_lookup. - assert grade_config.model is not None and grade_config.model == "claude-haiku-4-5" - grade_pricing = pricing_lookup(grade_config.model) - expected_draft = (1_000_000 / 1_000_000.0) * draft_pricing.input_per_mtok + ( + pricing = pricing_lookup(draft_config.model) + expected_draft = (1_000_000 / 1_000_000.0) * pricing.input_per_mtok + ( 4096 / 1_000_000.0 - ) * draft_pricing.output_per_mtok + ) * pricing.output_per_mtok artifact_count = 2 * 2 + 2 + int(3.5 * 2) - per_crit_in = (500 * artifact_count) / 1_000_000.0 * grade_pricing.input_per_mtok - per_crit_out = (50 * artifact_count) / 1_000_000.0 * grade_pricing.output_per_mtok + per_crit_in = (500 * artifact_count) / 1_000_000.0 * pricing.input_per_mtok + per_crit_out = (50 * artifact_count) / 1_000_000.0 * pricing.output_per_mtok expected_grade = n_criteria * (per_crit_in + per_crit_out) expected_total = expected_draft + expected_grade diff --git a/tests/cli/test_generate_estimate.py b/tests/cli/test_generate_estimate.py index 272baccf..71808b94 100644 --- a/tests/cli/test_generate_estimate.py +++ b/tests/cli/test_generate_estimate.py @@ -494,7 +494,7 @@ def test_generate_estimate_divergent_providers_fails_fast( try: project_dir = make_fake_dbt_project(tmp_path) # An explicit model is required for a custom provider (it's not in - # PROVIDER_FAST_MODELS, so #187 won't guess its fast default); supply one + # PROVIDER_DEFAULT_MODELS, so #187 won't guess its fast default); supply one # so config LOADS and the test exercises the divergent-provider check # (grade=fake-nocache vs draft=anthropic → tier-2 exit 2) rather than # incidentally tripping the missing-model config-load error (tier 1). diff --git a/tests/fixtures/estimate/anthropic_byte_identity_golden.txt b/tests/fixtures/estimate/anthropic_byte_identity_golden.txt index 7ee3fedf..e3fb969b 100644 --- a/tests/fixtures/estimate/anthropic_byte_identity_golden.txt +++ b/tests/fixtures/estimate/anthropic_byte_identity_golden.txt @@ -1,6 +1,6 @@ Estimate for model.shop.customers drafter: claude-sonnet-4-6 - grader: claude-haiku-4-5 + grader: claude-sonnet-4-6 Estimated draft cost: input tokens: 1,000 @@ -10,11 +10,11 @@ Estimated draft cost: Estimated grade cost: artifacts: 13 criteria: 4 calls: 52 per criterion: - clarity 13 calls 6,500 tokens $0.0078 - consistency 13 calls 6,500 tokens $0.0078 - rationale 13 calls 6,500 tokens $0.0078 - no-redundant 13 calls 6,500 tokens $0.0078 - cost: $0.0312 + clarity 13 calls 6,500 tokens $0.0292 + consistency 13 calls 6,500 tokens $0.0292 + rationale 13 calls 6,500 tokens $0.0292 + no-redundant 13 calls 6,500 tokens $0.0292 + cost: $0.1170 Estimated warehouse cost: bytes-per-row: ~1 (BigQuery dryRun) @@ -22,7 +22,7 @@ Estimated warehouse cost: sample size: 100,000 rows total bytes: ~68.4 KB -Total estimated LLM cost: $0.0956 +Total estimated LLM cost: $0.1814 Total estimated warehouse: ~68.4 KB Price table: 2026-05-28 | Heuristic: ~3.5 tests/column (canonical fixture average) diff --git a/tests/fixtures/grade/example_config.yml b/tests/fixtures/grade/example_config.yml index bec2fddb..88a0a92b 100644 --- a/tests/fixtures/grade/example_config.yml +++ b/tests/fixtures/grade/example_config.yml @@ -1,6 +1,6 @@ # signalforge.yml — grade stage configuration (v0.1) grade: - model: claude-sonnet-4-6 # explicit override (omit to auto-resolve to the provider fast default, #187 — anthropic→claude-haiku-4-5) + model: claude-sonnet-4-6 # also the anthropic default (omit to auto-resolve per provider, #187). Set claude-haiku-4-5 to opt into the faster/stricter Haiku judge. cache_ttl: 1h # Prompt-cache TTL ('5m' or '1h') max_output_tokens: 256 # explicit override (default is 1024 since #187) max_retries_429: 3 # Rate-limit retry budget diff --git a/tests/fixtures/grade/grade_event_v1.jsonl b/tests/fixtures/grade/grade_event_v1.jsonl index cf66ae88..f27999d7 100644 --- a/tests/fixtures/grade/grade_event_v1.jsonl +++ b/tests/fixtures/grade/grade_event_v1.jsonl @@ -1 +1 @@ -{"audit_schema_version":1,"signalforge_version":"0.1.0.dev0","run_id":"a1b2c3d4e5f6478890aabbccddeeff00","timestamp":"2026-05-01T17:42:13.123456Z","model_unique_id":"model.shop.dim_customers","artifact_id":"column.email.description","criterion_id":"clarity","score":0.8,"passed":true,"evidence":"The description states 'Email address of the customer at the time of order'.","reasoning":"The description is clear and specific about which email is captured. It would be improved by noting whether the value is normalised, but the meaning is unambiguous as written.","rubric_hash":"0123456789abcdef","prompt_version_template":"fedcba9876543210","criterion_prompt_hash":"1111222233334444","response_text_hash":"5555666677778888","model":"claude-haiku-4-5","input_tokens":1820,"output_tokens":140,"cache_creation_input_tokens":0,"cache_read_input_tokens":1500} +{"audit_schema_version":1,"signalforge_version":"0.1.0.dev0","run_id":"a1b2c3d4e5f6478890aabbccddeeff00","timestamp":"2026-05-01T17:42:13.123456Z","model_unique_id":"model.shop.dim_customers","artifact_id":"column.email.description","criterion_id":"clarity","score":0.8,"passed":true,"evidence":"The description states 'Email address of the customer at the time of order'.","reasoning":"The description is clear and specific about which email is captured. It would be improved by noting whether the value is normalised, but the meaning is unambiguous as written.","rubric_hash":"0123456789abcdef","prompt_version_template":"fedcba9876543210","criterion_prompt_hash":"1111222233334444","response_text_hash":"5555666677778888","model":"claude-sonnet-4-6","input_tokens":1820,"output_tokens":140,"cache_creation_input_tokens":0,"cache_read_input_tokens":1500} diff --git a/tests/grade/test_config.py b/tests/grade/test_config.py index 569983af..02ab886c 100644 --- a/tests/grade/test_config.py +++ b/tests/grade/test_config.py @@ -188,9 +188,10 @@ def test_grade_config_defaults_match_dec_023_to_027() -> None: drift here is a behaviour change masquerading as a refactor.""" cfg = GradeConfig() # #187 US-002 / DEC-004: ``model`` now defaults to the sentinel that - # resolves to the calling provider's fast model. With the default - # provider (``anthropic``) that is ``claude-haiku-4-5``. - assert cfg.model == "claude-haiku-4-5" + # resolves to the calling provider's default judge model. With the + # default provider (``anthropic``) that is ``claude-sonnet-4-6`` — the + # #187 calibration gate kept Sonnet the default (Haiku is opt-in). + assert cfg.model == "claude-sonnet-4-6" assert cfg.cache_ttl == "1h" # #187 DEC-004: raised from 256 to avoid one-line gemini-flash truncation. assert cfg.max_output_tokens == 1024 @@ -276,10 +277,12 @@ def test_load_grade_config_unknown_provider_fails_loud(tmp_path: Path) -> None: # ----- Per-provider fast-model resolution (#187 US-002 / DEC-004) ----- -def test_grade_config_model_resolves_anthropic_fast_default() -> None: +def test_grade_config_model_resolves_anthropic_default() -> None: """The sentinel ``model=None`` (default) resolves to the anthropic - fast model via :data:`PROVIDER_FAST_MODELS`.""" - assert GradeConfig().model == "claude-haiku-4-5" + default judge model (``claude-sonnet-4-6``) via + :data:`PROVIDER_DEFAULT_MODELS`. Haiku is an explicit opt-in — the + #187 calibration gate found it grades stricter than Sonnet.""" + assert GradeConfig().model == "claude-sonnet-4-6" def test_grade_config_model_resolves_openai_fast_default() -> None: diff --git a/tests/grade/test_provider_neutrality.py b/tests/grade/test_provider_neutrality.py index 5563e9d8..3c601403 100644 --- a/tests/grade/test_provider_neutrality.py +++ b/tests/grade/test_provider_neutrality.py @@ -153,7 +153,7 @@ def test_registering_provider_is_the_only_wiring_needed(_isolate_registry: None) assert provider_for(FAKE_NOCACHE_PROVIDER_NAME) is provider # The registry-validated config str accepts it. A custom provider is not in - # PROVIDER_FAST_MODELS, so #187 requires an explicit model (we can't guess a + # PROVIDER_DEFAULT_MODELS, so #187 requires an explicit model (we can't guess a # plugin provider's fast model) rather than silently defaulting it. config = GradeConfig(provider=FAKE_NOCACHE_PROVIDER_NAME, model="fake-nocache-judge") assert config.provider == FAKE_NOCACHE_PROVIDER_NAME diff --git a/tests/llm/test_providers.py b/tests/llm/test_providers.py index 8f51a2b9..0cef467a 100644 --- a/tests/llm/test_providers.py +++ b/tests/llm/test_providers.py @@ -19,7 +19,7 @@ from signalforge.llm.errors import UnknownProviderError from signalforge.llm.providers import ( - PROVIDER_FAST_MODELS, + PROVIDER_DEFAULT_MODELS, PROVIDER_SKU_PREFIXES, AnthropicProvider, ExceptionCategory, @@ -1438,7 +1438,7 @@ def test_unclean_finish_reason_message_default_returns_generic_diagnostic() -> N # --------------------------------------------------------------------------- -# #187 US-001 — PROVIDER_FAST_MODELS + PROVIDER_SKU_PREFIXES constants +# #187 US-001 — PROVIDER_DEFAULT_MODELS + PROVIDER_SKU_PREFIXES constants # --------------------------------------------------------------------------- @@ -1449,50 +1449,50 @@ def test_unclean_finish_reason_message_default_returns_generic_diagnostic() -> N @pytest.mark.unit @pytest.mark.llm -def test_provider_fast_models_keys_are_the_three_registered_providers() -> None: - """``PROVIDER_FAST_MODELS`` is keyed by exactly the three provider names +def test_provider_default_models_keys_are_the_three_registered_providers() -> None: + """``PROVIDER_DEFAULT_MODELS`` is keyed by exactly the three provider names registered in the module (#187 US-001). A new provider that ships without - a fast-model entry — or a dropped/renamed key — breaks this loudly.""" - assert set(PROVIDER_FAST_MODELS) == _REGISTERED_PROVIDER_NAMES + a default-model entry — or a dropped/renamed key — breaks this loudly.""" + assert set(PROVIDER_DEFAULT_MODELS) == _REGISTERED_PROVIDER_NAMES # Cross-check against the live registry, not just a hard-coded set, so a - # future registry change forces a fast-models update in lockstep. - for name in PROVIDER_FAST_MODELS: + # future registry change forces a default-models update in lockstep. + for name in PROVIDER_DEFAULT_MODELS: assert provider_for(name).name == name @pytest.mark.unit @pytest.mark.llm -def test_provider_fast_models_values_are_all_priced_skus() -> None: - """Every ``PROVIDER_FAST_MODELS`` value MUST be an exact key in +def test_provider_default_models_values_are_all_priced_skus() -> None: + """Every ``PROVIDER_DEFAULT_MODELS`` value MUST be an exact key in :data:`signalforge.llm.pricing.PRICES` so ``pricing.lookup(model)`` and the ``--estimate`` cost-preview path never raise on a fast default (#187 US-001).""" from signalforge.llm.pricing import PRICES, lookup - for provider, model in PROVIDER_FAST_MODELS.items(): - assert model in PRICES, f"{provider} fast model {model!r} is not a priced SKU" + for provider, model in PROVIDER_DEFAULT_MODELS.items(): + assert model in PRICES, f"{provider} default model {model!r} is not a priced SKU" # lookup() raising would surface the same gap as a hard failure; pin it. lookup(model) @pytest.mark.unit @pytest.mark.llm -def test_provider_sku_prefixes_keys_match_fast_models_keys() -> None: - """``PROVIDER_SKU_PREFIXES`` and ``PROVIDER_FAST_MODELS`` cover the same +def test_provider_sku_prefixes_keys_match_default_models_keys() -> None: + """``PROVIDER_SKU_PREFIXES`` and ``PROVIDER_DEFAULT_MODELS`` cover the same provider names — the two tables stay in lockstep (#187 US-001).""" - assert set(PROVIDER_SKU_PREFIXES) == set(PROVIDER_FAST_MODELS) + assert set(PROVIDER_SKU_PREFIXES) == set(PROVIDER_DEFAULT_MODELS) assert set(PROVIDER_SKU_PREFIXES) == _REGISTERED_PROVIDER_NAMES @pytest.mark.unit @pytest.mark.llm -def test_each_fast_model_starts_with_its_provider_prefix() -> None: +def test_each_default_model_starts_with_its_provider_prefix() -> None: """Each provider's fast model id begins with that provider's SKU prefix (#187 US-001) — a guard that the two tables describe the same SKUs.""" - for provider, model in PROVIDER_FAST_MODELS.items(): + for provider, model in PROVIDER_DEFAULT_MODELS.items(): prefix = PROVIDER_SKU_PREFIXES[provider] assert model.startswith(prefix), ( - f"{provider} fast model {model!r} does not start with prefix {prefix!r}" + f"{provider} default model {model!r} does not start with prefix {prefix!r}" ) @@ -1515,5 +1515,5 @@ def test_both_constants_are_exported() -> None: """Both constants are part of the module's public surface (#187 US-001).""" from signalforge.llm import providers as providers_module - assert "PROVIDER_FAST_MODELS" in providers_module.__all__ + assert "PROVIDER_DEFAULT_MODELS" in providers_module.__all__ assert "PROVIDER_SKU_PREFIXES" in providers_module.__all__ diff --git a/tests/research/187-haiku-calibration/test_haiku_calibration.py b/tests/research/187-haiku-calibration/test_haiku_calibration.py index b100dc7d..dc787fbb 100644 --- a/tests/research/187-haiku-calibration/test_haiku_calibration.py +++ b/tests/research/187-haiku-calibration/test_haiku_calibration.py @@ -98,15 +98,16 @@ def test_haiku_grade_concordance_vs_sonnet_baseline(tmp_path: Path) -> None: prune_result = empty_prune_result(model) baseline = load_baseline() - # Resolved Haiku default — model=None resolves to claude-haiku-4-5; - # max_output_tokens defaults to 1024. Explicit construction with the - # defaults documents the contract under test. - config = GradeConfig() - assert config.model == "claude-haiku-4-5", ( - "this gate measures the Haiku default; the resolver should have " - f"produced claude-haiku-4-5, got {config.model!r}" - ) + # The Haiku OPT-IN (post-calibration, the anthropic default is Sonnet — + # this gate is exactly why). We explicitly select claude-haiku-4-5 to + # measure whether the opt-in grades concordantly with the Sonnet default; + # the recorded result is that it does NOT (~77-82% < 85%), which is why + # Haiku stays opt-in. max_output_tokens defaults to 1024. + config = GradeConfig(model="claude-haiku-4-5") + assert config.model == "claude-haiku-4-5" assert config.max_output_tokens == 1024 + # Sanity: confirm the anthropic DEFAULT is Sonnet (Haiku is opt-in only). + assert GradeConfig().model == "claude-sonnet-4-6" # Sanity: the committed baseline must cover every artifact_id the # engine will grade (engineered determinism — no silent gaps). From 9f38a2e4795fed36e5fb920ba35d26855b53f5f8 Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 22:31:32 -0400 Subject: [PATCH 14/15] #187: record Gemini 1024-token check PASS (run live) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gemini-2.5-flash graded a verbose artifact at the new max_output_tokens=1024 default with no score=None truncation degrade — DEC-004 holds at the single-artifact scale (full-fixture runs may still want 4096 per #158). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/research/187-haiku-calibration.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/research/187-haiku-calibration.md b/docs/research/187-haiku-calibration.md index ab19eb46..2f34e84a 100644 --- a/docs/research/187-haiku-calibration.md +++ b/docs/research/187-haiku-calibration.md @@ -42,9 +42,11 @@ See § "Result" and § "Disposition". the rubric stricter than Sonnet**, concentrated on the **`no-redundant`** and **`clarity`** criteria. Haiku-as-judge would flag column rationales / descriptions that Sonnet passes. -- **A second gated check** verifies DEC-004's claim that the new - `max_output_tokens=1024` default leaves Gemini enough headroom (separate; see - § "Gemini check"). +- **A second gated check PASSED** — DEC-004's claim that the new + `max_output_tokens=1024` default leaves Gemini enough headroom holds at the + single-artifact scale: `gemini-2.5-flash` graded a verbose artifact at 1024 + tokens with no truncation degrade (see § "Gemini check"; full-fixture runs may + still want 4096 per #158). - **Default CI is untouched:** both checks are deselected by the `anthropic` / `gemini` markers in `pyproject.toml`'s `addopts` and skip-with-reason without keys. No live API call happens during normal validation. The concordance gate @@ -191,11 +193,15 @@ operator calibrated against. ### Gemini check The `gemini-2.5-flash` @ 1024-token no-truncation check -(`test_gemini_1024_no_truncation.py`) was **not run** in this session (no -`GOOGLE_API_KEY` available). It remains a separate maintainer step; DEC-004's -softened claim (1024 reduces but does not eliminate Gemini truncation at -full-fixture scale; the per-provider floors recommend 4096) already accounts for -the uncertainty. +(`test_gemini_1024_no_truncation.py`) was **run and PASSED** (2026-06-02, +`SF_RUN_GEMINI=1` + `GOOGLE_API_KEY`): grading a deliberately verbose artifact on +`gemini-2.5-flash` at the new `max_output_tokens=1024` default produced a clean +`GradingResult` with **no `score=None` truncation degrade**. So DEC-004's bump +(256 → 1024) is empirically sufficient at the **single-artifact-in-isolation** +scale. The full-fixture caveat still stands: #158 observed a minority of pairs +degrading at 1024/2048 across the whole Austin fixture, so the per-provider +floors recommend **4096** for Gemini-heavy runs — 1024 is a safe default-level +improvement, not a guarantee at volume. ## Disposition From 8445f25a847c8d330e693265a36d6d4af6b983af Mon Sep 17 00:00:00 2001 From: Wes Duenow Date: Tue, 2 Jun 2026 22:46:34 -0400 Subject: [PATCH 15/15] #187: address PR review (CodeRabbit + Copilot) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test_haiku_calibration: lazy _substrate import inside the test (no import-time sys.path mutation when deselected) — Copilot. - docstrings: 'never collects' -> 'deselected'; 'Haiku default' -> 'Haiku opt-in' across the calibration tests — Copilot. - llm-providers-ops: drafter row now honest (no per-provider drafter default; switching provider requires llm.model) with a footnote — Copilot. - plan doc: ```text fence on the dependency graph + blank line before the SKU table (markdownlint) — CodeRabbit; Phase -> complete. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/llm-providers-ops.md | 12 ++++- plans/super/187-fast-grade-defaults.md | 5 +- .../test_gemini_1024_no_truncation.py | 3 +- .../test_haiku_calibration.py | 48 +++++++++++-------- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/docs/llm-providers-ops.md b/docs/llm-providers-ops.md index b1377058..f262ae12 100644 --- a/docs/llm-providers-ops.md +++ b/docs/llm-providers-ops.md @@ -104,11 +104,19 @@ not a `Literal`). See [Adding a provider](#adding-a-provider) below. | **Server-side JSON mode** | n/a (Anthropic parser tolerant) | ✅ `response_format={"type":"json_object"}` | ✅ `response_mime_type="application/json"` | | **Pre-send `count_tokens` gate** | ✅ | ❌ (no SDK token-count API) | ❌ (deferred — Gemini has the API but we don't gate on it for cache parity) | | **`cache_ttl` config** | honoured (`"5m"` / `"1h"`) | silently ignored | silently ignored | -| **Default drafter model** | `claude-sonnet-4-6` | `gpt-4o` | unset | -| **Default grader model** | `claude-sonnet-4-6` (#187; Haiku is opt-in) | `gpt-4o-mini` (fast default) | `gemini-2.5-flash` (fast default) | +| **Default drafter model** | `claude-sonnet-4-6` | `claude-sonnet-4-6` † | `claude-sonnet-4-6` † | +| **Default grader model** | `claude-sonnet-4-6` (#187; Haiku is opt-in) | `gpt-4o-mini` (resolved per-provider, #187) | `gemini-2.5-flash` (resolved per-provider, #187) | | **Live smoke marker** | `@pytest.mark.anthropic` | `@pytest.mark.openai` | `@pytest.mark.gemini` | | **Live smoke env** | `ANTHROPIC_API_KEY` | `SF_RUN_OPENAI=1` + `OPENAI_API_KEY` | `SF_RUN_GEMINI=1` + `GOOGLE_API_KEY` | +† **The drafter has no per-provider default** — `DraftConfig.model` defaults to +`claude-sonnet-4-6` regardless of `llm.provider`. Switching the drafter to OpenAI +or Gemini therefore **requires setting `llm.model` explicitly** (e.g. +`llm.model: gpt-4o`); leaving it unset sends `claude-sonnet-4-6` to the other +vendor and fails at the API. This is the same footgun the grader's per-provider +resolver closed in #187 — but per #187 DEC-008 the drafter stayed out of scope, +so the drafter's per-provider default resolution is a future follow-up. + A `❌` on prompt caching does **not** mean the provider is unusable — it means every drafter call ships the full system + cached_block without a read discount. For a one-call-per-`generate` drafter this diff --git a/plans/super/187-fast-grade-defaults.md b/plans/super/187-fast-grade-defaults.md index ad9ecd72..1c88bdb1 100644 --- a/plans/super/187-fast-grade-defaults.md +++ b/plans/super/187-fast-grade-defaults.md @@ -7,7 +7,7 @@ - **Base branch:** `dev` (0.6.0.dev0). PRs target `dev`. - **Worktree:** `../worktrees/SignalForge/187-fast-grade-defaults` - **Branch:** `feature/187-fast-grade-defaults` -- **Phase:** devolved +- **Phase:** complete (implemented; PR #193) — calibration drove the final shape: **Sonnet stays the grade default, Haiku is opt-in** (the per-provider table is `PROVIDER_DEFAULT_MODELS`, anthropic→`claude-sonnet-4-6`). See `docs/research/187-haiku-calibration.md`. - **PR:** [#193](https://github.com/wjduenow/SignalForge/pull/193) (base `dev`) - **Sessions:** 1 (2026-06-02) @@ -71,6 +71,7 @@ This ticket flips the **grade-stage default** to a faster/cheaper model, and — - **Risk:** the unused draft placeholder uses the dated `claude-haiku-4-5-20251001`, which is NOT a pricing key. Adopting that exact string as the grade default would break `--estimate`/cost-rollup. The bare SKU `claude-haiku-4-5` matches pricing and the `sonnet-4-6`/`opus-4-7` convention. **Fastest/cheapest known SKU per provider (from pricing):** + | Provider | Cheapest known SKU | input $/MTok | output $/MTok | |---|---|---|---| | Anthropic | `claude-haiku-4-5` | 0.80 | 4.00 | @@ -248,7 +249,7 @@ No blockers remain after the refinement decisions below. ## Story dependency graph -``` +```text US-001 ─┬─> US-002 ─┬─> US-003 ─┐ │ ├─> US-005 ─┤ │ └─> US-006 ─┤ diff --git a/tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py b/tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py index 5693d582..b38766d2 100644 --- a/tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py +++ b/tests/research/187-haiku-calibration/test_gemini_1024_no_truncation.py @@ -16,7 +16,8 @@ * ``pytestmark = pytest.mark.gemini`` — the existing ``gemini`` marker, excluded from the default ``pytest`` run via :file:`pyproject.toml`'s - ``addopts``. Default CI never collects this test. + ``addopts``. Default CI **deselects** this test (it is imported during + collection but the test body never runs). * A runtime ``pytest.skip(...)`` when ``SF_RUN_GEMINI != "1"`` OR ``GOOGLE_API_KEY`` is unset/blank. diff --git a/tests/research/187-haiku-calibration/test_haiku_calibration.py b/tests/research/187-haiku-calibration/test_haiku_calibration.py index dc787fbb..aa606f56 100644 --- a/tests/research/187-haiku-calibration/test_haiku_calibration.py +++ b/tests/research/187-haiku-calibration/test_haiku_calibration.py @@ -22,8 +22,9 @@ * ``pytestmark = pytest.mark.anthropic`` — the existing ``anthropic`` marker, excluded from the default ``pytest`` run via :file:`pyproject.toml`'s - ``addopts = "... -m 'not anthropic ...'"``. Default CI never collects - this test. + ``addopts = "... -m 'not anthropic ...'"``. Default CI **deselects** + this test (the module is still imported during collection — hence the + lazy `_substrate` import below — but the test body never runs). * A runtime ``pytest.skip(...)`` when ``ANTHROPIC_API_KEY`` is unset (or blank) — so a maintainer who runs ``pytest -m anthropic`` without a key sees a clean skip-with-reason, not a noisy auth failure. @@ -42,20 +43,16 @@ import pytest -# Research-tier sibling import: the harness lives outside the importable -# package tree, so add this directory to ``sys.path`` for ``_substrate``. -sys.path.insert(0, str(Path(__file__).parent)) +from signalforge.grade import grade_artifacts +from signalforge.grade.config import GradeConfig -from _substrate import ( # noqa: E402 (path insert must precede import) - build_candidate, - build_model, - empty_prune_result, - expected_artifact_ids, - load_baseline, -) - -from signalforge.grade import grade_artifacts # noqa: E402 -from signalforge.grade.config import GradeConfig # noqa: E402 +# NOTE: the `_substrate` sibling import is intentionally LAZY (inside the test, +# after the skip) rather than module-level. The harness lives outside the +# importable package tree, so reaching `_substrate` needs a `sys.path` insert — +# doing that at import time would mutate `sys.path` during pytest collection +# even though this test is deselected by `-m 'not anthropic'` (Copilot PR +# review). Keeping it lazy means importing this module has no global side +# effects. pytestmark = pytest.mark.anthropic @@ -73,11 +70,11 @@ def _skip_reason() -> str | None: def test_haiku_grade_concordance_vs_sonnet_baseline(tmp_path: Path) -> None: - """Re-grade the pinned sample with the Haiku default; assert ≥ 85% concordance. + """Re-grade the pinned sample with the Haiku opt-in; assert ≥ 85% concordance. - Builds the resolved Haiku-default :class:`GradeConfig` (``model`` - resolves to ``claude-haiku-4-5`` via the #187 US-002 provider - fast-model resolver; ``max_output_tokens=1024``), grades the pinned + Builds the Haiku opt-in :class:`GradeConfig(model="claude-haiku-4-5")` + (the anthropic *default* is Sonnet post-calibration — this gate is why; + ``max_output_tokens=1024``), grades the pinned candidate over the default four-criterion rubric, joins each :class:`GradingResult` to the committed Sonnet baseline by ``(artifact_id, criterion_id)``, computes per-criterion pass/fail @@ -93,6 +90,17 @@ def test_haiku_grade_concordance_vs_sonnet_baseline(tmp_path: Path) -> None: if reason: pytest.skip(reason) + # Lazy sibling import (after the skip) — keeps `sys.path` un-mutated at + # collection time when this test is deselected (Copilot PR review). + sys.path.insert(0, str(Path(__file__).parent)) + from _substrate import ( + build_candidate, + build_model, + empty_prune_result, + expected_artifact_ids, + load_baseline, + ) + model = build_model() candidate = build_candidate() prune_result = empty_prune_result(model) @@ -187,7 +195,7 @@ def test_haiku_grade_concordance_vs_sonnet_baseline(tmp_path: Path) -> None: assert rate >= _CONCORDANCE_THRESHOLD, ( f"Haiku concordance {rate:.1%} below the {_CONCORDANCE_THRESHOLD:.0%} " f"decision rule ({agreements}/{comparable} agreements). " - "The Haiku default does NOT grade concordantly with the Sonnet " + "The Haiku opt-in does NOT grade concordantly with the Sonnet " "baseline on this sample; record the discordances in " "docs/research/187-haiku-calibration.md and reconsider the default." )