From 6d6cb91ef6d1214fc9f1784cf13f78e01e42955a Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 21:47:22 -0700 Subject: [PATCH 01/24] 6: super-plan for prune engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase-4 plan for issue #6 — drop always-pass and known-clean-fail candidate tests against warehouse data. Covers the load-bearing "signal over volume" commitment from CLAUDE.md. Architecture review surfaced two pivots resolved in DEC-012/DEC-013: the deterministic-sample predicate (TO_JSON_STRING(t)) reads every column, so US-003 live-verifies the cost model against bigquery- public-data before locking strategy; the warehouse adapter's QueryJobConfig.job_timeout_ms plumbing folds in as US-002 (~10 LOC) so the per-test budget actually enforces. 28 decisions, 16 stories (14 implementation + Quality Gate + Patterns & Memory). Stories trace to DEC-### and embed TDD on every logic-shaped surface (compiler, engine, audit, errors, models, config). Co-Authored-By: Claude Opus 4.7 (1M context) --- plans/super/6-prune-engine.md | 575 ++++++++++++++++++++++++++++++++++ 1 file changed, 575 insertions(+) create mode 100644 plans/super/6-prune-engine.md diff --git a/plans/super/6-prune-engine.md b/plans/super/6-prune-engine.md new file mode 100644 index 00000000..901ac031 --- /dev/null +++ b/plans/super/6-prune-engine.md @@ -0,0 +1,575 @@ +# Issue #6 — Test prune engine: drop always-pass and known-clean-fail tests + +## Meta + +- **Ticket:** [#6](https://github.com/wjduenow/SignalForge/issues/6) +- **Branch:** `feature/6-prune-engine` (off `dev`) +- **Worktree:** `/home/wesd/dev/worktrees/SignalForge/feature/6-prune-engine` (created via `git worktree add`) +- **Phase:** detailing (Phases 1–3 complete; awaiting story-set sign-off) +- **Sessions:** 1 (started 2026-04-30) +- **Plan author:** Claude Code (Opus 4.7, 1M context) +- **Milestone:** v0.1 (the **load-bearing** ticket — Architectural Commitment #1; gates #7 grader and #8 diff renderer) +- **Labels:** `evaluation` + +## Discovery + +### Ticket summary + +Run every candidate test against warehouse data and drop the ones with no signal. From the issue body: + +1. `signalforge.prune.prune_tests(candidates, adapter) -> PruneResult` +2. Each candidate test compiled to SQL via the adapter's dialect helper +3. Test "passes always" = zero failing rows on the sampled set OR zero failing rows on the full table (configurable) +4. Drop reasons recorded: `always-passes`, `requires-future-data`, `failed-on-known-clean-data`, `kept` +5. Conservative defaults: keep tests we cannot evaluate (don't silently drop) +6. Per-test runtime budget; tests over budget marked `kept-without-evidence` +7. Unit tests with synthetic data fixtures; integration test on a public BQ dataset + +The ticket explicitly notes: "This is the load-bearing premise of the project. If pruning hits >80% of candidates the story works; <20% means we're regenerating nothing useful and the design needs rework." + +This is the first ticket to actually execute against the warehouse with data-driven decisions. Every later stage (#7 grader operates on kept tests; #8 diff renderer operates on the kept/dropped split; #9 CLI wires prune into the flow) consumes the `PruneResult` contract. + +### Codebase findings (Subagent B — directly verified) + +**Upstream surfaces already shipped:** + +- **`signalforge.draft.CandidateSchema`** (`src/signalforge/draft/models.py:176-200`) — frozen Pydantic v2; carries `schema_version: Literal[1]`, `name`, `description`, `rationale: str | None`, `columns: tuple[CandidateColumn, ...]`, `tests: tuple[CandidateTest, ...]` (model-level tests). `CandidateColumn` (`models.py:151-173`) carries `name`, `description`, `rationale`, `tests: tuple[CandidateTest, ...]`, `meta: dict[str, Any] | None`. The drafter's anchor-contract validator (`signalforge.draft.parser._validate_anchor_contract`) guarantees every `CandidateColumn.tests[].column == column.name` and every test's `column` references a real model column — the prune layer can assume this holds on input. +- **`CandidateTest` discriminated union** (`models.py:133-148`) — exactly four variants: + - `CandidateTestNotNull(type="not_null", column, rationale?)` + - `CandidateTestUnique(type="unique", column, rationale?)` + - `CandidateTestAcceptedValues(type="accepted_values", column, values: tuple[str, ...], rationale?)` + - `CandidateTestRelationships(type="relationships", column, to: str, field: str, rationale?)` + No `where:` modifier. No dbt-utils test types. v0.1 prune compiles exactly these four; anything else is a parser-rejected case upstream. +- **`signalforge.warehouse.WarehouseAdapter`** ABC (`base.py:31-121`) exposes everything prune needs: + - `dialect() -> Dialect` — returns `BIGQUERY_DIALECT(quote_char="`", supports_qualify=True, ...)` in v0.1 + - `run_test_sql(sql: str, *, capture_failures: int = 0) -> TestResult` — the prune seam. SQL contract: a single SELECT returning failing rows; the adapter wraps with `COUNT(*) AS failures` (+ `ARRAY_AGG(...)` if `capture_failures>0`) and validates pre-flight via `_sql_safety.validate_test_sql`. Returns `TestResult(passed: bool, failure_count: int, sample_failures: list[dict] | None, row_schema: list[tuple[str, str]] | None)`. **The adapter already exists; prune does NOT need to add a new method.** + - `sample_rows(table, n, *, partition_filter=None) -> list[dict]` — deterministic hash-mod sampling (DEC-006 of #3). Prune uses this only if it materialises a sample to a temp table for batched evaluation; the typical case is "send the test SQL with the warehouse's hash-mod predicate inline." + - `column_stats(table, column) -> ColumnStats` — context-manager-batched aggregates. Prune may call to detect "warehouse-enforced uniqueness" hints (e.g., `count == distinct → unique always-passes`). +- **`TableRef.from_model(model: Model) -> TableRef`** (`models.py:149`) — the way prune resolves a candidate to a warehouse identity. Raises `ManifestProjectNotFoundError` / `ManifestSchemaNotFoundError` if model lacks `database` / `schema`. +- **`signalforge.warehouse.errors`** — 15 typed `WarehouseError` subclasses. Prune catches `BytesBilledExceededError`, `TableNotFoundError`, `ColumnNotFoundError`, `QuerySyntaxError`, `SamplingRequiresPartitionFilterError`, `UnknownTableSizeError` to route to drop reasons (`kept-without-evidence`). +- **`signalforge.manifest.Manifest`** (`src/signalforge/manifest/models.py`) — `models: dict[str, Model]` keyed by `unique_id`. Prune needs the manifest to resolve `relationships(to: model_name, field)` against parent models — the parent must exist in the manifest, else `requires-future-data`. + +**No prior prune code.** No test-compiler exists. No usage of `run_test_sql` outside of warehouse unit/integration tests. Prune is the first production caller. + +**Test fakes pattern** — `tests/safety/_fake_adapter.py::FakeAdapter` and `tests/warehouse/_fake.py::FakeBigQueryClient` both expose `expect_*` APIs. `FakeAdapter` deliberately raises `AssertionError("FakeAdapter does not support run_test_sql ...")` — the safety layer doesn't run tests. Prune tests need a `FakePruneAdapter` (preferred: extend `FakeAdapter` with `expect_run_test_sql(matching=..., returns=TestResult|Exception)`) **OR** test prune end-to-end against `BigQueryAdapter(client=FakeBigQueryClient(...))` (the warehouse-level fakes are already proven and let us exercise the SQL the test compiler emits). + +**No safety-layer dependency.** Prune runs SQL into the warehouse, not into an LLM. `LLMRequest` / `AuditEvent` are upstream concerns. Prune does NOT call `build_llm_request` — the AST audit-completeness scan (`tests/safety/test_public_api.py`) gates `LLMRequest` construction to `signalforge.safety.request` only. + +**No anthropic dependency.** Prune does not call the LLM. The drafter (#5) draws rationale text in advance; prune writes its own decision-time rationale (the drop reason + failure count + scope), not LLM-generated. + +**Validation command** (per `CLAUDE.md`): `pip install -e ".[dev]" && ruff check . && ruff format --check . && pyright && pytest`. Quote `".[dev]"` — `[dev]` is a glob in zsh. + +**Sibling open issues** confirm scope walls: +- **#7 (grader)** — consumes `PruneResult.kept` for rubric scoring. Prune's output is grader's input. +- **#8 (diff renderer)** — consumes `PruneResult` to render the kept/dropped table + per-artifact "why" line. The drop-reason taxonomy and the per-decision rationale text are renderer-load-bearing. +- **#9 (CLI)** — wires `signalforge generate` end-to-end: load manifest → safety policy → draft → **prune** → render → write. Constructs the adapter via `WarehouseAdapter.from_profile`; injects into prune. +- **#10 (smoke test)** — exercises the whole pipeline against `bigquery-public-data`. Includes prune. +- **#7/#8/#9 are downstream consumers; their behaviour is stable as long as the `PruneResult` contract holds.** + +### Domain research (Subagent D — dbt test compilation + statistical pruning) + +**Failing-rows SQL patterns** (BigQuery dialect, backtick-quoted identifiers): + +| Test | Failing-rows SELECT | +|------|--------------------| +| `not_null(col)` | `SELECT \`col\` FROM \`p.d.t\` WHERE \`col\` IS NULL` | +| `unique(col)` | `SELECT \`col\` FROM \`p.d.t\` WHERE \`col\` IS NOT NULL GROUP BY \`col\` HAVING COUNT(*) > 1` | +| `accepted_values(col, values)` | `SELECT \`col\` FROM \`p.d.t\` WHERE \`col\` IS NOT NULL AND \`col\` NOT IN ({values})` | +| `relationships(child_col, to=parent, field)` | `SELECT child.\`child_col\` FROM \`p.d.t_child\` AS child LEFT JOIN \`p.d.t_parent\` AS parent ON child.\`child_col\` = parent.\`field\` WHERE child.\`child_col\` IS NOT NULL AND parent.\`field\` IS NULL` | + +**Critical NULL-exclusion conventions** (must match dbt-core verbatim, else prune verdicts diverge from `dbt test` runtime verdicts): + +- `unique` excludes NULLs (multiple NULLs are NOT considered duplicates) +- `accepted_values` excludes NULLs (combine with `not_null` if NULL-disallowed) +- `relationships` excludes NULL on the child side (orphan-NULL is a `not_null` concern) + +**Always-pass targets** (the prune's high-confidence drops): `not_null` on a `REQUIRED`-mode column; `unique` on a primary-key surrogate (`GENERATE_UUID()`, `FARM_FINGERPRINT()`, `ROW_NUMBER()`); `accepted_values` against a column built from a closed `CASE WHEN ... THEN 'a' ... END`. + +**Sample vs. full evaluation:** +- Sample: ~free in BigQuery (column footprint × sample bytes). False-negative rate = rule of three (≤ 3/N at 95% CI for "no failures observed in N rows"). 100k-row sample → ≤ ~3-in-100k false negative. +- Full: correct but bytes-scanned proportional to column footprint × table size. Already partition-filter-gated by adapter's `SamplingRequiresPartitionFilterError` for tables ≥ 100M rows. + +**Per-test budget mechanism** (BigQuery): +- `QueryJobConfig.job_timeout_ms` is the cost-aware path — server-side cancellation. +- `QueryJob.result(timeout=...)` is Python-side wait only; query keeps running and bills. +- `signal.alarm` is unsafe (main-thread-only, no Windows, conflicts with pytest). +- The adapter's existing `_default_job_config` (DEC-015 of #3) is the single seam for `QueryJobConfig` — extending with `timeout_ms` is the right shape. + +**Cost-saving optimisations:** +- **Per-column COUNTIF batching** — one query, multiple verdicts: `SELECT COUNTIF(col IS NULL) AS nn, COUNTIF(col NOT IN (...)) AS av FROM ...`. Cost win at scale; brittle to a single SQL syntax error (one bad test breaks the batch). +- **Sample-materialised temp table** — write the deterministic sample once into a session temp table; query it N times for free. BigQuery temp tables are session-scoped + cached for ~24h. +- **dbt itself does neither** — one query per test. Whatever batching SignalForge does is a project-specific differentiator. + +**"Known-clean" evidence channels** (the four-drop-reason taxonomy says `failed-on-known-clean-data` requires evidence the data is clean; how do we know?): +- A — **dbt `run_results.json` history**: extends manifest loader to parse `target/run_results.json`; checks "model has been built successfully N times AND its dbt-defined tests have passed M times." Technically richest. Scope creep against #2. +- B — **User-marked trusted models in `signalforge.yml`**: `prune.trusted_models: ["model.shop.dim_customers", "model.shop.fact_orders"]`. Explicit. Cheapest v0.1. +- C — **Time-since-deploy heuristic**: messy; relies on run-history substrate; conflates "no failures" with "no test was looking." +- D — **Defer the reason entirely for v0.1**: emit only `always-passes`, `requires-future-data`, `kept`, `kept-without-evidence`. The ticket lists `failed-on-known-clean-data` but doesn't specify the evidence channel; deferring is conservative and within "keep tests we cannot evaluate." (See Q1.) + +**Determinism + decision identity:** +- Adapter's hash-mod sampling is already deterministic. Prune adds zero non-determinism (no `CURRENT_TIMESTAMP`, no `RAND()`, no unseeded RNG, no dict-iteration-order-dependent SQL). +- Mirror existing precedent: `prune_decision_id = blake2b-8(canonical(model_unique_id, test_type, test_args, scope, sample_bucket))` — matches `LLMResponseEvent`'s `parsed_schema_hash` family. + +**Prior-art cross-reference:** +- **dbt-core**: failing-rows SQL pattern (we copy verbatim). +- **great-expectations**: validation-result document idea (rich per-test verdict with row counts, observed values, decision rationale) — pattern to borrow for `PruneDecision`. +- **dbt-codegen**: rule-based scaffolder; complementary; no overlap. +- **Recce / dbt-checkpoint / dbt-coverage**: complementary; no overlap. +- **Nobody else runs candidate tests against real warehouse data and drops noise.** This is the differentiator the ticket protects. + +### Project rules (`.claude/rules/`) audit (Subagent C) + +`.claude/rules/` has seven files (no `workflow-project.md`). Phase 4 stories validate against: + +1. **safety-layer.md** (load-bearing) — DEC-011 fail-closed audit semantics if prune writes a `PruneEvent` JSONL (no try/except inside writer; size cap before file open; propagation IS the defence). DEC-022 ANSI-safe lazy-format JSON logging — every `_LOGGER.{info,warning,debug,error}` in `signalforge.prune.*` uses lazy-format with `json.dumps()` for any user-controlled string; **never** f-string. Grep gate at `tests/prune/test_logger_grep_gate.py`. DEC-025 `signalforge.yml` namespace — claim a NEW top-level `prune:` key; do NOT pile under `safety:`. +2. **manifest-readers.md** — Pydantic v2 frozen + `extra="ignore"` on read-back models; pair with `extra="forbid"` drift detector. Typed exceptions subclass a module base (`PruneError`) with `remediation: str` kwarg; `__str__` renders both message and `↳ Remediation:` line. No logs in stage-0 modules; observability lives at the executor seam where stage labels are known. +3. **python-build.md** — src layout under `src/signalforge/prune/`. No new wheel target; existing `packages = ["src/signalforge"]` covers the subpackage. No `tests/__init__.py`. +4. **testing-signal.md** (load-bearing) — every test must be capable of failing. Strict markers — both settings. **Hand-rolled fakes only** (no `MagicMock`, no `pytest-bigquery-mock`). Pair every `extra="ignore"` model with a one-off `extra="forbid"` drift detector against a committed JSON fixture. +5. **warehouse-adapters.md** — call the ABC, not concrete classes. Trust adapter-validated identifiers. Trust adapter's `use_query_cache=False` + `maximum_bytes_billed` defaults. Inherit deterministic-sampling fail-loud behaviour. **Do NOT bake BigQuery-isms into prune core** — Architectural Commitment #3. +6. **llm-drafter.md** — applicable patterns even though prune doesn't call an LLM: + - Module-level `_sleep` / `_rand_uniform` aliases for test-time injection (DEC-004) — relevant if prune has retry / wait logic. + - Whole-output fail-loud collection (DEC-003 / DEC-022) — if prune validates that all candidate tests reference real columns, collect every violation, never short-circuit. Mirrors the anchor-contract pattern. + - Fail-closed response audit (DEC-006 / DEC-008 / DEC-013) — applies if prune writes a JSONL audit (Q3 below). + - AST audit-completeness scan extension (DEC-013) — if prune emits a `PruneEvent` type, gate construction to `signalforge.prune.audit` via `tests/test_audit_completeness.py`. +7. **ci-supply-chain.md** — no new workflow needed; prune runs in the existing `pytest` step (Python 3.11, BigQuery integration tests behind `-m bigquery` marker, mirrored as `-m prune_integration` if needed). + +### CLAUDE.md commitments that bite this ticket + +- **#1 Signal over volume.** This is THE ticket. The prune step IS the signal-vs-volume gate. The drafter (#5) intentionally over-generates; prune intentionally drops noise. A prune layer that drops < 20% of candidates indicates the design failed; > 80% indicates it's working. We design conservatively (kept-without-evidence is a real outcome, not a fallback) but lean aggressive on the always-pass branch. +- **#3 Warehouse-agnostic.** Prune calls `WarehouseAdapter.run_test_sql(sql)` and `WarehouseAdapter.dialect()`. The dialect provides quote_char + identifier_case so prune's compiler can produce per-warehouse SQL. No `from google.cloud import bigquery` in prune code. v0.2 Snowflake/Postgres adapters slot in without prune changes. +- **#5 Explainable diffs.** Every kept/dropped test ships with a structured `PruneDecision` carrying `decision: kept | dropped`, `reason: always-passes | requires-future-data | failed-on-known-clean-data | kept | kept-without-evidence`, `failures: int`, `sampled_rows: int`, `scope: sample | full`, `elapsed_ms: int`, `compiled_sql_hash: str`, the literal SQL (for `--explain`), and the human-readable "why" line the diff renderer (#8) consumes. +- **Roadmap anchor.** v0.1 = single-model draft + warehouse prune, BigQuery only. Multi-warehouse prune is v0.2 (but the design must NOT close it off — keep the dialect seam clean). + +### Out of scope (explicit) + +- **The CLI** — issue #9. This ticket exposes `prune_tests(...)` plus the `PruneResult` contract. CLI wiring is #9. +- **The grader** — issue #7. Prune outputs kept tests with reasons; grader scores them. +- **The diff renderer** — issue #8. Prune returns typed `PruneResult`; rendering to a human-readable kept/dropped table is #8. +- **dbt-utils test types** — `dbt_utils.unique_combination_of_columns`, `dbt_utils.accepted_range`, etc. The drafter's `CandidateTest` union has exactly four variants; prune compiles exactly four. v0.2 territory. +- **`where:` test modifier** — dbt-core supports `tests: [- not_null: where: "is_active = true"]`. The drafter doesn't emit it today; prune doesn't consume it. v0.2. +- **`severity: warn`** — dbt severity escalation; v0.2 alongside `mostly:` for confidence-based passing. +- **Historical always-pass** — running tests against multiple `run_results.json` snapshots to assert "never failed in last N runs." v0.2. +- **Multi-model prune in one call** — v0.1 is single-model (drafter is single-model). Batching multiple models is v0.2 if/when the drafter goes multi-model. +- **`prune_decision_id`-keyed checkpoint / resumption** — long-running prune runs that resume from disk after a crash. v0.2. +- **Snowflake/Postgres dialect support** — adapter is BigQuery-only in v0.1. The prune compiler uses `Dialect.quote_char` + `Dialect.identifier_case` to stay portable, but no Snowflake-specific SQL paths. +- **Confidence intervals on always-pass** — surfacing "≤ 3/N upper-bound failure rate at 95% CI" on the decision record. Nice to have; v0.2. +- **`mostly:` thresholds** — great-expectations-style "test passes if ≥ 99% of rows satisfy." v0.2. +- **LLM-generated rationale on `kept` decisions** — the grader (#7) does this, not prune. + +### Phase 1 housekeeping defaults (set unless flagged in Phase 2/3) + +- New subpackage at `src/signalforge/prune/`: + - `__init__.py` re-exports the public surface + - `engine.py` — `prune_tests(...)` entry + the per-test executor + - `compiler.py` — `compile_test(test, table_ref, dialect, manifest) -> str` (the four failing-rows patterns) + - `models.py` — `PruneResult`, `PruneDecision`, `DropReason` literal, `Scope` literal, `PruneConfig` + - `errors.py` — `PruneError` hierarchy + - `audit.py` — fail-closed `PruneEvent` JSONL writer (if Q3 = yes) + - `config.py` — `load_prune_config(path) -> PruneConfig` +- Public API surface: + - `signalforge.prune.prune_tests(candidates: CandidateSchema, model: Model, manifest: Manifest, adapter: WarehouseAdapter, *, config: PruneConfig | None = None) -> PruneResult` + - `PruneResult(model_unique_id: str, kept: tuple[KeptTest, ...], dropped: tuple[DroppedTest, ...], decisions: tuple[PruneDecision, ...], elapsed_ms: int, signalforge_version: str, prune_schema_version: int = 1)` + - `PruneDecision(test_anchor: str, test_type: str, test_args: dict, decision: Literal["kept","dropped"], reason: DropReason, failures: int, sampled_rows: int | None, scope: Scope, elapsed_ms: int, compiled_sql_hash: str, compiled_sql: str, why: str)` + - `DropReason = Literal["always-passes", "requires-future-data", "failed-on-known-clean-data", "kept", "kept-without-evidence"]` + - `Scope = Literal["sample", "full"]` +- `signalforge.yml` config block: top-level key `prune:`; fields `scope: "sample" | "full"` (default `"sample"`), `sample_size: int` (default 100_000), `test_timeout_seconds: int` (default 30), `total_budget_seconds: int` (default 600), `capture_failure_rows: int` (default 3 — matches existing `TestResult.sample_failures` capture cap). Outer wrapper `extra="ignore"`; inner `PruneConfig` `extra="forbid"`. +- `PruneConfig` and config-shaped models use `extra="forbid"` (typo-fail-loud); read-back models (`PruneDecision`, `PruneResult`, `PruneEvent`) use `extra="ignore"` paired with one-off `extra="forbid"` drift detectors. +- Identifier validation reuse: prune's compiler does NOT re-validate `TableRef.dataset` / `TableRef.name` (already validated at `TableRef.__init__`). It DOES `repr()`-quote any value-list (`accepted_values.values`) before SQL interpolation; the `accepted_values` interpolation seam needs explicit string-escaping to prevent quote-injection (`O'Brien` → `'O\'Brien'`). +- Tests live under `tests/prune/`. No `__init__.py`. Synthetic-fixture unit tests use a hand-rolled `expect_run_test_sql(matching=..., returns=...)` extension to `FakeBigQueryClient` (NOT to `FakeAdapter` — the safety fake's `run_test_sql` raises by design). Integration test under `@pytest.mark.bigquery` (existing marker) hits `bigquery-public-data.iowa_liquor_sales.sales` (or similar small public table). +- AST audit-completeness scan extended to gate `PruneEvent` construction to `signalforge.prune.audit` only. +- Logger grep-gate scan extended to `src/signalforge/prune/`. +- `signalforge.draft` — adapter's pyright shim model is mirrored if prune adds anything that needs pyright-noise containment. Anticipated: nothing new (we reuse the adapter's `run_test_sql` seam). + +## Phase 1 Scoping Questions + +Five questions. All scope-shaping. Answer with letters; "I'll write more" / free-text accepted. + +--- + +**Q1. The `failed-on-known-clean-data` drop reason — evidence channel.** + +The ticket lists this as one of four drop reasons: a candidate test ran, returned ≥ 1 failure, AND we have evidence the data is clean ⇒ the test is buggy. How do we know data is clean? + +- **A.** Defer the reason for v0.1. Emit only `always-passes`, `requires-future-data`, `kept`, `kept-without-evidence`. Most conservative; ships fastest. v0.2 picks the channel. +- **B.** User-marked trusted models in `signalforge.yml` (`prune.trusted_models: [...]`). Explicit, cheapest v0.1, no new artefact loaders. The user opts-in per model; non-trusted models never emit `failed-on-known-clean-data`. +- **C.** Parse `target/run_results.json` for dbt run/test history. Technically richest — "this model has built successfully 30 times and its dbt-defined tests have all passed" is strong evidence. Extends issue #2's manifest loader scope significantly. +- **D.** Aggressive default: any candidate test that fails on a tiny fraction of rows (e.g., < 0.1%) is presumed buggy. Risky — silently drops legitimate findings. **Not recommended.** + +--- + +**Q2. Sample-vs-full evaluation default.** + +The ticket: "Test 'passes always' = zero failing rows on the sampled set OR zero failing rows on the full table (configurable)." + +- **A.** Sample-mode default; user opts up to full per run via `prune.scope: full` in `signalforge.yml`. Cheap, fast, ~3-in-N false-negative rate (rule of three). The deterministic-sample contract from #3 already gives reproducibility. +- **B.** Full-mode default; user opts down to sample for cost. Correct, but expensive at scale (every test reads the full column on every run). Already-partition-filter-gated by `SamplingRequiresPartitionFilterError` for tables ≥ 100M rows. +- **C.** Hybrid: sample-mode for the always-pass claim + full-mode confirmation only for tests the prune is about to drop. Spends bytes only on high-value drop decisions; defends against false-positives in always-pass. More complex. + +--- + +**Q3. Decision audit — fail-closed JSONL or in-memory result only?** + +Safety (#4) and draft (#5) both write fail-closed JSONL audits adjacent to their stage outputs (`audit.jsonl`, `llm_response.jsonl`). The justification: "every privacy-relevant action leaves a durable receipt." Prune's actions are warehouse cost decisions, not privacy decisions — but the explainability principle (Commitment #5) suggests every kept/dropped test ships with a "why" the reviewer can replay. + +- **A.** Fail-closed `prune.jsonl` audit at the same path as the safety/draft audits. Mirrors precedent. Adds a small amount of complexity (size-cap, fsync, AST scan extension). Reviewers can pull the JSONL and replay any decision. +- **B.** In-memory `PruneResult` only; serialisation to disk is the diff renderer's job (#8). Simpler v0.1. Trade-off: if the renderer doesn't serialise everything, the audit trail dies at process exit. +- **C.** Both — in-memory `PruneResult` for #8 to render + JSONL audit for the explainability/replay path. Defaults to "audit on" with `prune.audit.enabled: false` opt-out. + +--- + +**Q4. Test batching strategy.** + +Per-test cost is dominated by column footprint × bytes scanned. Without batching, N tests on the same model = N column scans. + +- **A.** One query per test (dbt's pattern). Simplest. Most expensive at scale. Works fine for v0.1's single-model unit-of-work. +- **B.** Per-column COUNTIF batching (`SELECT COUNTIF(col IS NULL) AS nn, COUNTIF(col NOT IN (...)) AS av FROM ...`). One read, multiple verdicts. Cost win; brittle (one syntax error breaks the batch — mitigation: per-test fallback). Doesn't help `relationships` (that's a JOIN, batches with other `relationships` on the same parent only). +- **C.** Sample-materialised temp table — write the deterministic sample once into a session temp table; query it N times for free. BigQuery temp tables are session-scoped and free for reads (you pay only the materialisation cost). Larger one-time cost but free per-test thereafter. Cleanest for #1's "signal over volume" because pruning N candidates costs the same as pruning 1. +- **D.** Defer all batching to v0.2. v0.1 ships A. Acceptable given the v0.1 unit-of-work is one model. + +--- + +**Q5. Per-test runtime budget — server-side, Python-side, or both?** + +`kept-without-evidence` is the budget-exceeded outcome. + +- **A.** Server-side `QueryJobConfig.job_timeout_ms` only. BigQuery cancels the job; minimal Python complexity. Tail bytes-billed possible (BigQuery cancels opportunistically). +- **B.** Python-side `concurrent.futures.ThreadPoolExecutor.submit(...).result(timeout=...)` with explicit `query_job.cancel()` in the `finally`. Tighter wall-clock control; bills full query if cancellation is slow. +- **C.** Both — server-side cap + Python-side circuit-breaker (e.g., server cap 30s, Python cap 35s with cancel). Defence-in-depth against an SDK call that hangs. More moving parts. + +--- + +### Locked answers (2026-04-30, "keep defaults") + +| Q | Pick | Implication | +|---|------|-------------| +| Q1 | **B** — User-marked trusted models in `signalforge.yml` | `prune.trusted_models: [...]` opts-in per model. Non-trusted models never emit `failed-on-known-clean-data`; their failure-on-data outcomes route to `kept`. Cheapest v0.1; explicit; no `run_results.json` loader scope-creep. | +| Q2 | **A** — Sample-mode default; `prune.scope: full` opt-up | Inherits adapter's deterministic hash-mod sampling. `sample_size: 100_000` default. Per-test override field on `PruneConfig`. | +| Q3 | **A** — Fail-closed `prune.jsonl` audit | Mirrors safety + draft precedent (`O_APPEND \| O_CREAT \| 0o600`, fsync, size cap before open, propagation IS the defence). New AST scan gate for `PruneEvent` construction in `signalforge.prune.audit` only. | +| Q4 | **A** — One query per test (defer batching to v0.2) | Simplest. v0.1 unit-of-work is one model — N is bounded by candidate count per model (typically 5–30 tests). COUNTIF batching + temp-table materialisation are v0.2 candidates. | +| Q5 | **A** — Server-side `QueryJobConfig.job_timeout_ms` only | BigQuery cancels server-side. Default `test_timeout_seconds: 30`. Extend `_default_job_config` (warehouse-adapters.md DEC-015) with a `timeout_ms` kwarg — that's the existing single-seam for query config. | + +These five choices keep v0.1 tight: no new artefact loader, no new optimisation engine, no defence-in-depth complexity beyond what's already proven in adjacent layers. + +## Architecture Review + +Six parallel reviews (security, performance, data-model, API-design, observability, testing). Verdicts: + +| Area | Pass | Concern | Blocker | +|------|------|---------|---------| +| Security | 12 | 0 | 0 | +| Performance | 7 | 4 | 1 | +| Data Model | 9 | 3 | 3 | +| API Design | 7 | 3 | 0 | +| Observability | 12 | 1 | 0 | +| Testing | 10 | 1 | 0 | + +### Blockers (must resolve before refinement) + +- **AR-B1 (Performance) — `TO_JSON_STRING(t)` reads ALL columns of the row.** The adapter's deterministic sampler is `WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), bucket) < 1`. `TO_JSON_STRING(t)` serialises the entire row — BigQuery cannot column-prune through a function argument. **Implication:** sample-mode is NOT the cheap "read 1 column × 100k rows" the Phase 1 cost model assumes; it's "read ALL columns × full table" to compute the predicate. The Phase 1 24MB / 30 tests / ~$0.0001 figure is wrong by potentially 50–500×. Either (a) the adapter MUST materialise the deterministic sample once into a session-scoped temp table and prune queries that (this is Q4=C from Phase 1 — we deferred it; reopening); (b) prune issues SQL inline, accepting the full-row cost; or (c) the adapter exposes a "sample column subset" mode. Live verification against `bigquery-public-data` recommended before lock. + +- **AR-B2 (Performance) — Adapter's `_default_job_config` does not yet accept `timeout_ms`.** Q5=A (server-side `QueryJobConfig.job_timeout_ms`) requires extending `make_query_job_config` (`src/signalforge/warehouse/adapters/_client.py:76`) and `_default_job_config` (`src/signalforge/warehouse/adapters/bigquery.py:221`) with a `timeout_ms` kwarg threaded into `QueryJobConfig.job_timeout_ms`. **This is a prerequisite story (warehouse adapter extension) before prune can enforce per-test budgets.** Adds one upstream story (US-W1). + +- **AR-B3 (Data Model) — Drift detectors are mandatory, not optional.** Each `extra="ignore"` read-back model (`PruneResult`, `PruneDecision`, `PruneEvent`) MUST pair with a one-off `Strict(extra="forbid")` validated against a committed JSON fixture in `tests/prune/test_drift_detector.py`. This is non-negotiable per `manifest-readers.md` and `safety-layer.md`; a missing detector loses forward-compat safety. (Surfaced as a BLOCKER because the Phase 1 housekeeping defaults didn't make it explicit; locking it as a story now.) + +### Concerns (worth resolving in refinement) + +**Performance** +- **AR-C1 — `relationships` parent join cost in sample-mode.** The failing-rows pattern joins the sampled child to the **full parent**. On large parents this can dominate cost. Document the asymmetry in ops; defer "sample parent too" to v0.2. +- **AR-C2 — Total-budget semantics need precision.** When `total_budget_seconds` exceeds, do we (a) cancel in-flight + skip remaining (each marked `kept-without-evidence`), or (b) let in-flight finish + skip only not-yet-started? Recommend (a) for tighter wall-clock guarantees. +- **AR-C3 — `partition_filter` knob missing from `PruneConfig`.** Full-mode on tables ≥ 100M rows REQUIRES a partition filter (adapter raises `SamplingRequiresPartitionFilterError`). User has no config seam to provide one. Add `prune.partition_filter: dict | None` (rendered to `PartitionFilter` ADT — typed value object, not raw string) or document that full-mode on huge tables is unsupported in v0.1. + +**Data Model** +- **AR-C4 — `KeptTest` / `DroppedTest` redundancy.** Phase 1 housekeeping mentioned both `kept`/`dropped` tuples AND a unified `decisions`. **Drop the type duplication.** `PruneResult` carries one `decisions: tuple[PruneDecision, ...]` plus computed properties `.kept_decisions` / `.dropped_decisions` (filter by `decision` field). One source of truth; one drift-detector burden. +- **AR-C5 — `test_args: dict` is loose.** A v0.1 reader can't validate v0.2 test-arg shapes. Recommendation: serialise the original `CandidateTest` discriminated union directly (`test: CandidateTest`) so the typed shape round-trips; v0.1 readers fail loud on a v0.2 test type (acceptable — they shouldn't process unknown drop reasons either). +- **AR-C6 — Hash conventions.** Mixed precedent in repo: `safety.policy_hash` is `sha256(...)[:16]` (16 hex); `draft` uses `blake2b(..., digest_size=8).hexdigest()` (16 hex). Lock: `compiled_sql_hash` → blake2b-8 (matches draft); `config_hash` → sha256-16 (matches safety policy_hash). Document both helpers in `prune/audit.py`. + +**API Design** +- **AR-C7 — Signature order divergence from drafter.** Drafter is `(model, adapter, policy, manifest, *, config)`. Prune's `(candidates, model, manifest, adapter, *, config)` breaks the visual pattern. Recommendation: align to `(model, adapter, candidates, manifest, *, config)` — model+adapter front-paired (same scaffolding the CLI threads through stages), then primary input (candidates), then context (manifest), then config. +- **AR-C8 — `PruneCompilerError` is misnamed.** Compilation always succeeds; failures (e.g., `relationships(to: unknown_model)`) emit a drop reason, not an exception. **Drop the class.** Final error hierarchy: `PruneError` → `PruneConfigError` → `PruneTrustedModelNotFoundError`; `PruneTimeoutError`, `PruneAuditWriteError`, `PruneAuditRecordTooLargeError`. Six classes, no compiler exception. +- **AR-C9 — `PruneEvent` re-export ambiguity.** Re-export the type for downstream callers/typing, but the AST audit-completeness scan gates *construction* to `signalforge.prune.audit` only. Mirrors draft's `LLMResponseEvent` precedent exactly. + +**Trusted-models validation timing (DEC-elect)** +- `PruneConfig.trusted_models` is a tuple of unique_id strings. Pydantic can't cross-validate against the manifest at config-load time (manifest isn't loaded yet). **Validate at `prune_tests(...)` entry**: iterate `config.trusted_models`, raise `PruneTrustedModelNotFoundError(unique_id=...)` on first miss. Loud failure on typos. + +**Observability** +- **AR-C10 — `bytes_billed` per decision deferred to v0.2.** `TestResult` doesn't carry `total_bytes_billed` today; adding it is portability-sensitive (Snowflake/Postgres adapters won't have a direct analogue). v0.1 records `elapsed_ms` only; ops doc notes the gap. + +**Testing** +- **AR-C11 — Integration test cost.** One end-to-end test against `bigquery-public-data` per ticket. Behind `@pytest.mark.bigquery` (existing marker; CI excludes via `-m 'not bigquery'`). Slow, but ticket-required and non-optional for closing the bytes/cost loop on AR-B1. + +### Architecture review summary + +Two genuine architecture surprises emerged: +1. **The cost model is wrong** (AR-B1) — `TO_JSON_STRING(t)` is the entire row. Phase 1 understated sample-mode cost; we may need to reopen Q4 (deferred batching) and adopt Q4=C (temp-table-materialised sample) as the v0.1 default rather than Q4=A. +2. **Adapter prerequisite work** (AR-B2) — `QueryJobConfig.job_timeout_ms` plumbing isn't wired in the warehouse adapter today. Q5=A requires a small upstream story (US-W1: extend `_default_job_config` with `timeout_ms`). + +Everything else is concern-level: data-model tightening (AR-C4–C6), API-shape alignment with the drafter (AR-C7–C9), and config polish (AR-C3 partition_filter, trusted-models validation timing). All resolvable in Phase 3 refinement. + +## Refinement Log + +Architecture-blocker resolutions (Q-AR1=B, Q-AR2=B) plus all concerns folded into 28 numbered decisions. + +### Cost-model resolution (Q-AR1=B) + +The `TO_JSON_STRING(t)` finding (AR-B1) is real: BigQuery cannot column-prune through a function argument, so the deterministic-sample predicate reads every column. We **live-verify before locking strategy**. v0.1 ships **Q4=A (one query per test)** as the default; the diagnostic probe story (US-003) measures `total_bytes_billed` against a small public table and either confirms the cost model is acceptable for v0.1 or escalates to Q4=C (temp-table-materialised sample). The escalation is a v0.2 follow-up only if the probe produces a result the user is unwilling to ship. + +### Adapter prerequisite resolution (Q-AR2=B) + +Adapter extension (`timeout_ms` plumbing through `make_query_job_config` and `_default_job_config`) folds into this ticket as the second story (US-002), not a separate ticket. Single self-contained PR; ~10 LOC plus tests. + +### Decisions + +- **DEC-001 — Subpackage layout.** `src/signalforge/prune/{__init__.py, engine.py, compiler.py, models.py, errors.py, audit.py, config.py}`. `_`-prefixed helpers stay non-public. +- **DEC-002 — `prune_tests()` signature.** `prune_tests(model: Model, adapter: WarehouseAdapter, candidates: CandidateSchema, manifest: Manifest, *, config: PruneConfig | None = None) -> PruneResult`. Mirrors drafter's `(model, adapter, ...)` front-pairing (AR-C7). +- **DEC-003 — Drop `KeptTest` / `DroppedTest`.** `PruneResult` has a single `decisions: tuple[PruneDecision, ...]` plus computed properties `kept_decisions` / `dropped_decisions`. One source of truth (AR-C4). +- **DEC-004 — Preserve typed `CandidateTest` on decision.** `PruneDecision.test: CandidateTest` carries the original discriminated union. No loose `test_args: dict`. v0.1 readers fail loud on a v0.2 test type — acceptable; they couldn't process unknown drop reasons either (AR-C5). +- **DEC-005 — Hash conventions (locked).** `compiled_sql_hash` = `blake2b(sql.encode(), digest_size=8).hexdigest()` (16 hex; matches draft). `config_hash` = `sha256(canonical.encode()).hexdigest()[:16]` (16 hex; matches `safety.policy_hash`). Helpers in `prune/audit.py` (AR-C6). +- **DEC-006 — Error hierarchy (locked).** Six classes: `PruneError(SignalForgeError)` → `PruneConfigError` → `PruneTrustedModelNotFoundError`; siblings `PruneTimeoutError`, `PruneAuditWriteError`, `PruneAuditRecordTooLargeError`. **No `PruneCompilerError`** — compilation always succeeds; `relationships(to: unknown)` emits a `requires-future-data` drop reason, not an exception (AR-C8). +- **DEC-007 — `PruneEvent` re-exported, construction-gated.** Re-exported from `signalforge.prune.__init__` for downstream typing. AST audit-completeness scan (`tests/test_audit_completeness.py`) gates `Call(func=Name(id="PruneEvent"))` to `signalforge/prune/audit.py` only — mirrors `LLMResponseEvent` precedent exactly (AR-C9). +- **DEC-008 — Trusted-models validation timing.** `prune.trusted_models: tuple[str, ...]` of `unique_id` strings is validated at `prune_tests(...)` entry, NOT at config-load (manifest isn't loaded yet). Each entry must appear in `manifest.models`; otherwise `PruneTrustedModelNotFoundError(unique_id=...)` raised before any warehouse call. Mirrors Q1=B intent. +- **DEC-009 — `partition_filter` config knob.** `PruneConfig.partition_filter: PartitionFilter | None = None` — the typed value object from `signalforge.warehouse.models`, NOT a raw string. YAML loader renders the dict-shaped input via Pydantic into the typed `PartitionFilter`. Required for full-mode on tables ≥ 100M rows (AR-C3). +- **DEC-010 — Drift detectors are mandatory.** `tests/prune/test_drift_detector.py` defines `StrictPruneResult`, `StrictPruneDecision`, `StrictPruneEvent` (each `extra="forbid"`) and validates a committed `tests/fixtures/prune/prune_event_v1.jsonl` fixture. Adding a field to production without updating the strict mirror OR the fixture breaks the test loudly (AR-B3). +- **DEC-011 — Total-budget semantics.** When `total_budget_seconds` is exceeded, the orchestrator (a) cancels the in-flight test (best-effort `query_job.cancel()`), and (b) marks every remaining un-started test as `kept-without-evidence` with `why="total prune budget exceeded before evaluation"`. No partial-evaluation results (AR-C2). +- **DEC-012 — Live cost verification first (Q-AR1=B).** US-003 measures `total_bytes_billed` from a deterministic-sample run on `bigquery-public-data.iowa_liquor_sales.sales`. If bytes-billed is within ~10× the per-column-only estimate, Q4=A ships v0.1 unchanged. If 50–500× the estimate, escalate to Q4=C (temp-table materialisation) as a v0.1 amendment. +- **DEC-013 — Adapter extension folded (Q-AR2=B).** US-002 extends `make_query_job_config` (`src/signalforge/warehouse/adapters/_client.py`) and `_default_job_config` (`src/signalforge/warehouse/adapters/bigquery.py`) with a `timeout_ms: int | None = None` kwarg threaded into `QueryJobConfig.job_timeout_ms`. ~10 LOC plus tests. Lands before prune's engine. +- **DEC-014 — `PruneEvent` is flat, not nested.** Mirrors `safety.AuditEvent` and `draft.LLMResponseEvent`. Fields: `audit_schema_version: Literal[1] = 1`, `signalforge_version`, `record_id` (uuid4), `timestamp` (ISO 8601 UTC, Z-suffixed), `config_hash`, `model_unique_id`, plus the flattened `PruneDecision` fields (`test`, `decision`, `reason`, `failures`, `sampled_rows`, `scope`, `elapsed_ms`, `compiled_sql_hash`, `compiled_sql`, `why`, `sample_failures`). +- **DEC-015 — `extra=` placement (locked).** `PruneConfig`, `_PruneConfigFile` → `extra="forbid"`. `PruneResult`, `PruneDecision`, `PruneEvent` → `extra="ignore"` paired with `extra="forbid"` Strict mirrors per DEC-010. Mirrors safety-layer.md DEC-015. +- **DEC-016 — Audit at `/.signalforge/prune.jsonl`.** Hardcoded path relative to safety/draft audit's parent (`.signalforge/`). Symlink-hardened via `signalforge.warehouse._path_safety.canonicalise_path` at writer entry. Fail-closed (`O_APPEND | O_CREAT | 0o600` + fsync; size cap `_PRUNE_AUDIT_RECORD_LIMIT_BYTES = 4000` checked before `os.open`; **no try/except inside writer** — propagation is the defence). Mirrors safety/draft semantics exactly. +- **DEC-017 — ANSI-safe lazy-format JSON logger.** Every `_LOGGER.{info,warning,debug,error}` in `signalforge.prune.*` uses lazy-format with `json.dumps()` for any user-controlled string. `_LOGGER = logging.getLogger(__name__)` per module. Grep gate at `tests/prune/test_logger_grep_gate.py` extends the existing `tests/llm/test_logger_grep_gate.py` scan to `src/signalforge/prune/` (or merges the two). Mirrors safety-layer.md DEC-022. +- **DEC-018 — AST audit-completeness scan extension.** `tests/test_audit_completeness.py` adds a fifth scan: `PruneEvent` construction confined to `src/signalforge/prune/audit.py` only. Sanity check that at least one construction exists in the blessed module. Mirrors llm-drafter.md DEC-013. +- **DEC-019 — Module-level `_sleep` alias.** `signalforge.prune.engine` declares `_sleep = time.sleep` at module scope so tests can reassign without monkey-patching globally. Used for the total-budget enforcement loop. Mirrors llm-drafter.md DEC-004. +- **DEC-020 — `prune:` top-level namespace in `signalforge.yml`.** Outer `_PruneConfigFile(extra="ignore")` so unknown sibling stages (`safety:`, `llm:`, `grade:`) don't break the loader. Inner `PruneConfig(extra="forbid")` so typos like `scop:` fail loud. Mirrors llm-drafter.md DEC-027. +- **DEC-021 — Public re-export surface.** `signalforge/prune/__init__.py` exposes: `prune_tests`, `PruneResult`, `PruneDecision`, `PruneConfig`, `load_prune_config`, `DropReason`, `Scope`, `PruneEvent`, `PruneError`, `PruneConfigError`, `PruneTrustedModelNotFoundError`, `PruneTimeoutError`, `PruneAuditWriteError`, `PruneAuditRecordTooLargeError`. `_compile_test`, `_render_why`, `_compute_*_hash`, `_write_prune_event`, `_sleep` are not exported. +- **DEC-022 — `repr()`-quote user input in error messages.** `_format_value(v) = repr(v)` helper. A model unique_id containing `\x1b[31m` doesn't inject into log viewers. `__repr__` on `PruneResult` shows only `model_unique_id`, `kept_count`, `dropped_count`, `elapsed_ms` — no test bodies, no SQL. Mirrors warehouse-adapters.md DEC-022. +- **DEC-023 — NULL-exclusion in compiler matches dbt-core verbatim.** `unique` includes `WHERE col IS NOT NULL`; `accepted_values` includes `WHERE col IS NOT NULL AND col NOT IN (...)`; `relationships` includes child-side `WHERE child.col IS NOT NULL`. Snapshot fixtures pin exact SQL bytes; divergence from dbt-core verdicts is a regression. +- **DEC-024 — `accepted_values.values` escaping reuses adapter helper.** `signalforge.warehouse._sql_safety.escape_bq_string_literal` already exists for the adapter's needs. Prune compiler imports and applies it before single-quoting `accepted_values` literals. No new escaping helper. +- **DEC-025 — Compiler is dialect-driven, not BigQuery-specific.** `_compile_test(test, table_ref, dialect: Dialect, manifest)` reads `dialect.quote_char`, `dialect.identifier_case`, `dialect.supports_qualify` to render SQL. **No `from google.cloud import bigquery` in `signalforge/prune/`.** v0.2 Snowflake/Postgres adapters return their own `Dialect` and the compiler routes accordingly. +- **DEC-026 — `relationships` parent join uses adapter's `TableRef.from_model`.** Resolves the `to` model name to a `TableRef` via the manifest's `Model` lookup; if the model is absent, the compiler returns a sentinel and the engine routes to `requires-future-data` (no warehouse call). +- **DEC-027 — `bytes_billed` per decision deferred to v0.2.** `PruneDecision.elapsed_ms` only in v0.1. `TestResult` stays unchanged. Documented in `docs/prune-ops.md`. v0.2 may extend `TestResult` and add `bytes_billed` to the audit payload. +- **DEC-028 — Single-threaded prune in v0.1.** No `concurrent.futures`, no `asyncio`. Tests are evaluated sequentially against a single adapter context. Concurrency is a v0.2 candidate (Snowflake's per-warehouse parallelism shape may differ from BigQuery's slot model). + +## Detailed Breakdown + +16 stories. US-001…US-014 are implementation; QG and PM are the standard tail. Project ordering: scaffold → adapter prereq → diagnostic probe → primitives (models, config, errors, compiler) → audit → orchestrator (depends on all primitives) → drift detector → cross-cutting tests → integration → re-exports → docs → quality gate → patterns. + +Each story embeds **TDD** where the surface is logic-shaped (compiler, engine, audit, errors, models). Migration- or wiring-shaped stories (scaffold, re-exports, docs) skip TDD. + +### US-001 — Subpackage scaffold + `prune:` namespace + +- **Description:** Create the empty `signalforge.prune` subpackage skeleton and reserve the `prune:` top-level key in `signalforge.yml`. +- **Traces to:** DEC-001, DEC-020 +- **Acceptance Criteria:** `src/signalforge/prune/{__init__.py, engine.py, compiler.py, models.py, errors.py, audit.py, config.py}` exist as stubs; `pip install -e ".[dev]" && ruff check . && ruff format --check . && pyright && pytest` passes. +- **Done when:** `import signalforge.prune` succeeds; `signalforge.prune.__doc__` exists; subpackage appears in `pyright` output with no errors. +- **Files:** `src/signalforge/prune/__init__.py` (docstring + module exports placeholder); `src/signalforge/prune/{engine,compiler,models,errors,audit,config}.py` (each: module docstring, `from __future__ import annotations`, `_LOGGER = logging.getLogger(__name__)` where applicable). No `tests/prune/__init__.py`. +- **Depends on:** none. + +### US-002 — Warehouse adapter `timeout_ms` plumbing (folded US-W1) + +- **Description:** Extend `make_query_job_config` and `_default_job_config` with a `timeout_ms: int | None = None` kwarg threaded into `QueryJobConfig.job_timeout_ms`. Required for prune's per-test budget (Q5=A). +- **Traces to:** DEC-013, AR-B2 +- **Acceptance Criteria:** `make_query_job_config(stage, ..., timeout_ms=30_000)` produces a `QueryJobConfig` with `job_timeout_ms == 30_000`; `timeout_ms=None` leaves the field unset; existing call sites unchanged. Validation suite passes. +- **Done when:** New unit test `tests/warehouse/test_default_job_config_timeout.py` covers (a) None default, (b) explicit value, (c) propagation through `_default_job_config(self, *, stage, timeout_ms=...)`. A long-running query in the integration suite uses the new kwarg with a 1s timeout and observes `JobCancelledError`-equivalent. +- **Files:** `src/signalforge/warehouse/adapters/_client.py` (extend `make_query_job_config`); `src/signalforge/warehouse/adapters/bigquery.py` (extend `_default_job_config`); `tests/warehouse/test_default_job_config_timeout.py` (new); `docs/warehouse-adapter-ops.md` (one-line addendum about the kwarg). +- **TDD:** Yes. + - Test: `make_query_job_config` with `timeout_ms=None` → `job.job_timeout_ms is None` + - Test: `make_query_job_config` with `timeout_ms=30_000` → `job.job_timeout_ms == 30_000` + - Test: `_default_job_config(stage="warehouse_test", timeout_ms=30_000)` propagates value + - Test: existing callers (no `timeout_ms` arg) still produce the same config (regression guard) +- **Depends on:** US-001. + +### US-003 — Diagnostic cost probe (resolves AR-B1) + +- **Description:** Live-verify the deterministic-sample cost model against `bigquery-public-data`. Confirms whether Q4=A (one-query-per-test) is viable for v0.1 or whether we must escalate to Q4=C (temp-table materialisation). +- **Traces to:** DEC-012, AR-B1 +- **Acceptance Criteria:** `tests/warehouse/test_sample_cost_probe.py::test_sample_rows_bytes_billed_within_budget` runs `BigQueryAdapter.sample_rows` against a small public table (`bigquery-public-data.iowa_liquor_sales.sales` or similar), asserts `total_bytes_billed` is within a documented bound (e.g., `< 100_000_000` for the 100k-row deterministic sample), and emits a `WARNING` log if cost is ≥ 10× the column-footprint estimate. Marked `@pytest.mark.bigquery`. Result documented in `docs/prune-ops.md`. +- **Done when:** Test passes against live BigQuery (gated on `SF_RUN_BQ=1`); documented bytes-billed figure recorded in `docs/prune-ops.md` cost-model section. If the test reveals 50–500× the estimate, opens a v0.1 amendment ticket to add Q4=C; v0.1 ships unchanged otherwise. +- **Files:** `tests/warehouse/test_sample_cost_probe.py` (new); `docs/prune-ops.md` (new — initial scaffold; full content lands in US-014). +- **TDD:** No (this is the diagnostic itself). +- **Depends on:** US-001. + +### US-004 — `PruneResult`, `PruneDecision`, primitive types + +- **Description:** Define the prune layer's read-back data classes: `PruneResult`, `PruneDecision`, `DropReason`, `Scope`. All frozen, tuples for sequences, `extra="ignore"`, computed-property aggregates on `PruneResult`. +- **Traces to:** DEC-003, DEC-004, DEC-014, DEC-015 +- **Acceptance Criteria:** `PruneDecision` carries `test: CandidateTest` (typed discriminated union; not a loose dict); `PruneResult.kept_decisions` / `.dropped_decisions` are `@property` filters; `model_dump_json` round-trips through a fixture. +- **Done when:** `tests/prune/test_models.py` covers (a) frozen-set raises, (b) tuple coercion (list input → tuple), (c) `kept_decisions`/`dropped_decisions` correctness, (d) `extra="ignore"` accepts unknown fields, (e) `DropReason` Literal rejects typos. +- **Files:** `src/signalforge/prune/models.py`; `tests/prune/test_models.py`. +- **TDD:** Yes. + - Test: `PruneResult.decisions` accepts a list, stores a tuple + - Test: `PruneResult.kept_decisions` returns only `decision == "kept"` records + - Test: `PruneDecision(reason="phantom-reason")` raises `ValidationError` + - Test: setting any field on a constructed `PruneDecision` raises (frozen) + - Test: `PruneDecision(test=CandidateTestNotNull(column="c"))` round-trips via `model_dump_json` +- **Depends on:** US-001. + +### US-005 — `PruneConfig` + `load_prune_config` + +- **Description:** Define `PruneConfig` (`extra="forbid"`) and `_PruneConfigFile` wrapper (`extra="ignore"`). Load `signalforge.yml`, validate, return `PruneConfig`. Defaults match Phase 1 housekeeping. +- **Traces to:** DEC-009, DEC-015, DEC-020 +- **Acceptance Criteria:** `load_prune_config(Path("signalforge.yml"))` returns `PruneConfig(scope="sample", sample_size=100_000, test_timeout_seconds=30, total_budget_seconds=600, capture_failure_rows=3, trusted_models=(), partition_filter=None)` when the file is absent or `prune:` block is empty. Typos (`scop:`) raise `PruneConfigError`. Sibling stages (`safety:`, `llm:`) are tolerated. `partition_filter` accepts a dict matching `PartitionFilter` shape and renders to the typed object. +- **Done when:** `tests/prune/test_config.py` covers (a) round-trip with a full YAML, (b) defaults when keys absent, (c) typo at `prune:` → `PruneConfigError`, (d) sibling top-level keys ignored, (e) `partition_filter: {column: dt, op: ">=", value: "2026-01-01"}` parses to a `PartitionFilter`. +- **Files:** `src/signalforge/prune/config.py`; `tests/prune/test_config.py`; `tests/fixtures/prune/signalforge_full.yml` (new); `tests/fixtures/prune/signalforge_typo.yml` (new); `tests/fixtures/prune/signalforge_minimal.yml` (new). +- **TDD:** Yes. + - Test: `load_prune_config(missing_path)` returns defaults + - Test: typo at `prune.scop` → `PruneConfigError` with line/col info + - Test: sibling `safety:` block tolerated by `_PruneConfigFile` + - Test: `partition_filter` dict round-trips through Pydantic into `PartitionFilter` + - Test: `yaml.safe_load` rejects Python object constructors +- **Depends on:** US-001, US-004. + +### US-006 — `PruneError` hierarchy + +- **Description:** Define six error classes: `PruneError(SignalForgeError)`, `PruneConfigError`, `PruneTrustedModelNotFoundError`, `PruneTimeoutError`, `PruneAuditWriteError`, `PruneAuditRecordTooLargeError`. Each carries `default_remediation`; `__str__` renders both message and `↳ Remediation:` line; user input is `repr()`-quoted. +- **Traces to:** DEC-006, DEC-022 +- **Acceptance Criteria:** `str(PruneTrustedModelNotFoundError(unique_id="model.shop.\x1b[31mevil"))` does NOT contain a raw ANSI escape; the unique_id is `repr()`-quoted. Each subclass renders message + remediation. +- **Done when:** `tests/prune/test_errors.py` covers each subclass + the `repr()`-quoting defence. +- **Files:** `src/signalforge/prune/errors.py`; `tests/prune/test_errors.py`. +- **TDD:** Yes. + - Test: `str(PruneError("msg"))` contains `↳ Remediation:` + - Test: `default_remediation` per subclass renders when `remediation=None` + - Test: `PruneTrustedModelNotFoundError(unique_id="\x1b[31m")` `__str__` doesn't leak ANSI + - Test: each subclass inherits `PruneError` +- **Depends on:** US-001. + +### US-007 — Compiler — failing-rows SQL for the four test types + +- **Description:** Implement `_compile_test(test, table_ref, dialect, manifest) -> str | _RequiresFutureData` for `not_null`, `unique`, `accepted_values`, `relationships`. Uses `dialect.quote_char` for identifier quoting. Reuses `signalforge.warehouse._sql_safety.escape_bq_string_literal` for `accepted_values.values`. NULL-exclusion matches dbt-core. `relationships(to: unknown)` returns the `_RequiresFutureData` sentinel (not an exception). +- **Traces to:** DEC-023, DEC-024, DEC-025, DEC-026 +- **Acceptance Criteria:** Snapshot fixtures `tests/fixtures/prune/compiled_sql/{not_null,unique,accepted_values,relationships}.sql` pin exact SQL bytes. NULL-exclusion assertions per test type. Adversarial `accepted_values` strings (`O'Brien`, `\x1b[31m`, `\n`, `; DROP TABLE x;--`) round-trip through `escape_bq_string_literal` + the compiler without breaking SQL or producing injection. +- **Done when:** `tests/prune/test_compiler.py` covers ~20 cases: 4 happy-path snapshots, 3 NULL-exclusion assertions, ~9 adversarial values, identifier-backtick verification, `_RequiresFutureData` sentinel for unknown `relationships.to`, determinism (same input → same SQL), hash stability. +- **Files:** `src/signalforge/prune/compiler.py`; `tests/prune/test_compiler.py`; `tests/fixtures/prune/compiled_sql/*.sql` (new); `tests/fixtures/prune/regenerate.sh` (new — documents how to refresh snapshots). +- **TDD:** Yes. + - Test: `_compile_test(NotNull(column="c"), ref, BIGQUERY_DIALECT, manifest)` matches `not_null.sql` snapshot + - Test: `_compile_test(Unique(column="c"), ...)` SQL contains `IS NOT NULL` + - Test: `_compile_test(AcceptedValues(column="c", values=("a",)), ...)` SQL contains `IS NOT NULL AND` and `NOT IN` + - Test: `_compile_test(Relationships(column="c", to="parent", field="id"), ...)` SQL contains `LEFT JOIN` + `child.\`c\` IS NOT NULL` + `parent.\`id\` IS NULL` + - Test: `accepted_values=(O'Brien,)` produces `'O\\'Brien'` in SQL (escaped, balanced quotes) + - Test: `accepted_values=(; DROP TABLE x;--,)` is fully quoted; `validate_test_sql` does not reject the wrapped result + - Test: `Relationships(to="model.unknown")` returns `_RequiresFutureData(reason=...)` sentinel + - Test: same input produces same SQL byte-for-byte across two invocations + - Test: `compiled_sql_hash` is `blake2b-8(sql.encode()).hexdigest()` (16 hex) +- **Depends on:** US-001, US-004 (for `CandidateTest` re-import only). + +### US-008 — Audit (fail-closed `prune.jsonl` + `PruneEvent`) + +- **Description:** Implement `signalforge.prune.audit` with `PruneEvent` model + `_write_prune_event(event, path)` writer. Fail-closed: `O_APPEND | O_CREAT | 0o600`, fsync, size cap before `os.open`, no try/except inside writer. Audit-completeness AST scan extension gates `PruneEvent` construction to this module. +- **Traces to:** DEC-007, DEC-014, DEC-016, DEC-018 +- **Acceptance Criteria:** `_write_prune_event(event, audit_dir / "prune.jsonl")` produces a single JSONL line; oversize record (≥ 4000 bytes) raises `PruneAuditRecordTooLargeError` BEFORE `os.open` (no on-disk artefact); `OSError`/`PermissionError`/encoding failure propagate raw. AST scan in `tests/test_audit_completeness.py` rejects `PruneEvent(...)` calls outside `signalforge/prune/audit.py`. +- **Done when:** `tests/prune/test_audit.py` covers (a) happy-path single-line write, (b) `O_APPEND|O_CREAT|0o600` mode, (c) `os.fsync` invoked, (d) oversize raises before `os.open`, (e) `OSError` propagates raw, (f) ANSI in `compiled_sql` round-trips through JSON safely, (g) concurrent appends from 10 threads × 50 writes don't interleave (POSIX `O_APPEND` atomicity); `tests/test_audit_completeness.py::test_prune_event_construction_only_in_prune_audit_module` passes. +- **Files:** `src/signalforge/prune/audit.py`; `tests/prune/test_audit.py`; `tests/test_audit_completeness.py` (extend with the prune scan). +- **TDD:** Yes. + - Test: writer emits one JSONL line with `audit_schema_version: 1` + - Test: writer opens with `O_APPEND | O_CREAT | 0o600` + - Test: writer calls `os.fsync` before close + - Test: oversize record raises `PruneAuditRecordTooLargeError`; `audit.jsonl` does not exist + - Test: `OSError` from `os.write` propagates as-is (no wrapping) + - Test: `PruneEvent(compiled_sql="\x1b[31m")` JSON round-trip is ANSI-safe (escaped to ``) + - Test: 10-thread × 50-write concurrent append produces 500 well-formed lines + - Test: AST scan rejects `PruneEvent(...)` in any module other than `prune/audit.py` +- **Depends on:** US-001, US-004, US-006. + +### US-009 — Engine (`prune_tests` orchestrator) + +- **Description:** Implement `prune_tests(model, adapter, candidates, manifest, *, config)` end-to-end: validate trusted_models against manifest; iterate candidates; compile each test (or route to `requires-future-data`); call `adapter.run_test_sql` with per-test `timeout_ms`; route the outcome to the correct `DropReason`; enforce total budget; write audit; return `PruneResult`. Module-level `_sleep` alias for deterministic test override. +- **Traces to:** DEC-002, DEC-008, DEC-011, DEC-019, plus all DROP-reason routing +- **Acceptance Criteria:** Six routing branches verified — `always-passes`, `requires-future-data`, `failed-on-known-clean-data` (only for trusted models), `kept-without-evidence` (per-test timeout), `kept-without-evidence` (total budget exceeded), `kept`. Audit row written per decision. `PruneAuditWriteError` propagation aborts the run (no `PruneResult`). `PruneTrustedModelNotFoundError` raised at entry on typo. Total-budget exceedance cancels in-flight + marks remaining `kept-without-evidence`. +- **Done when:** `tests/prune/test_engine.py` covers ~20 cases against an extended `FakeBigQueryClient.expect_run_test_sql(...)`. +- **Files:** `src/signalforge/prune/engine.py`; `tests/prune/test_engine.py`; `tests/warehouse/_fake.py` (extend with `expect_run_test_sql(matching, returns)` helper). +- **TDD:** Yes. + - Test: 5 always-pass + 3 failed + 2 timeout → `PruneResult.dropped_decisions` count = 5+2; reasons match + - Test: `relationships(to: missing_model)` short-circuits — no `run_test_sql` call; `dropped`, `requires-future-data` + - Test: per-test `timeout_ms` exceeded → adapter raises (or returns timeout marker); engine routes `kept-without-evidence`; `why` includes elapsed_ms + - Test: total-budget exceeded after N tests → remaining (N+1…end) marked `kept-without-evidence`, no further `run_test_sql` invocations + - Test: trusted model in `config.trusted_models` + test failure → `failed-on-known-clean-data` + - Test: non-trusted model + test failure → `kept` + - Test: `config.trusted_models = ("model.unknown",)` raises `PruneTrustedModelNotFoundError` at entry; no warehouse call + - Test: `PruneAuditWriteError` from audit writer aborts the run; no `PruneResult` returned + - Test: `WarehouseError` subclass (`TableNotFoundError`, `ColumnNotFoundError`, `QuerySyntaxError`, `BytesBilledExceededError`) routes to `kept-without-evidence` with the error message in `why` + - Test: `_sleep` alias is reassignable (test injects a recorder; assert call count equals total-budget loop iterations) + - Test: deterministic `config_hash` across two identical configs +- **Depends on:** US-002, US-004, US-005, US-006, US-007, US-008. + +### US-010 — Drift detectors + +- **Description:** `tests/prune/test_drift_detector.py` defines `StrictPruneResult`, `StrictPruneDecision`, `StrictPruneEvent` (each `extra="forbid"` mirrors of the production models) and validates a committed JSONL fixture. Adding a field to production without updating the strict mirror or the fixture breaks the test loudly. +- **Traces to:** DEC-010, DEC-015 +- **Acceptance Criteria:** `tests/fixtures/prune/prune_event_v1.jsonl` (3-5 events covering happy path + each drop reason) validates against `StrictPruneEvent`. Three strict mirrors each pair with field-set assertions against the production model. +- **Done when:** Test passes; deliberately adding a field to `PruneEvent` without updating the mirror breaks the test. +- **Files:** `tests/prune/test_drift_detector.py`; `tests/fixtures/prune/prune_event_v1.jsonl`; `tests/fixtures/prune/prune_result_v1.json`; `tests/fixtures/prune/prune_decision_v1.json`. +- **TDD:** Yes. + - Test: `StrictPruneEvent.model_validate_json(line)` for every line in fixture passes + - Test: `StrictPruneEvent.model_fields.keys() == PruneEvent.model_fields.keys()` (field-set parity) + - Same two tests for `StrictPruneResult` + `StrictPruneDecision` +- **Depends on:** US-004, US-008. + +### US-011 — Logger grep gate extension + +- **Description:** Extend `tests/llm/test_logger_grep_gate.py` (or create `tests/prune/test_logger_grep_gate.py`) to scan `src/signalforge/prune/` for `_LOGGER\.\w+\(f"` patterns. Reject any hits. +- **Traces to:** DEC-017 +- **Acceptance Criteria:** Test runs against current `signalforge.prune.*` and finds zero violations. Deliberately adding `_LOGGER.info(f"...{user_input}...")` to any prune module breaks the test. +- **Done when:** Grep gate covers all three modules (`llm`, `draft`, `prune`). +- **Files:** Extend `tests/llm/test_logger_grep_gate.py` OR create `tests/prune/test_logger_grep_gate.py` and update its scan paths. +- **TDD:** No (test-only). +- **Depends on:** US-009 (so there's something for the gate to scan). + +### US-012 — Integration test against `bigquery-public-data` + +- **Description:** End-to-end prune against a small public BQ table. One comprehensive test under `@pytest.mark.bigquery` (existing marker; CI excludes via `-m 'not bigquery'`). +- **Traces to:** ticket acceptance criterion #7, AR-C11 +- **Acceptance Criteria:** `tests/prune/test_integration_bigquery.py::test_prune_iowa_liquor_sales` runs `prune_tests` against four candidate tests on `bigquery-public-data.iowa_liquor_sales.sales` (or similar). Asserts at least one always-pass drop AND at least one kept failure. Marked `@pytest.mark.bigquery` + `@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), ...)`. Documented in `docs/prune-ops.md` "running real-warehouse tests". +- **Done when:** Test passes against live BigQuery (gated on `SF_RUN_BQ=1`); excluded from default `pytest` run. +- **Files:** `tests/prune/test_integration_bigquery.py` (new). +- **TDD:** No (integration). +- **Depends on:** US-009. + +### US-013 — Public re-exports + +- **Description:** Populate `signalforge/prune/__init__.py` with the public surface. Re-export `PruneEvent` for typing (construction is AST-gated to `audit.py` per DEC-007). +- **Traces to:** DEC-021 +- **Acceptance Criteria:** `from signalforge.prune import prune_tests, PruneResult, PruneDecision, PruneConfig, load_prune_config, DropReason, Scope, PruneEvent, PruneError, PruneConfigError, PruneTrustedModelNotFoundError, PruneTimeoutError, PruneAuditWriteError, PruneAuditRecordTooLargeError` works. `signalforge.prune.__all__` lists each. `pyright` clean. +- **Done when:** Smoke test `tests/prune/test_smoke.py::test_public_api_imports` passes. +- **Files:** `src/signalforge/prune/__init__.py`; `tests/prune/test_smoke.py`. +- **TDD:** No (re-exports). +- **Depends on:** US-004, US-005, US-006, US-008, US-009. + +### US-014 — Operational reference doc + +- **Description:** Write `docs/prune-ops.md` mirroring `docs/safety-ops.md` and `docs/draft-ops.md` shape. Cover: public API + signature; config block reference; drop-reason taxonomy; cost model + the AR-B1 verification result from US-003; how to invoke against BigQuery; audit JSONL schema (event fields + record contract); running real-warehouse tests; v0.2 deferrals (`bytes_billed`, batching, multi-warehouse). +- **Traces to:** DEC-027, all v0.1 surfaces +- **Acceptance Criteria:** Doc exists; cross-linked from the `## What's shipped` section of `CLAUDE.md`. +- **Done when:** `docs/prune-ops.md` published; `CLAUDE.md` "Repository status" section updated to list `#6 (prune engine)`; `## Public API surface (v0.1)` section in `CLAUDE.md` lists the prune surface. +- **Files:** `docs/prune-ops.md` (new); `CLAUDE.md` (extend). +- **TDD:** No (docs). +- **Depends on:** US-003, US-009, US-012. + +### Quality Gate (QG) + +- **Description:** Run code reviewer four times across the full changeset, fixing all real bugs found each pass. Run CodeRabbit review. Project validation (`ruff check . && ruff format --check . && pyright && pytest`) must pass after all fixes. +- **Acceptance Criteria:** Four code-review passes, CodeRabbit pass, validation suite green. +- **Done when:** All four reviews complete; outstanding CodeRabbit comments resolved or deferred with justification; CI green. +- **Files:** Whatever each pass surfaces. +- **Depends on:** US-001 through US-014. + +### Patterns & Memory (PM) + +- **Description:** Distill the rules learned in this ticket into `.claude/rules/prune-engine.md`; update `CLAUDE.md` to reference it; add `bd remember` entries for any patterns that should be reachable cross-ticket (e.g., "when extending the audit family, always extend the AST audit-completeness scan"). +- **Acceptance Criteria:** New `.claude/rules/prune-engine.md` exists with the standard "Established by issue #6 ..." preamble; `CLAUDE.md` lists it; `.claude/rules/llm-drafter.md`-shape conventions covered (single seam, fail-closed audit, AST gates, ANSI-safe logger, dialect-driven SQL). +- **Done when:** Rules file lands; `CLAUDE.md` cross-links it; pattern memory entries created. +- **Files:** `.claude/rules/prune-engine.md` (new); `CLAUDE.md`. +- **Depends on:** QG. + +## Beads Manifest + +_Pending Phase 7 (devolve)._ + +## Refinement Log + +_Pending Phase 3._ + +## Detailed Breakdown + +_Pending Phase 4._ + +## Beads Manifest + +_Pending Phase 7 (devolve)._ From 965c2d04f291bd31d8a84b5f687e4067494e72ad Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 21:48:09 -0700 Subject: [PATCH 02/24] 6: bump plan phase to published PR #20 is open; phase marker updated for re-invocation. Co-Authored-By: Claude Opus 4.7 (1M context) --- plans/super/6-prune-engine.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/super/6-prune-engine.md b/plans/super/6-prune-engine.md index 901ac031..4229799b 100644 --- a/plans/super/6-prune-engine.md +++ b/plans/super/6-prune-engine.md @@ -5,7 +5,7 @@ - **Ticket:** [#6](https://github.com/wjduenow/SignalForge/issues/6) - **Branch:** `feature/6-prune-engine` (off `dev`) - **Worktree:** `/home/wesd/dev/worktrees/SignalForge/feature/6-prune-engine` (created via `git worktree add`) -- **Phase:** detailing (Phases 1–3 complete; awaiting story-set sign-off) +- **Phase:** published (PR [#20](https://github.com/wjduenow/SignalForge/pull/20) draft; awaiting review) - **Sessions:** 1 (started 2026-04-30) - **Plan author:** Claude Code (Opus 4.7, 1M context) - **Milestone:** v0.1 (the **load-bearing** ticket — Architectural Commitment #1; gates #7 grader and #8 diff renderer) From 80ac03c9342e0e91b3f1f9652b6daebb65b5d96d Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 21:52:46 -0700 Subject: [PATCH 03/24] =?UTF-8?q?6:=20devolve=20plan=20=E2=80=94=20beads?= =?UTF-8?q?=20manifest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Epic bd_1-scaffolding-y8y + 14 implementation tasks + QG + PM. Dependencies wired so `bd ready` returns only US-001 at start. Phase marker bumped to devolved. Co-Authored-By: Claude Opus 4.7 (1M context) --- plans/super/6-prune-engine.md | 40 +++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/plans/super/6-prune-engine.md b/plans/super/6-prune-engine.md index 4229799b..147fdd06 100644 --- a/plans/super/6-prune-engine.md +++ b/plans/super/6-prune-engine.md @@ -5,7 +5,7 @@ - **Ticket:** [#6](https://github.com/wjduenow/SignalForge/issues/6) - **Branch:** `feature/6-prune-engine` (off `dev`) - **Worktree:** `/home/wesd/dev/worktrees/SignalForge/feature/6-prune-engine` (created via `git worktree add`) -- **Phase:** published (PR [#20](https://github.com/wjduenow/SignalForge/pull/20) draft; awaiting review) +- **Phase:** devolved (epic + 16 tasks live in beads 2026-04-30; PR [#20](https://github.com/wjduenow/SignalForge/pull/20) draft) - **Sessions:** 1 (started 2026-04-30) - **Plan author:** Claude Code (Opus 4.7, 1M context) - **Milestone:** v0.1 (the **load-bearing** ticket — Architectural Commitment #1; gates #7 grader and #8 diff renderer) @@ -560,16 +560,30 @@ Each story embeds **TDD** where the surface is logic-shaped (compiler, engine, a ## Beads Manifest -_Pending Phase 7 (devolve)._ +Devolved 2026-04-30. Epic + 14 implementation tasks + Quality Gate + Patterns & Memory. -## Refinement Log - -_Pending Phase 3._ - -## Detailed Breakdown - -_Pending Phase 4._ - -## Beads Manifest - -_Pending Phase 7 (devolve)._ +- **Worktree:** `/home/wesd/dev/worktrees/SignalForge/feature/6-prune-engine` +- **Branch:** `feature/6-prune-engine` (off `dev`) +- **PR:** [#20](https://github.com/wjduenow/SignalForge/pull/20) (draft, plan only) + +| Story | Beads ID | +|-------|----------| +| Epic — 6: Test prune engine | `bd_1-scaffolding-y8y` | +| US-001 Subpackage scaffold | `bd_1-scaffolding-y8y.1` | +| US-002 Adapter `timeout_ms` plumbing | `bd_1-scaffolding-y8y.2` | +| US-003 Diagnostic cost probe | `bd_1-scaffolding-y8y.3` | +| US-004 `PruneResult` / `PruneDecision` / primitives | `bd_1-scaffolding-y8y.4` | +| US-005 `PruneConfig` + `load_prune_config` | `bd_1-scaffolding-y8y.5` | +| US-006 `PruneError` hierarchy | `bd_1-scaffolding-y8y.6` | +| US-007 Compiler — failing-rows SQL × 4 | `bd_1-scaffolding-y8y.7` | +| US-008 Audit — `prune.jsonl` + AST scan | `bd_1-scaffolding-y8y.8` | +| US-009 Engine — `prune_tests` orchestrator | `bd_1-scaffolding-y8y.9` | +| US-010 Drift detectors | `bd_1-scaffolding-y8y.10` | +| US-011 Logger grep gate extension | `bd_1-scaffolding-y8y.11` | +| US-012 BQ integration test | `bd_1-scaffolding-y8y.12` | +| US-013 Public re-exports | `bd_1-scaffolding-y8y.13` | +| US-014 `docs/prune-ops.md` | `bd_1-scaffolding-y8y.14` | +| QG Quality Gate | `bd_1-scaffolding-y8y.15` | +| PM Patterns & Memory | `bd_1-scaffolding-y8y.16` | + +Dependencies wired so `bd ready` returns only the epic + US-001 at start; subsequent tasks unblock as predecessors close. From 8c7db22c83433b5c46139c07e557020283fcf62b Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 21:58:23 -0700 Subject: [PATCH 04/24] bd_1-scaffolding-y8y.1: scaffold signalforge.prune subpackage Empty docstring-only stubs for engine, compiler, models, errors, audit, and config modules. Subsequent stories (US-002 ... US-014) land the actual logic. Plan: plans/super/6-prune-engine.md (DEC-001). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/prune/__init__.py | 16 ++++++++++++++++ src/signalforge/prune/audit.py | 18 ++++++++++++++++++ src/signalforge/prune/compiler.py | 14 ++++++++++++++ src/signalforge/prune/config.py | 14 ++++++++++++++ src/signalforge/prune/engine.py | 18 ++++++++++++++++++ src/signalforge/prune/errors.py | 14 ++++++++++++++ src/signalforge/prune/models.py | 16 ++++++++++++++++ 7 files changed, 110 insertions(+) create mode 100644 src/signalforge/prune/__init__.py create mode 100644 src/signalforge/prune/audit.py create mode 100644 src/signalforge/prune/compiler.py create mode 100644 src/signalforge/prune/config.py create mode 100644 src/signalforge/prune/engine.py create mode 100644 src/signalforge/prune/errors.py create mode 100644 src/signalforge/prune/models.py diff --git a/src/signalforge/prune/__init__.py b/src/signalforge/prune/__init__.py new file mode 100644 index 00000000..18297cd6 --- /dev/null +++ b/src/signalforge/prune/__init__.py @@ -0,0 +1,16 @@ +"""SignalForge prune layer. + +The signal-vs-volume gate. Runs every candidate test from the drafter against +real warehouse data via the warehouse adapter and drops the ones with no +signal: tests that always pass on warehouse samples, tests that fail on +known-clean data, and tests whose required parent data does not yet exist. +Conservative defaults keep tests we cannot evaluate (`kept-without-evidence`) +rather than silently dropping them, and every kept/dropped test ships with a +structured `PruneDecision` carrying the drop reason, failure count, scope, and +the literal compiled SQL so downstream stages (#7 grader, #8 diff renderer) can +explain every artifact. + +Public API lands in US-013; this module currently re-exports nothing. + +See plans/super/6-prune-engine.md for the full design. +""" diff --git a/src/signalforge/prune/audit.py b/src/signalforge/prune/audit.py new file mode 100644 index 00000000..21bb68b9 --- /dev/null +++ b/src/signalforge/prune/audit.py @@ -0,0 +1,18 @@ +"""Fail-closed prune-event audit writer. + +Mirrors the safety layer's audit and the drafter's response audit at the +prune-decision boundary: opens with `O_APPEND | O_CREAT | 0o600`, writes one +JSONL line per `PruneEvent`, calls `os.fsync`, closes. Catches no exceptions +internally — `OSError`, `PermissionError`, encoding failures all propagate so +an unaudited prune decision can never silently land. Size cap is checked +before any file open. AST audit-completeness scan gates `PruneEvent` +construction to this module only. + +See plans/super/6-prune-engine.md for the full design. +""" + +from __future__ import annotations + +import logging + +_LOGGER = logging.getLogger(__name__) diff --git a/src/signalforge/prune/compiler.py b/src/signalforge/prune/compiler.py new file mode 100644 index 00000000..05c2fd8f --- /dev/null +++ b/src/signalforge/prune/compiler.py @@ -0,0 +1,14 @@ +"""Candidate-test SQL compiler. + +Translates each variant of the drafter's `CandidateTest` discriminated union +(`not_null`, `unique`, `accepted_values`, `relationships`) into a failing-rows +SELECT statement using `WarehouseAdapter.dialect()` for quote character and +identifier casing. Matches dbt-core's NULL-exclusion conventions verbatim so +prune verdicts agree with `dbt test` runtime verdicts. Quote-escapes +user-controlled values (notably `accepted_values.values`) before SQL +interpolation; trusts adapter-validated identifiers on `TableRef`. + +See plans/super/6-prune-engine.md for the full design. +""" + +from __future__ import annotations diff --git a/src/signalforge/prune/config.py b/src/signalforge/prune/config.py new file mode 100644 index 00000000..2481f489 --- /dev/null +++ b/src/signalforge/prune/config.py @@ -0,0 +1,14 @@ +"""Prune-layer config loader. + +Loads the `prune:` top-level block from `signalforge.yml` into a typed +`PruneConfig` (scope, sample_size, test_timeout_seconds, +total_budget_seconds, capture_failure_rows). Outer file wrapper uses +`extra="ignore"` so sibling stage namespaces (`safety:`, `llm:`, future +`grade:`) don't break the loader; the inner `PruneConfig` block uses +`extra="forbid"` so a typo like `scop:` fails loud rather than silently +no-op'ing. + +See plans/super/6-prune-engine.md for the full design. +""" + +from __future__ import annotations diff --git a/src/signalforge/prune/engine.py b/src/signalforge/prune/engine.py new file mode 100644 index 00000000..bbc95837 --- /dev/null +++ b/src/signalforge/prune/engine.py @@ -0,0 +1,18 @@ +"""Prune engine entry point and per-test executor. + +Hosts `prune_tests(candidates, model, manifest, adapter, *, config=None)` — +the public seam that compiles every CandidateTest to failing-rows SQL via the +compiler module, runs each through `WarehouseAdapter.run_test_sql`, and folds +the per-test verdicts into a typed `PruneResult`. The executor enforces the +per-test runtime budget, the total-budget cap, and the conservative +"keep-on-error" default that routes warehouse exceptions to +`kept-without-evidence` rather than dropping silently. + +See plans/super/6-prune-engine.md for the full design. +""" + +from __future__ import annotations + +import logging + +_LOGGER = logging.getLogger(__name__) diff --git a/src/signalforge/prune/errors.py b/src/signalforge/prune/errors.py new file mode 100644 index 00000000..47bb5486 --- /dev/null +++ b/src/signalforge/prune/errors.py @@ -0,0 +1,14 @@ +"""Prune-layer typed exception hierarchy. + +`PruneError` is the module base; every distinct failure mode (config load, +test compilation, executor budget exhaustion, audit write) is a subclass. +Each carries a class-level `default_remediation` and accepts a `remediation` +kwarg; the base `__str__` renders both message and `↳ Remediation:` line so +the CLI (#9) and diff renderer (#8) can pattern-match on type rather than +sniffing message text. Mirrors the manifest, warehouse, safety, and draft +layers' error-hierarchy conventions. + +See plans/super/6-prune-engine.md for the full design. +""" + +from __future__ import annotations diff --git a/src/signalforge/prune/models.py b/src/signalforge/prune/models.py new file mode 100644 index 00000000..53a509fd --- /dev/null +++ b/src/signalforge/prune/models.py @@ -0,0 +1,16 @@ +"""Typed shapes for the prune layer. + +Hosts `PruneResult`, `PruneDecision`, the `DropReason` and `Scope` literals, +and the user-facing `PruneConfig`. `PruneResult` carries the kept/dropped +split plus the full per-test decision list and the schema version for +forward-compat. `PruneDecision` carries every field the diff renderer (#8) +and grader (#7) need to explain a verdict: anchor, type, args, decision, +reason, failure count, scope, elapsed time, compiled-SQL hash, the literal +SQL, and a one-line "why". Read-back models use `extra="ignore"` paired with +a one-off `extra="forbid"` drift detector; `PruneConfig` uses `extra="forbid"` +so config typos fail loud. + +See plans/super/6-prune-engine.md for the full design. +""" + +from __future__ import annotations From fb99a31736f519073031dc42cfa6d9d9b1677ffc Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:03:05 -0700 Subject: [PATCH 05/24] bd_1-scaffolding-y8y.4: PruneResult / PruneDecision / primitives Frozen Pydantic v2 models for the prune layer's read-back surface. DropReason and Scope discriminator literals. Computed-property aggregates (kept_decisions, dropped_decisions, kept_count, etc.) for the diff renderer (#8). PruneDecision carries the typed CandidateTest discriminated union (DEC-004), not a loose dict. Plan: plans/super/6-prune-engine.md (DEC-003, DEC-004, DEC-014, DEC-015). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/prune/models.py | 173 +++++++++++++++++++++++++++++--- tests/prune/test_models.py | 134 +++++++++++++++++++++++++ 2 files changed, 294 insertions(+), 13 deletions(-) create mode 100644 tests/prune/test_models.py diff --git a/src/signalforge/prune/models.py b/src/signalforge/prune/models.py index 53a509fd..b708d54d 100644 --- a/src/signalforge/prune/models.py +++ b/src/signalforge/prune/models.py @@ -1,16 +1,163 @@ -"""Typed shapes for the prune layer. - -Hosts `PruneResult`, `PruneDecision`, the `DropReason` and `Scope` literals, -and the user-facing `PruneConfig`. `PruneResult` carries the kept/dropped -split plus the full per-test decision list and the schema version for -forward-compat. `PruneDecision` carries every field the diff renderer (#8) -and grader (#7) need to explain a verdict: anchor, type, args, decision, -reason, failure count, scope, elapsed time, compiled-SQL hash, the literal -SQL, and a one-line "why". Read-back models use `extra="ignore"` paired with -a one-off `extra="forbid"` drift detector; `PruneConfig` uses `extra="forbid"` -so config typos fail loud. - -See plans/super/6-prune-engine.md for the full design. +"""Typed read-back shapes for the prune layer (US-004). + +Defines the read-back-stable Pydantic models the prune engine returns +to its callers: :class:`PruneResult` and :class:`PruneDecision`, plus +the :data:`DropReason` and :data:`Scope` discriminator literals. +Downstream stages — grader (#7), diff renderer (#8), CLI (#9) — +consume these shapes; v0.2 readers MAY also load the persisted JSON +representation back into these models, so forward-compat matters. + +Design commitments operationalised here: + +* **DEC-003** — No standalone ``KeptTest`` / ``DroppedTest`` types. + Filter views (``kept_decisions`` / ``dropped_decisions``) and the + count aggregates are :func:`pydantic.computed_field` properties + derived from the canonical ``decisions`` tuple, so a renderer that + builds a :class:`PruneResult` from a JSONL log gets the same view + as a freshly produced one. +* **DEC-004** — :attr:`PruneDecision.test` is the typed + :data:`signalforge.draft.CandidateTest` discriminated union, not a + loose ``dict[str, Any]``. The grader (#7) and diff renderer (#8) + reuse the drafter's per-variant display logic; v0.1 readers fail + loud on a v0.2 test type they don't recognise. +* **DEC-005** — ``compiled_sql_hash`` is the hex digest of + ``blake2b(sql.encode(), digest_size=8)`` (16 hex characters), + matching the precedent established by + :class:`signalforge.draft.LLMResponseEvent`. +* **DEC-014 / DEC-015** — :class:`PruneResult` carries + :attr:`prune_schema_version` (``Literal[1]``) so on-disk JSON + consumers can branch on shape changes; field *additions* are + handled by ``extra="ignore"`` plus a one-off ``extra="forbid"`` + drift detector (US-010). Read-back semantics — no ``extra="forbid"`` + on these models, that's reserved for config-shaped models per + ``.claude/rules/safety-layer.md``. +* **Transitive immutability** — sequences are :class:`tuple` rather + than :class:`list` so a caller cannot mutate ``decisions`` after + construction; ``frozen=True`` blocks attribute reassignment. + +This module declares only data shapes. The compilation of a +:class:`signalforge.draft.CandidateTest` to SQL, the warehouse +execution that produces ``failures``, and the always-passes / clean-data +verdict logic all live in sibling modules under +:mod:`signalforge.prune` and are not part of this US. """ from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, computed_field + +from signalforge.draft import CandidateTest + +DropReason = Literal[ + "always-passes", + "requires-future-data", + "failed-on-known-clean-data", + "kept", + "kept-without-evidence", +] +"""Closed set of verdict reasons emitted by the prune engine. + +The four ``dropped`` reasons cover the noise-direction splits called +out in :file:`CLAUDE.md` (always-pass tests AND tests that fail on +known-clean data are both dropped). The two ``kept`` reasons cover +the with-evidence and without-evidence cases — the latter happens +when the warehouse sample is too small to support a verdict but the +test references real columns and is well-formed. +""" + +Scope = Literal["sample", "full"] +"""Whether ``failures`` was measured against a sampled or full scan. + +When ``scope == "full"``, :attr:`PruneDecision.sampled_rows` is +``None`` because every row in the model was inspected. +""" + +_BASE_CONFIG = ConfigDict(frozen=True, extra="ignore", populate_by_name=True) + + +class PruneDecision(BaseModel): + """One verdict per candidate test. + + Carries the original :data:`signalforge.draft.CandidateTest` (the + typed discriminated union, not a loose dict) so #7/#8 can reuse the + drafter's per-variant display logic and v0.1 readers fail loud on a + v0.2 test type. ``test_anchor`` is ``"column."`` for + column-scoped tests and the literal string ``"model"`` for + model-level tests. + + Hash conventions (DEC-005): ``compiled_sql_hash`` is + ``blake2b(sql.encode(), digest_size=8).hexdigest()`` — 16 hex + characters, matching the precedent set by + :class:`signalforge.draft.LLMResponseEvent`. + """ + + model_config = _BASE_CONFIG + + test_anchor: str + test: CandidateTest + decision: Literal["kept", "dropped"] + reason: DropReason + failures: int + sampled_rows: int | None + scope: Scope + elapsed_ms: int + compiled_sql_hash: str + compiled_sql: str + why: str + sample_failures: tuple[dict[str, Any], ...] | None = None + + +class PruneResult(BaseModel): + """Aggregate result of pruning all candidates for one model. + + ``prune_schema_version`` is bumped only when the persisted JSON / + JSONL shape changes; field *additions* are handled by + ``extra="ignore"`` (DEC-015) plus the one-off ``extra="forbid"`` + drift detector that lands in US-010. The kept / dropped views and + count aggregates are :func:`pydantic.computed_field` properties + derived from ``decisions`` (DEC-003), so a result reconstructed from + a JSONL log carries identical views to a freshly produced one. + """ + + model_config = _BASE_CONFIG + + prune_schema_version: Literal[1] = 1 + model_unique_id: str + decisions: tuple[PruneDecision, ...] + elapsed_ms: int + signalforge_version: str + + @computed_field # type: ignore[prop-decorator] + @property + def kept_decisions(self) -> tuple[PruneDecision, ...]: + return tuple(d for d in self.decisions if d.decision == "kept") + + @computed_field # type: ignore[prop-decorator] + @property + def dropped_decisions(self) -> tuple[PruneDecision, ...]: + return tuple(d for d in self.decisions if d.decision == "dropped") + + @computed_field # type: ignore[prop-decorator] + @property + def kept_count(self) -> int: + return len(self.kept_decisions) + + @computed_field # type: ignore[prop-decorator] + @property + def dropped_count(self) -> int: + return len(self.dropped_decisions) + + @computed_field # type: ignore[prop-decorator] + @property + def total_tests(self) -> int: + return len(self.decisions) + + +__all__ = ( + "DropReason", + "PruneDecision", + "PruneResult", + "Scope", +) diff --git a/tests/prune/test_models.py b/tests/prune/test_models.py new file mode 100644 index 00000000..e4a5f6e6 --- /dev/null +++ b/tests/prune/test_models.py @@ -0,0 +1,134 @@ +"""Tests for ``signalforge.prune.models`` (US-004). + +Covers the read-back-stable shapes the prune engine returns: tuple +coercion of the ``decisions`` sequence, computed-property partitioning +into kept/dropped, frozen immutability, validator rejection of unknown +``DropReason`` literals, and discriminated-union round-trip of the +embedded :class:`CandidateTest` (DEC-004 — the typed union, not a loose +dict). The drift detector (one-off ``extra="forbid"`` mirror) lands in +US-010. +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from signalforge.draft.models import ( + CandidateTest, + CandidateTestAcceptedValues, + CandidateTestNotNull, +) +from signalforge.prune.models import PruneDecision, PruneResult + + +def _make_decision( + *, + decision: str = "kept", + reason: str = "kept", + test: CandidateTest | None = None, + test_anchor: str = "column.email", + failures: int = 0, + sampled_rows: int | None = 1000, + scope: str = "sample", +) -> PruneDecision: + test_value: CandidateTest = test if test is not None else CandidateTestNotNull(column="email") + return PruneDecision( + test_anchor=test_anchor, + test=test_value, + decision=decision, # type: ignore[arg-type] + reason=reason, # type: ignore[arg-type] + failures=failures, + sampled_rows=sampled_rows, + scope=scope, # type: ignore[arg-type] + elapsed_ms=12, + compiled_sql_hash="0123456789abcdef", + compiled_sql="SELECT 1", + why="kept because signal", + ) + + +def test_prune_result_decisions_accepts_list_stores_tuple() -> None: + d1 = _make_decision() + d2 = _make_decision(test_anchor="column.id") + result = PruneResult( + model_unique_id="model.demo.customers", + decisions=[d1, d2], # type: ignore[arg-type] + elapsed_ms=24, + signalforge_version="0.1.0", + ) + assert result.decisions == (d1, d2) + assert isinstance(result.decisions, tuple) + + +def test_prune_result_kept_decisions_filters_correctly() -> None: + kept_a = _make_decision(decision="kept", reason="kept", test_anchor="column.a") + kept_b = _make_decision(decision="kept", reason="kept-without-evidence", test_anchor="column.b") + dropped_a = _make_decision(decision="dropped", reason="always-passes", test_anchor="column.c") + dropped_b = _make_decision( + decision="dropped", + reason="failed-on-known-clean-data", + test_anchor="column.d", + ) + dropped_c = _make_decision( + decision="dropped", + reason="requires-future-data", + test_anchor="column.e", + ) + result = PruneResult( + model_unique_id="model.demo.x", + decisions=(kept_a, kept_b, dropped_a, dropped_b, dropped_c), + elapsed_ms=40, + signalforge_version="0.1.0", + ) + assert result.kept_decisions == (kept_a, kept_b) + assert result.dropped_decisions == (dropped_a, dropped_b, dropped_c) + assert result.kept_count == 2 + assert result.dropped_count == 3 + assert result.total_tests == 5 + assert result.kept_count + result.dropped_count == result.total_tests + + +def test_prune_decision_invalid_drop_reason_raises() -> None: + with pytest.raises(ValidationError): + PruneDecision( + test_anchor="column.email", + test=CandidateTestNotNull(column="email"), + decision="dropped", + reason="phantom-reason", # type: ignore[arg-type] + failures=0, + sampled_rows=1000, + scope="sample", + elapsed_ms=10, + compiled_sql_hash="deadbeefdeadbeef", + compiled_sql="SELECT 1", + why="x", + ) + + +def test_prune_decision_is_frozen() -> None: + decision = _make_decision() + with pytest.raises(ValidationError): + decision.failures = 999 # type: ignore[misc] + + +def test_prune_decision_carries_typed_candidate_test() -> None: + # not_null variant + nn = _make_decision(test=CandidateTestNotNull(column="email")) + assert nn.test.type == "not_null" + nn_json = nn.model_dump_json() + nn_round = PruneDecision.model_validate_json(nn_json) + assert nn_round == nn + assert nn_round.test.type == "not_null" + + # accepted_values variant — confirms discriminated-union round-trip + av_test = CandidateTestAcceptedValues(column="status", values=("placed", "shipped")) + av = _make_decision(test=av_test, test_anchor="column.status") + assert av.test.type == "accepted_values" + av_json = av.model_dump_json() + av_round = PruneDecision.model_validate_json(av_json) + assert av_round == av + assert av_round.test.type == "accepted_values" + # discriminator dispatched the right concrete subclass + assert isinstance(av_round.test, CandidateTestAcceptedValues) + assert av_round.test.values == ("placed", "shipped") From ec037b030248bf35710ab2579c35cc1e0374719e Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:08:14 -0700 Subject: [PATCH 06/24] bd_1-scaffolding-y8y.6: PruneError hierarchy Six typed exception classes: PruneError, PruneConfigError, PruneTrustedModelNotFoundError, PruneTimeoutError, PruneAuditWriteError, PruneAuditRecordTooLargeError. Each carries default_remediation; __str__ renders message + remediation; user input is repr()-quoted to defeat ANSI injection. Introduces signalforge.errors.SignalForgeError as the project-wide root so PruneError(SignalForgeError) wires per DEC-006. Existing layer roots (SafetyError, DraftError, WarehouseError, ...) stay untouched per task scope; future stories can rebase them. Plan: plans/super/6-prune-engine.md (DEC-006, DEC-022). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/errors.py | 36 ++++++ src/signalforge/prune/errors.py | 219 ++++++++++++++++++++++++++++++-- tests/prune/test_errors.py | 120 +++++++++++++++++ 3 files changed, 366 insertions(+), 9 deletions(-) create mode 100644 src/signalforge/errors.py create mode 100644 tests/prune/test_errors.py diff --git a/src/signalforge/errors.py b/src/signalforge/errors.py new file mode 100644 index 00000000..eff05762 --- /dev/null +++ b/src/signalforge/errors.py @@ -0,0 +1,36 @@ +"""Project-wide error base class. + +:class:`SignalForgeError` is the conceptual root of every typed exception +SignalForge raises. Each layer (manifest, warehouse, safety, llm, draft, +prune, ...) defines its own intermediate base — :class:`SafetyError`, +:class:`WarehouseError`, :class:`PruneError`, etc. — so callers can +``except Error`` to catch every failure that originated inside that +layer without sniffing message text. + +A caller that wants to catch "any SignalForge-typed failure" (e.g. a top-level +CLI handler that wants to render a remediation line and exit non-zero) can +catch :class:`SignalForgeError`. Anything that isn't a subclass is, by +definition, an unexpected programming error and should propagate. + +This module deliberately stays minimal: no message formatting, no +``default_remediation``, no ``__str__`` override. Each layer's intermediate +base owns its own rendering convention (the ``message + ↳ Remediation:`` +shape established by :mod:`signalforge.manifest.errors` and mirrored across +the other layers). Centralising the rendering here would force every layer +to inherit it whether they want to or not. +""" + +from __future__ import annotations + + +class SignalForgeError(Exception): + """Project-wide root of every typed SignalForge exception. + + Each subpackage's intermediate base (e.g. :class:`PruneError`, + :class:`SafetyError`) inherits from this class. Callers that want to + catch any SignalForge-typed failure may catch this directly; callers + that want layer-specific catches should catch the intermediate base. + """ + + +__all__ = ["SignalForgeError"] diff --git a/src/signalforge/prune/errors.py b/src/signalforge/prune/errors.py index 47bb5486..b454cff2 100644 --- a/src/signalforge/prune/errors.py +++ b/src/signalforge/prune/errors.py @@ -1,14 +1,215 @@ """Prune-layer typed exception hierarchy. -`PruneError` is the module base; every distinct failure mode (config load, -test compilation, executor budget exhaustion, audit write) is a subclass. -Each carries a class-level `default_remediation` and accepts a `remediation` -kwarg; the base `__str__` renders both message and `↳ Remediation:` line so -the CLI (#9) and diff renderer (#8) can pattern-match on type rather than -sniffing message text. Mirrors the manifest, warehouse, safety, and draft -layers' error-hierarchy conventions. - -See plans/super/6-prune-engine.md for the full design. +Implements DEC-006 (six-class hierarchy rooted at :class:`PruneError`) and +DEC-022 (user-supplied strings rendered via ``repr()`` so adversarial input — +embedded quotes, control chars, ANSI escapes — cannot smuggle special +characters into log viewers or error messages). Mirrors the style established +by :mod:`signalforge.safety.errors` and :mod:`signalforge.draft.errors`: +every error carries a class-level ``default_remediation`` that the base +``__str__`` renders on a separate ``↳ Remediation:`` line. + +The remediation pattern operationalises the README's "explainable diffs" +commitment at the prune layer's failure surface; every distinct failure mode +the prune machinery can produce gets a typed exception so the orchestrator / +CLI / diff renderer can pattern-match on type rather than sniffing message +text. + +DEC-006 deliberately omits a ``PruneCompilerError`` class. Compilation always +succeeds; failures like ``relationships(to: unknown)`` emit a structured +drop reason (``requires-future-data``) via the orchestrator rather than an +exception. Adding a compiler-error class would let callers paper over a +classified drop with a generic catch and lose signal. + +See ``plans/super/6-prune-engine.md`` for the full design. """ from __future__ import annotations + +from typing import ClassVar + +from signalforge.errors import SignalForgeError + + +def _format_value(v: object) -> str: + """Quote a user-supplied value via ``repr()`` for safe inclusion in + error messages (DEC-022). + + Embedding raw user input in error strings is a log-injection seam: a + crafted unique_id like ``"model.shop.\\x1b[31mevil"`` (or a value + containing ``"foo'\\nINFO: spoofed log line"``) could pollute log + viewers or stack traces. Routing every user-controlled value through + ``repr()`` quotes the string, escapes control characters, and makes + whitespace visible. + """ + return repr(v) + + +class PruneError(SignalForgeError): + """Base class for all prune-layer errors. + + Subclasses set a class-level ``default_remediation`` string; instances + may override it via the ``remediation=`` keyword argument. ``__str__`` + renders the message and the remediation on separate lines so log output + and CLI output both read cleanly. + """ + + default_remediation: ClassVar[str] = "(no remediation set — this is the base class)" + + def __init__(self, message: str, *, remediation: str | None = None) -> None: + super().__init__(message) + self.message = message + self.remediation = ( + remediation if remediation is not None else type(self).default_remediation + ) + + def __str__(self) -> str: + return f"{self.message}\n ↳ Remediation: {self.remediation}" + + +class PruneConfigError(PruneError): + """The ``signalforge.yml`` ``prune:`` block failed parse / schema + validation. + + Wraps either a YAML parse failure, a wrong-shape top level, or a + Pydantic ``ValidationError`` from :class:`signalforge.prune.config.PruneConfig` + (the config-shaped model uses ``extra="forbid"`` per ``safety-layer.md`` + DEC-015 so typos like ``scop:`` instead of ``scope:`` fail loud rather + than silently no-op). + """ + + default_remediation: ClassVar[str] = ( + "Inspect the `prune:` block of signalforge.yml — likely a typo in a " + "key (config-shaped models use extra='forbid'), an unknown `scope` " + "value (must be 'sample' or 'full'), or a non-positive " + "`test_timeout_seconds` / `total_budget_seconds`. See " + "docs/prune-ops.md for the field reference." + ) + + +class PruneTrustedModelNotFoundError(PruneConfigError): + """``prune.trusted_models`` references a ``unique_id`` that does not + appear in the loaded :class:`signalforge.manifest.Manifest`. + + Raised at ``prune_tests(...)`` entry — DEC-008 validates trusted-models + at orchestrator entry rather than at config load time, because the + manifest isn't loaded yet when ``load_prune_config`` runs. Surfacing + the typo loudly keeps a misspelled trusted-models entry from silently + losing its "treat clean-data failure as a real failure" semantics. + """ + + default_remediation: ClassVar[str] = ( + "Verify the unique_id matches a model in manifest.json (e.g. " + "`model..`). Trusted-models is validated at " + "prune_tests() entry, NOT at config load — typos surface here, " + "before any warehouse call is issued." + ) + + def __init__(self, unique_id: str, *, remediation: str | None = None) -> None: + self.unique_id = unique_id + message = f"Trusted model {_format_value(unique_id)} is not present in the loaded manifest." + super().__init__(message, remediation=remediation) + + +class PruneTimeoutError(PruneError): + """The prune run exceeded its ``total_budget_seconds`` budget. + + Wraps adapter cancellation when the orchestrator pulls the plug on an + in-flight test. DEC-011: the orchestrator catches this internally and + routes the in-flight + every remaining un-started test to + ``kept-without-evidence`` with ``why="total prune budget exceeded + before evaluation"``. Callers of :func:`signalforge.prune.prune_tests` + do NOT see this exception; it is an internal control-flow signal. + """ + + default_remediation: ClassVar[str] = ( + "Total prune budget exceeded. Either raise `total_budget_seconds` in " + "signalforge.yml, narrow the candidate set, or scope the run with a " + "`partition_filter`. Tests not yet evaluated will ship as " + "`kept-without-evidence` so no signal is silently dropped." + ) + + +class PruneAuditWriteError(PruneError): + """The fail-closed prune-audit writer (DEC-016) could not durably + persist the per-decision receipt. + + Mirrors safety's :class:`signalforge.safety.errors.AuditWriteError` and + draft's :class:`signalforge.draft.errors.LLMResponseAuditWriteError`: + the writer catches **no** exceptions internally; any ``OSError`` / + ``PermissionError`` / encoding failure / ``fsync`` failure propagates + out via this class. The orchestrator must NOT return a + :class:`signalforge.prune.models.PruneResult` whose decision audit + didn't durably hit disk — an unaudited prune decision is, by definition, + a kept/dropped artefact without a receipt, exactly the failure mode the + audit exists to prevent. + """ + + default_remediation: ClassVar[str] = ( + "Verify the prune-audit path (/.signalforge/prune.jsonl) is " + "writable (permissions / disk space / SELinux contexts) and re-run. " + "The prune run is intentionally aborted when the audit write fails — " + "re-running after fixing the underlying I/O issue is the supported " + "recovery path." + ) + + def __init__( + self, + message: str, + *, + cause: BaseException, + remediation: str | None = None, + ) -> None: + self.cause = cause + super().__init__(message, remediation=remediation) + # Chain the underlying I/O cause so ``raise PruneAuditWriteError(...)`` + # exposes ``exc.__cause__`` for callers that branch on the OS-level + # detail (mirrors ``raise X from cause``). + self.__cause__ = cause + + +class PruneAuditRecordTooLargeError(PruneError): + """A prune-audit JSONL record would exceed the POSIX atomic-append size cap. + + POSIX guarantees ``write(2)`` is atomic only for payloads up to + ``PIPE_BUF`` bytes (typically 4 KiB on Linux). The prune-audit writer + (DEC-016) enforces a size cap to keep concurrent appends from + interleaving partial records. Mirrors safety's + :class:`signalforge.safety.errors.AuditRecordTooLargeError` and draft's + :class:`signalforge.draft.errors.LLMResponseAuditRecordTooLargeError`. + + Raised BEFORE any file is opened, so an oversize record leaves no + on-disk artefact — the orchestrator sees the typed error and aborts + the prune run rather than emitting a partial receipt. + """ + + default_remediation: ClassVar[str] = ( + "Prune-audit records must stay under the configured byte limit for " + "atomic concurrent appends; the decision payload was unusually " + "large. Consider tightening `capture_failure_rows` in signalforge.yml " + "or trimming `compiled_sql` size by simplifying the candidate test. " + "Note: 4 KB is the POSIX-atomic-append guarantee on Linux — exceeding " + "it makes concurrent writers unsafe." + ) + + def __init__(self, size: int, limit: int, *, remediation: str | None = None) -> None: + self.size = size + self.limit = limit + message = f"Prune audit record size {size} exceeds atomic-append limit {limit}." + if remediation is None: + remediation = ( + f"Prune-audit records must stay under {limit} bytes for " + "atomic concurrent appends; reduce capture_failure_rows or " + "trim the compiled_sql payload." + ) + super().__init__(message, remediation=remediation) + + +# Sorted alphabetically (mirrors safety/draft/warehouse error modules). +__all__ = [ + "PruneAuditRecordTooLargeError", + "PruneAuditWriteError", + "PruneConfigError", + "PruneError", + "PruneTimeoutError", + "PruneTrustedModelNotFoundError", +] diff --git a/tests/prune/test_errors.py b/tests/prune/test_errors.py new file mode 100644 index 00000000..ef5b3622 --- /dev/null +++ b/tests/prune/test_errors.py @@ -0,0 +1,120 @@ +"""Tests for ``signalforge.prune.errors`` (US-006, DEC-006, DEC-022). + +Covers the six-class typed exception hierarchy: every distinct prune-layer +failure mode (config load, trusted-models manifest mismatch, total-budget +timeout, fail-closed audit-write, audit-record size cap) gets a typed +exception so the orchestrator / CLI can pattern-match on type rather than +sniffing message text. + +Each subclass renders message + ``↳ Remediation:`` and quotes user-supplied +strings via :func:`repr` so adversarial input — embedded quotes, control +chars, ANSI escapes — cannot smuggle special characters into log viewers +or error messages. +""" + +from __future__ import annotations + +import pytest + +from signalforge.errors import SignalForgeError +from signalforge.prune.errors import ( + PruneAuditRecordTooLargeError, + PruneAuditWriteError, + PruneConfigError, + PruneError, + PruneTimeoutError, + PruneTrustedModelNotFoundError, +) + + +def test_prune_error_renders_message_and_remediation() -> None: + """Base ``__str__`` renders message and remediation on separate lines.""" + rendered = str(PruneError("the SQL was malformed")) + assert "the SQL was malformed" in rendered + assert "↳ Remediation:" in rendered + + +@pytest.mark.parametrize( + "exc", + [ + PruneConfigError("invalid prune config"), + PruneTrustedModelNotFoundError(unique_id="model.shop.customers"), + PruneTimeoutError("total budget exceeded"), + PruneAuditWriteError("fsync failed", cause=OSError("disk full")), + PruneAuditRecordTooLargeError(size=5000, limit=4000), + ], +) +def test_subclasses_have_default_remediation(exc: PruneError) -> None: + """Each concrete subclass exposes a non-empty remediation string when + no explicit ``remediation=`` kwarg is supplied. + + The remediation is either the class-level ``default_remediation`` + verbatim or a templated form derived from it (e.g. + :class:`PruneAuditRecordTooLargeError` formats the limit number into + the remediation at construction time — matches the safety / draft + precedent for the same shape of error). + """ + # Class-level default is set to something concrete, not the base sentinel. + assert "(no remediation set" not in type(exc).default_remediation + assert type(exc).default_remediation.strip() != "" + # Instance remediation is non-empty (constructor either uses the class + # default verbatim or substitutes a templated form derived from it). + assert exc.remediation.strip() != "" + assert "(no remediation set" not in exc.remediation + + +def test_prune_trusted_model_not_found_repr_quotes_unique_id() -> None: + """A unique_id containing an ANSI escape MUST NOT render as a raw + escape sequence (DEC-022 — log-injection defence). + + The value is rendered via ``repr()`` so ``\\x1b`` shows as the literal + four characters ``\\x1b`` rather than the actual ESC byte. + """ + exc = PruneTrustedModelNotFoundError(unique_id="model.shop.\x1b[31mevil") + rendered = str(exc) + # The raw ANSI escape (single byte 0x1b) MUST NOT appear in output. + assert "\x1b" not in rendered + # repr()-quoted form: the four literal chars ``\x1b`` MUST appear. + assert "\\x1b" in rendered + # The exception still exposes the original (un-quoted) unique_id field + # so callers can branch on it. + assert exc.unique_id == "model.shop.\x1b[31mevil" + + +def test_prune_audit_write_error_carries_cause() -> None: + """``PruneAuditWriteError`` carries the underlying I/O error as + ``__cause__`` so ``except ... as exc: exc.__cause__`` works for + callers that need to log the OS-level detail.""" + cause = OSError("disk full") + exc = PruneAuditWriteError("fsync failed", cause=cause) + assert exc.__cause__ is cause + # The attribute is also exposed for direct introspection. + assert exc.cause is cause + + +def test_prune_audit_record_too_large_error_carries_size_and_limit() -> None: + """``PruneAuditRecordTooLargeError`` exposes both numbers on the + instance and renders both into the message so the operator can see + the gap at a glance.""" + exc = PruneAuditRecordTooLargeError(size=5000, limit=4000) + assert exc.size == 5000 + assert exc.limit == 4000 + rendered = str(exc) + assert "5000" in rendered + assert "4000" in rendered + + +def test_subclass_inheritance_chain() -> None: + """Every prune error subclasses ``PruneError`` (and through it, + :class:`signalforge.errors.SignalForgeError`). + + ``PruneTrustedModelNotFoundError`` is a :class:`PruneConfigError` + subclass so callers that catch "any config-shaped failure" get + trusted-models mismatches for free (DEC-006). + """ + assert issubclass(PruneTrustedModelNotFoundError, PruneConfigError) + assert issubclass(PruneConfigError, PruneError) + assert issubclass(PruneTimeoutError, PruneError) + assert issubclass(PruneAuditWriteError, PruneError) + assert issubclass(PruneAuditRecordTooLargeError, PruneError) + assert issubclass(PruneError, SignalForgeError) From 9e6eb1dfc3344ccf0e07ee46aff61d3dcee195b1 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:09:56 -0700 Subject: [PATCH 07/24] bd_1-scaffolding-y8y.6: drop spurious signalforge.errors module The worker's first commit added signalforge/errors.SignalForgeError to satisfy the plan's PruneError(SignalForgeError) note, but every existing layer (Safety/Draft/Warehouse/Manifest) subclasses Exception directly with no shared root. Adding a project-wide root for prune alone left the codebase inconsistent. Match the established pattern: PruneError(Exception). The plan note referencing SignalForgeError was a planning bug; the precedent in the other layers is the source of truth. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/errors.py | 36 --------------------------------- src/signalforge/prune/errors.py | 4 +--- tests/prune/test_errors.py | 8 ++++---- 3 files changed, 5 insertions(+), 43 deletions(-) delete mode 100644 src/signalforge/errors.py diff --git a/src/signalforge/errors.py b/src/signalforge/errors.py deleted file mode 100644 index eff05762..00000000 --- a/src/signalforge/errors.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Project-wide error base class. - -:class:`SignalForgeError` is the conceptual root of every typed exception -SignalForge raises. Each layer (manifest, warehouse, safety, llm, draft, -prune, ...) defines its own intermediate base — :class:`SafetyError`, -:class:`WarehouseError`, :class:`PruneError`, etc. — so callers can -``except Error`` to catch every failure that originated inside that -layer without sniffing message text. - -A caller that wants to catch "any SignalForge-typed failure" (e.g. a top-level -CLI handler that wants to render a remediation line and exit non-zero) can -catch :class:`SignalForgeError`. Anything that isn't a subclass is, by -definition, an unexpected programming error and should propagate. - -This module deliberately stays minimal: no message formatting, no -``default_remediation``, no ``__str__`` override. Each layer's intermediate -base owns its own rendering convention (the ``message + ↳ Remediation:`` -shape established by :mod:`signalforge.manifest.errors` and mirrored across -the other layers). Centralising the rendering here would force every layer -to inherit it whether they want to or not. -""" - -from __future__ import annotations - - -class SignalForgeError(Exception): - """Project-wide root of every typed SignalForge exception. - - Each subpackage's intermediate base (e.g. :class:`PruneError`, - :class:`SafetyError`) inherits from this class. Callers that want to - catch any SignalForge-typed failure may catch this directly; callers - that want layer-specific catches should catch the intermediate base. - """ - - -__all__ = ["SignalForgeError"] diff --git a/src/signalforge/prune/errors.py b/src/signalforge/prune/errors.py index b454cff2..c594356e 100644 --- a/src/signalforge/prune/errors.py +++ b/src/signalforge/prune/errors.py @@ -27,8 +27,6 @@ from typing import ClassVar -from signalforge.errors import SignalForgeError - def _format_value(v: object) -> str: """Quote a user-supplied value via ``repr()`` for safe inclusion in @@ -44,7 +42,7 @@ def _format_value(v: object) -> str: return repr(v) -class PruneError(SignalForgeError): +class PruneError(Exception): """Base class for all prune-layer errors. Subclasses set a class-level ``default_remediation`` string; instances diff --git a/tests/prune/test_errors.py b/tests/prune/test_errors.py index ef5b3622..3e06b28f 100644 --- a/tests/prune/test_errors.py +++ b/tests/prune/test_errors.py @@ -16,7 +16,6 @@ import pytest -from signalforge.errors import SignalForgeError from signalforge.prune.errors import ( PruneAuditRecordTooLargeError, PruneAuditWriteError, @@ -105,8 +104,9 @@ def test_prune_audit_record_too_large_error_carries_size_and_limit() -> None: def test_subclass_inheritance_chain() -> None: - """Every prune error subclasses ``PruneError`` (and through it, - :class:`signalforge.errors.SignalForgeError`). + """Every prune error subclasses :class:`PruneError`, which itself + subclasses ``Exception`` directly — same per-layer pattern as + ``SafetyError``, ``DraftError``, ``WarehouseError``, ``ManifestError``. ``PruneTrustedModelNotFoundError`` is a :class:`PruneConfigError` subclass so callers that catch "any config-shaped failure" get @@ -117,4 +117,4 @@ def test_subclass_inheritance_chain() -> None: assert issubclass(PruneTimeoutError, PruneError) assert issubclass(PruneAuditWriteError, PruneError) assert issubclass(PruneAuditRecordTooLargeError, PruneError) - assert issubclass(PruneError, SignalForgeError) + assert issubclass(PruneError, Exception) From 7f4f9cd0509f30809a3d74fb2c723fe992db2c0a Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:16:35 -0700 Subject: [PATCH 08/24] =?UTF-8?q?bd=5F1-scaffolding-y8y.7:=20compiler=20?= =?UTF-8?q?=E2=80=94=20failing-rows=20SQL=20x=204=20test=20types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure SQL transform: not_null, unique, accepted_values, relationships to BigQuery failing-rows SELECTs. Dialect-driven (Dialect.quote_char) so v0.2 adapters drop in without changes. Reuses warehouse._sql_safety.escape_bq_string_literal for accepted_values literals (DEC-024). _RequiresFutureData sentinel for relationships(to: unknown) instead of an exception (DEC-006). NULL- exclusion matches dbt-core verbatim (DEC-023). Plan: plans/super/6-prune-engine.md (DEC-023, DEC-024, DEC-025, DEC-026). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/prune/compiler.py | 267 +++++++++++++- .../prune/compiled_sql/accepted_values.sql | 1 + .../fixtures/prune/compiled_sql/not_null.sql | 1 + .../prune/compiled_sql/relationships.sql | 1 + tests/fixtures/prune/compiled_sql/unique.sql | 1 + tests/prune/test_compiler.py | 348 ++++++++++++++++++ 6 files changed, 610 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/prune/compiled_sql/accepted_values.sql create mode 100644 tests/fixtures/prune/compiled_sql/not_null.sql create mode 100644 tests/fixtures/prune/compiled_sql/relationships.sql create mode 100644 tests/fixtures/prune/compiled_sql/unique.sql create mode 100644 tests/prune/test_compiler.py diff --git a/src/signalforge/prune/compiler.py b/src/signalforge/prune/compiler.py index 05c2fd8f..c6c3dad7 100644 --- a/src/signalforge/prune/compiler.py +++ b/src/signalforge/prune/compiler.py @@ -1,14 +1,263 @@ """Candidate-test SQL compiler. -Translates each variant of the drafter's `CandidateTest` discriminated union -(`not_null`, `unique`, `accepted_values`, `relationships`) into a failing-rows -SELECT statement using `WarehouseAdapter.dialect()` for quote character and -identifier casing. Matches dbt-core's NULL-exclusion conventions verbatim so -prune verdicts agree with `dbt test` runtime verdicts. Quote-escapes -user-controlled values (notably `accepted_values.values`) before SQL -interpolation; trusts adapter-validated identifiers on `TableRef`. - -See plans/super/6-prune-engine.md for the full design. +Translates each variant of the drafter's :class:`CandidateTest` discriminated +union (``not_null``, ``unique``, ``accepted_values``, ``relationships``) into +a failing-rows SELECT statement using :class:`Dialect.quote_char` for +identifier quoting. Matches dbt-core's NULL-exclusion conventions verbatim +so prune verdicts agree with ``dbt test`` runtime verdicts. Quote-escapes +user-controlled values (notably ``accepted_values.values``) before SQL +interpolation; trusts adapter-validated identifiers on :class:`TableRef`. + +Design commitments operationalised here: + +* **DEC-023** — Every failing-rows SELECT excludes ``NULL`` from the + candidate set the way dbt-core does (``unique`` and ``accepted_values`` + both filter ``IS NOT NULL`` before the violation predicate; ``not_null`` + is the inverse, selecting only ``IS NULL`` rows). Diverging from + dbt-core's conventions would cause prune verdicts to disagree with + ``dbt test`` verdicts on the same model — a UX-breaking inconsistency. +* **DEC-024** — :func:`signalforge.warehouse._sql_safety.escape_bq_string_literal` + is the single string-literal escape seam shared between the partition + filter renderer (US-004) and the ``accepted_values`` compiler. Reusing it + keeps the escape rules in lockstep across the warehouse and prune + layers; a divergence would surface as a SQL-injection seam in either + direction. +* **DEC-025** — :func:`_compile_test` dispatches on + :attr:`Dialect.quote_char` rather than on dialect ``name``. v0.2 ports + (Snowflake's double-quoted identifiers; Postgres') drop in by adding a + sibling :class:`Dialect` constant in + :mod:`signalforge.warehouse.models` — the compiler does not need to + branch on warehouse name. +* **DEC-026** — A ``relationships`` test whose ``to`` parent model is not + present in the loaded manifest returns a :class:`_RequiresFutureData` + sentinel. The orchestrator routes the sentinel to the + ``requires-future-data`` drop reason without issuing a warehouse call. + Returning a sentinel rather than raising keeps compilation total: every + candidate produces either compiled SQL or a structured no-op, and the + caller distinguishes via :func:`isinstance`. +* **DEC-005** — :func:`_compute_compiled_sql_hash` mirrors the + blake2b-8 / 16-hex-char convention from + :mod:`signalforge.draft.audit` so prune-audit and response-audit + records use the same hash domain. + +The module is a pure transform: no warehouse calls, no logging, no I/O. +Compilation never raises (DEC-006). The returned strings are passed to +:meth:`signalforge.warehouse.WarehouseAdapter.run_test_sql` by the +orchestrator (US-008+). + +See ``plans/super/6-prune-engine.md`` for the full design. """ from __future__ import annotations + +from dataclasses import dataclass +from hashlib import blake2b +from typing import TYPE_CHECKING + +from signalforge.draft.models import ( + CandidateTest, + CandidateTestAcceptedValues, + CandidateTestNotNull, + CandidateTestRelationships, + CandidateTestUnique, +) +from signalforge.warehouse._sql_safety import escape_bq_string_literal +from signalforge.warehouse.models import TableRef + +if TYPE_CHECKING: + from signalforge.manifest.models import Manifest + from signalforge.warehouse.models import Dialect + + +@dataclass(frozen=True, slots=True) +class _RequiresFutureData: + """Sentinel returned by :func:`_compile_test` when a ``relationships`` + test references a manifest-absent parent model. + + The orchestrator routes the sentinel to ``drop_reason=requires-future-data`` + without issuing a warehouse call. The :attr:`reason` field carries the + human-readable why-line that surfaces in the prune diff (DEC-026). + """ + + reason: str + + +def _quote(identifier: str, quote_char: str) -> str: + """Wrap ``identifier`` in ``quote_char`` for SQL embedding. + + The compiler trusts adapter-validated identifiers on + :class:`TableRef` — :func:`signalforge.warehouse._sql_safety.validate_identifier` + runs at :class:`TableRef` construction time and rejects anything + outside ``[A-Za-z_][A-Za-z0-9_]*``, so we do not re-validate here. + Column names from :class:`signalforge.draft.models.CandidateTest` + pass through the drafter's anchor-contract validator before reaching + the compiler; the orchestrator's :class:`TableRef` construction is + the gate, not this function. + """ + return f"{quote_char}{identifier}{quote_char}" + + +def _qualified_table_name(table_ref: TableRef, quote_char: str) -> str: + """Render a fully-qualified ``project.dataset.name`` table identifier. + + Matches the BigQuery convention ``\\`project.dataset.table\\``` (entire + qualified path inside one pair of backticks). Dialects with a different + quote_char get the same shape with their own quote character so v0.2 + ports drop in without compiler changes (DEC-025). + """ + if table_ref.project is None: + return f"{quote_char}{table_ref.dataset}.{table_ref.name}{quote_char}" + return f"{quote_char}{table_ref.project}.{table_ref.dataset}.{table_ref.name}{quote_char}" + + +def _compile_not_null( + test: CandidateTestNotNull, + table_ref: TableRef, + quote_char: str, +) -> str: + """Compile ``not_null(col)`` to ``SELECT col FROM t WHERE col IS NULL``.""" + col = _quote(test.column, quote_char) + table = _qualified_table_name(table_ref, quote_char) + return f"SELECT {col} FROM {table} WHERE {col} IS NULL" + + +def _compile_unique( + test: CandidateTestUnique, + table_ref: TableRef, + quote_char: str, +) -> str: + """Compile ``unique(col)`` to a GROUP BY ... HAVING COUNT(*) > 1. + + DEC-023 NULL-exclusion: ``IS NOT NULL`` filters NULL rows out of the + grouped set, matching dbt-core (multiple NULLs in a column do not + violate uniqueness in dbt's convention). + """ + col = _quote(test.column, quote_char) + table = _qualified_table_name(table_ref, quote_char) + return f"SELECT {col} FROM {table} WHERE {col} IS NOT NULL GROUP BY {col} HAVING COUNT(*) > 1" + + +def _compile_accepted_values( + test: CandidateTestAcceptedValues, + table_ref: TableRef, + quote_char: str, +) -> str: + """Compile ``accepted_values(col, values)`` to a ``NOT IN`` predicate. + + Each value goes through + :func:`signalforge.warehouse._sql_safety.escape_bq_string_literal` + (DEC-024) so embedded quotes, backslashes, newlines, and ANSI escapes + cannot break out of the literal. The escaped value is wrapped in + single quotes; the resulting SQL passes + :func:`signalforge.warehouse._sql_safety.validate_test_sql` even for + adversarial inputs (the entire injection attempt stays inside the + quoted string). + """ + col = _quote(test.column, quote_char) + table = _qualified_table_name(table_ref, quote_char) + rendered_values = ", ".join(f"'{escape_bq_string_literal(v)}'" for v in test.values) + return f"SELECT {col} FROM {table} WHERE {col} IS NOT NULL AND {col} NOT IN ({rendered_values})" + + +def _resolve_parent_table_ref( + parent_name: str, + manifest: Manifest, +) -> TableRef | _RequiresFutureData: + """Resolve a ``relationships(to=parent_name)`` to its parent TableRef. + + The drafter's :class:`CandidateTestRelationships.to` field carries + only the parent model's :attr:`Model.name` (not a full ``unique_id``); + :class:`Manifest` indexes by ``unique_id``, so the lookup scans + :attr:`Manifest.nodes` for a model whose :attr:`Model.name` matches. + + Returns a :class:`_RequiresFutureData` sentinel when no match is + found (DEC-026). When a match is found, returns the parent's + :class:`TableRef` via :meth:`TableRef.from_model`; that call may + raise :class:`ManifestProjectNotFoundError` or + :class:`ManifestSchemaNotFoundError` if the parent model lacks + ``database`` / ``schema`` — those are manifest-shape problems and + propagate, not prune problems to swallow. + """ + for parent_model in manifest.nodes.values(): + if parent_model.name == parent_name: + return TableRef.from_model(parent_model) + return _RequiresFutureData(reason=f"relationships parent {parent_name!r} not in manifest") + + +def _compile_relationships( + test: CandidateTestRelationships, + table_ref: TableRef, + quote_char: str, + manifest: Manifest, +) -> str | _RequiresFutureData: + """Compile ``relationships(child_col, to=parent, field=parent_col)``. + + Renders a LEFT JOIN orphan-detection SELECT: rows in the child where + the foreign key is non-null but the parent has no matching row. + + Returns a :class:`_RequiresFutureData` sentinel when the parent + model is not in the manifest (DEC-026); the orchestrator routes + that to the ``requires-future-data`` drop reason without issuing a + warehouse call. + """ + parent_table_ref = _resolve_parent_table_ref(test.to, manifest) + if isinstance(parent_table_ref, _RequiresFutureData): + return parent_table_ref + + child_col = _quote(test.column, quote_char) + parent_col = _quote(test.field, quote_char) + child_table = _qualified_table_name(table_ref, quote_char) + parent_table = _qualified_table_name(parent_table_ref, quote_char) + return ( + f"SELECT child.{child_col} " + f"FROM {child_table} AS child " + f"LEFT JOIN {parent_table} AS parent " + f"ON child.{child_col} = parent.{parent_col} " + f"WHERE child.{child_col} IS NOT NULL AND parent.{parent_col} IS NULL" + ) + + +def _compile_test( + test: CandidateTest, + table_ref: TableRef, + dialect: Dialect, + manifest: Manifest, +) -> str | _RequiresFutureData: + """Render a candidate test as a failing-rows SELECT. + + The returned string is a SELECT whose rows are violations: zero rows + means the test passes; one or more rows mean it fails. The adapter's + :meth:`signalforge.warehouse.WarehouseAdapter.run_test_sql` wraps the + returned string with ``SELECT COUNT(*) AS failures FROM (...) AS t`` + (plus an optional ``ARRAY_AGG`` for sample-failure capture). + + Dispatch is on :attr:`Dialect.quote_char` (DEC-025) so v0.2 + Snowflake / Postgres adapters drop in by adding a sibling + :class:`Dialect` constant — no compiler changes required. + + A ``relationships`` test whose parent is not in the manifest returns + :class:`_RequiresFutureData` rather than raising (DEC-006, DEC-026); + the orchestrator distinguishes the two return shapes via + :func:`isinstance`. + """ + quote_char = dialect.quote_char + if isinstance(test, CandidateTestNotNull): + return _compile_not_null(test, table_ref, quote_char) + if isinstance(test, CandidateTestUnique): + return _compile_unique(test, table_ref, quote_char) + if isinstance(test, CandidateTestAcceptedValues): + return _compile_accepted_values(test, table_ref, quote_char) + # Only CandidateTestRelationships remains in the discriminated union. + return _compile_relationships(test, table_ref, quote_char, manifest) + + +def _compute_compiled_sql_hash(sql: str) -> str: + """Compute the 16-hex-char blake2b-8 hash of a compiled SQL string. + + Mirrors the hash convention in :mod:`signalforge.draft.audit` + (DEC-005). The prune-audit writer (US-009) records this hash on + every :class:`signalforge.prune.models.PruneDecision` so a reviewer + can correlate decisions across the prune-audit JSONL and the + response-audit JSONL by hash. + """ + return blake2b(sql.encode("utf-8"), digest_size=8).hexdigest() diff --git a/tests/fixtures/prune/compiled_sql/accepted_values.sql b/tests/fixtures/prune/compiled_sql/accepted_values.sql new file mode 100644 index 00000000..48d7a5c5 --- /dev/null +++ b/tests/fixtures/prune/compiled_sql/accepted_values.sql @@ -0,0 +1 @@ +SELECT `status` FROM `fake_project.dataset.orders` WHERE `status` IS NOT NULL AND `status` NOT IN ('placed', 'shipped', 'cancelled') \ No newline at end of file diff --git a/tests/fixtures/prune/compiled_sql/not_null.sql b/tests/fixtures/prune/compiled_sql/not_null.sql new file mode 100644 index 00000000..7fe178b3 --- /dev/null +++ b/tests/fixtures/prune/compiled_sql/not_null.sql @@ -0,0 +1 @@ +SELECT `customer_id` FROM `fake_project.dataset.orders` WHERE `customer_id` IS NULL \ No newline at end of file diff --git a/tests/fixtures/prune/compiled_sql/relationships.sql b/tests/fixtures/prune/compiled_sql/relationships.sql new file mode 100644 index 00000000..c92b5617 --- /dev/null +++ b/tests/fixtures/prune/compiled_sql/relationships.sql @@ -0,0 +1 @@ +SELECT child.`customer_id` FROM `fake_project.dataset.orders` AS child LEFT JOIN `fake_project.dataset.customers` AS parent ON child.`customer_id` = parent.`id` WHERE child.`customer_id` IS NOT NULL AND parent.`id` IS NULL \ No newline at end of file diff --git a/tests/fixtures/prune/compiled_sql/unique.sql b/tests/fixtures/prune/compiled_sql/unique.sql new file mode 100644 index 00000000..5cd8f305 --- /dev/null +++ b/tests/fixtures/prune/compiled_sql/unique.sql @@ -0,0 +1 @@ +SELECT `customer_id` FROM `fake_project.dataset.orders` WHERE `customer_id` IS NOT NULL GROUP BY `customer_id` HAVING COUNT(*) > 1 \ No newline at end of file diff --git a/tests/prune/test_compiler.py b/tests/prune/test_compiler.py new file mode 100644 index 00000000..6d7c91ce --- /dev/null +++ b/tests/prune/test_compiler.py @@ -0,0 +1,348 @@ +"""Tests for ``signalforge.prune.compiler`` (US-007). + +Pinned snapshot fixtures verify the byte-exact failing-rows SELECT for the +four candidate-test variants (``not_null``, ``unique``, ``accepted_values``, +``relationships``) against the BigQuery dialect. The four NULL-exclusion +checks pin DEC-023 (matches dbt-core verbatim — diverging would cause +prune verdicts to disagree with ``dbt test`` runtime verdicts on the same +model). The escape-hardening tests pin DEC-024 (every adversarial value — +embedded single quotes, backslashes, newlines, ANSI escapes, full SQL +injection attempts — stays inside the quoted string and the wrapped SQL +passes :func:`signalforge.warehouse._sql_safety.validate_test_sql`). The +``relationships`` parent-resolution and dialect-dispatch tests pin +DEC-025 (custom :class:`Dialect` quote_char dispatches without a +warehouse-specific code path) and DEC-026 (manifest-absent parent +returns a sentinel rather than raising). The hash test pins DEC-005 +(blake2b-8 / 16-hex-char convention shared with +:mod:`signalforge.draft.audit`). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from signalforge.draft.models import ( + CandidateTestAcceptedValues, + CandidateTestNotNull, + CandidateTestRelationships, + CandidateTestUnique, +) +from signalforge.manifest.models import Column, Manifest, Model +from signalforge.prune.compiler import ( + _compile_test, + _compute_compiled_sql_hash, + _RequiresFutureData, +) +from signalforge.warehouse._sql_safety import validate_test_sql +from signalforge.warehouse.models import BIGQUERY_DIALECT, Dialect, TableRef + +_FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" / "prune" / "compiled_sql" + + +def _read_fixture(name: str) -> str: + """Read a snapshot fixture file as raw text (no normalisation).""" + return (_FIXTURES_DIR / name).read_text(encoding="utf-8") + + +def _make_orders_table_ref() -> TableRef: + return TableRef(project="fake_project", dataset="dataset", name="orders") + + +def _make_orders_model() -> Model: + return Model( + unique_id="model.shop.orders", + name="orders", + resource_type="model", + package_name="shop", + original_file_path="models/orders.sql", + path="orders.sql", + database="fake_project", + schema="dataset", # type: ignore[call-arg] + columns={"customer_id": Column(name="customer_id")}, + raw_code="select 1", + ) + + +def _make_customers_model() -> Model: + return Model( + unique_id="model.shop.customers", + name="customers", + resource_type="model", + package_name="shop", + original_file_path="models/customers.sql", + path="customers.sql", + database="fake_project", + schema="dataset", # type: ignore[call-arg] + columns={"id": Column(name="id")}, + raw_code="select 1", + ) + + +def _make_manifest(*, with_customers: bool = True) -> Manifest: + nodes: dict[str, Model] = {"model.shop.orders": _make_orders_model()} + if with_customers: + nodes["model.shop.customers"] = _make_customers_model() + return Manifest(metadata={"dbt_schema_version": "v12"}, nodes=nodes) + + +# --------------------------------------------------------------------------- +# Snapshot tests — pinned byte-exact output for each variant. +# --------------------------------------------------------------------------- + + +def test_compile_not_null_matches_snapshot() -> None: + expected = _read_fixture("not_null.sql") + test = CandidateTestNotNull(column="customer_id") + actual = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert actual == expected + + +def test_compile_unique_matches_snapshot() -> None: + expected = _read_fixture("unique.sql") + test = CandidateTestUnique(column="customer_id") + actual = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert actual == expected + + +def test_compile_accepted_values_matches_snapshot() -> None: + expected = _read_fixture("accepted_values.sql") + test = CandidateTestAcceptedValues(column="status", values=("placed", "shipped", "cancelled")) + actual = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert actual == expected + + +def test_compile_relationships_matches_snapshot() -> None: + expected = _read_fixture("relationships.sql") + test = CandidateTestRelationships(column="customer_id", to="customers", field="id") + actual = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert actual == expected + + +# --------------------------------------------------------------------------- +# DEC-023: NULL-exclusion conventions match dbt-core. +# --------------------------------------------------------------------------- + + +def test_unique_sql_includes_null_exclusion() -> None: + test = CandidateTestUnique(column="customer_id") + sql = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(sql, str) + assert "IS NOT NULL" in sql + + +def test_accepted_values_sql_includes_null_exclusion() -> None: + test = CandidateTestAcceptedValues(column="status", values=("a",)) + sql = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(sql, str) + assert "IS NOT NULL" in sql + assert "NOT IN" in sql + + +def test_relationships_sql_includes_child_null_exclusion() -> None: + test = CandidateTestRelationships(column="customer_id", to="customers", field="id") + sql = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(sql, str) + # Child-side: filter rows where the foreign key is non-null (orphan candidates). + assert "child.`customer_id` IS NOT NULL" in sql + # Parent-side: the orphan check looks for parents that don't exist. + assert "parent.`id` IS NULL" in sql + + +# --------------------------------------------------------------------------- +# DEC-024: accepted_values value escaping (every adversarial input stays inside +# the quoted literal; the wrapped SQL passes validate_test_sql). +# --------------------------------------------------------------------------- + + +def test_accepted_values_escapes_single_quote() -> None: + test = CandidateTestAcceptedValues(column="name", values=("O'Brien",)) + sql = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(sql, str) + # The escaped form is `O\'Brien` wrapped in single quotes: 'O\'Brien'. + assert r"'O\'Brien'" in sql + + +def test_accepted_values_escapes_backslash() -> None: + test = CandidateTestAcceptedValues(column="name", values=("a\\b",)) + sql = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(sql, str) + # Backslash gets doubled; expect the literal `a\\b` (four chars in the + # rendered SQL: 'a', '\\', '\\', 'b') wrapped in single quotes. + assert "'a\\\\b'" in sql + + +def test_accepted_values_escapes_newline() -> None: + test = CandidateTestAcceptedValues(column="name", values=("a\nb",)) + sql = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(sql, str) + # The newline should NOT survive raw — it'd terminate the literal in BQ + # standard SQL. escape_bq_string_literal turns it into `\n` (two chars). + assert "\n" not in sql + assert "'a\\nb'" in sql + # The wrapped SQL (mirroring the adapter's run_test_sql wrap) must + # pass the cheap-rejects validator. + wrapped = f"SELECT COUNT(*) FROM ({sql}) AS t" + validate_test_sql(wrapped) # raises if rejected + + +def test_accepted_values_escapes_ansi() -> None: + test = CandidateTestAcceptedValues(column="name", values=("\x1b[31mfoo",)) + sql = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(sql, str) + # The ANSI-bytes string is opaque inside the BQ literal; the wrapped + # SQL must still pass the cheap-rejects validator. + wrapped = f"SELECT COUNT(*) FROM ({sql}) AS t" + validate_test_sql(wrapped) # raises if rejected + + +def test_accepted_values_escapes_sql_injection_attempt() -> None: + """Verify the entire injection attempt stays inside the quoted literal. + + Strip the single-quoted segments out of the rendered SQL and confirm + no ``;`` or ``--`` survives in the residue. Then pass the wrapped SQL + through the adapter-level validator and confirm it does NOT raise. + """ + payload = "; DROP TABLE x;--" + test = CandidateTestAcceptedValues(column="name", values=(payload,)) + sql = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(sql, str) + + # Strip every single-quoted literal segment. The residue is the + # SQL skeleton without any user-supplied bytes. + residue: list[str] = [] + in_literal = False + i = 0 + while i < len(sql): + ch = sql[i] + if ch == "'": + if in_literal and i + 1 < len(sql) and sql[i + 1] == "'": + # Doubled quote escape — stay inside the literal. + i += 2 + continue + in_literal = not in_literal + i += 1 + continue + if not in_literal: + residue.append(ch) + i += 1 + residue_str = "".join(residue) + assert ";" not in residue_str + assert "--" not in residue_str + + # Wrap the way the adapter does and confirm the cheap-rejects pass. + wrapped = f"SELECT COUNT(*) FROM ({sql}) AS t" + validate_test_sql(wrapped) + + +# --------------------------------------------------------------------------- +# DEC-026: relationships(to: unknown) returns a sentinel (not a string). +# --------------------------------------------------------------------------- + + +def test_relationships_returns_requires_future_data_when_parent_missing() -> None: + test = CandidateTestRelationships(column="customer_id", to="nonexistent_model", field="id") + result = _compile_test( + test, + _make_orders_table_ref(), + BIGQUERY_DIALECT, + _make_manifest(with_customers=False), + ) + assert isinstance(result, _RequiresFutureData) + assert "nonexistent_model" in result.reason + + +def test_relationships_resolves_parent_via_manifest_name_lookup() -> None: + """Manifest indexes by ``unique_id``; ``to`` is a model ``name``. + + The lookup must scan ``manifest.nodes.values()`` to match + ``model.name == to`` rather than a direct unique_id key lookup. + """ + test = CandidateTestRelationships(column="customer_id", to="customers", field="id") + result = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(result, str) + # Parent table is rendered as `fake_project.dataset.customers` (drawn + # from the parent model's database / schema_ / name). + assert "`fake_project.dataset.customers`" in result + + +# --------------------------------------------------------------------------- +# DEC-025: dispatch on Dialect.quote_char (v0.2 readiness). +# --------------------------------------------------------------------------- + + +def test_compiler_dispatches_on_dialect_quote_char() -> None: + """Custom :class:`Dialect` with ``quote_char='"'`` produces double-quoted SQL. + + v0.1 ships only :data:`BIGQUERY_DIALECT` (backtick-quoted); this + test pins the dispatch so a v0.2 Snowflake / Postgres dialect drops + in without compiler changes (DEC-025). + """ + snowflake_like = Dialect( + name="snowflake_like", + supports_tablesample=True, + supports_qualify=True, + quote_char='"', + identifier_case="preserve", + ) + test = CandidateTestNotNull(column="customer_id") + sql = _compile_test(test, _make_orders_table_ref(), snowflake_like, _make_manifest()) + assert isinstance(sql, str) + assert '"customer_id"' in sql + assert '"fake_project.dataset.orders"' in sql + # No backticks anywhere — the BQ-specific quote should not appear. + assert "`" not in sql + + +# --------------------------------------------------------------------------- +# Determinism + hash convention. +# --------------------------------------------------------------------------- + + +def test_compile_is_deterministic() -> None: + test = CandidateTestAcceptedValues(column="status", values=("placed", "shipped", "cancelled")) + a = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + b = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert a == b + + +def test_compute_compiled_sql_hash_is_blake2b_8() -> None: + """16-hex-char digest from blake2b with ``digest_size=8``. + + Pinned expected value lets a regression in the hash-domain choice + surface immediately (DEC-005 — must match draft.audit's convention). + """ + sql = "SELECT 1" + digest = _compute_compiled_sql_hash(sql) + assert len(digest) == 16 + assert all(c in "0123456789abcdef" for c in digest) + # Pinned: blake2b("SELECT 1", digest_size=8).hexdigest() + assert digest == "60e6b55ff57fdf38" + + +def test_compute_compiled_sql_hash_distinguishes_distinct_inputs() -> None: + a = _compute_compiled_sql_hash("SELECT 1") + b = _compute_compiled_sql_hash("SELECT 2") + assert a != b + + +# --------------------------------------------------------------------------- +# DEC-013: identifier validation runs at TableRef construction, not in the +# compiler. Confirm by asserting that an invalid identifier raises at +# TableRef construction time and never reaches the compiler. +# --------------------------------------------------------------------------- + + +def test_validate_identifier_already_done_at_table_ref_construction() -> None: + """The compiler trusts ``TableRef`` — invalid identifiers fail upstream. + + Confirms the safety boundary is at :class:`TableRef` construction + (per DEC-013), not in the compiler. A would-be SQL-injection + identifier never reaches :func:`_compile_test`; the constructor + rejects it first. + """ + from signalforge.warehouse.errors import InvalidIdentifierError + + with pytest.raises(InvalidIdentifierError): + TableRef(project="fake_project", dataset="dataset", name="orders; DROP TABLE x") From 10921f85c7d3e24f1e819040940891359e53f544 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:22:21 -0700 Subject: [PATCH 09/24] =?UTF-8?q?bd=5F1-scaffolding-y8y.8:=20prune=20audit?= =?UTF-8?q?=20=E2=80=94=20fail-closed=20prune.jsonl=20writer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PruneEvent + _write_prune_event(event, path) with O_APPEND|O_CREAT| 0o600, fsync, size cap (4000 bytes) before os.open, no try/except. Mirrors safety + draft fail-closed audit semantics. _build_prune_event is the single construction seam (DEC-018). _compute_config_hash uses sha256(...)[:16] to match safety.policy_hash (DEC-005). AST audit-completeness scan extended (tests/test_audit_completeness.py) with a fifth scan: PruneEvent construction confined to signalforge/prune/audit.py only. Plan: plans/super/6-prune-engine.md (DEC-007, DEC-014, DEC-016, DEC-018). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/prune/audit.py | 240 +++++++++++++++++++++++++++-- tests/prune/test_audit.py | 255 +++++++++++++++++++++++++++++++ tests/test_audit_completeness.py | 60 ++++++++ 3 files changed, 546 insertions(+), 9 deletions(-) create mode 100644 tests/prune/test_audit.py diff --git a/src/signalforge/prune/audit.py b/src/signalforge/prune/audit.py index 21bb68b9..930a3230 100644 --- a/src/signalforge/prune/audit.py +++ b/src/signalforge/prune/audit.py @@ -1,18 +1,240 @@ -"""Fail-closed prune-event audit writer. +"""Fail-closed JSONL prune-audit writer (US-008). -Mirrors the safety layer's audit and the drafter's response audit at the -prune-decision boundary: opens with `O_APPEND | O_CREAT | 0o600`, writes one -JSONL line per `PruneEvent`, calls `os.fsync`, closes. Catches no exceptions -internally — `OSError`, `PermissionError`, encoding failures all propagate so -an unaudited prune decision can never silently land. Size cap is checked -before any file open. AST audit-completeness scan gates `PruneEvent` -construction to this module only. +Mirrors :mod:`signalforge.safety.audit` (DEC-011 of safety-layer.md) and +:mod:`signalforge.draft.audit` (DEC-006/008/013 of llm-drafter.md) at the +prune-decision boundary: opens with ``O_APPEND | O_CREAT | 0o600``, writes +one JSONL line per :class:`PruneEvent`, calls :func:`os.fsync`, closes. +This is the *third* instance of the convention; reach for it (not ad-hoc +``Path.write_text`` calls) any time a new pipeline stage needs a durable, +fail-closed JSONL receipt. -See plans/super/6-prune-engine.md for the full design. +Three load-bearing properties, all inherited verbatim from the safety and +draft layers: + +* **Atomic concurrent appends.** ``os.open`` with ``O_APPEND`` + a single + ``os.write`` call. POSIX guarantees ``write(2)`` is atomic up to + ``PIPE_BUF`` (typically 4 KiB on Linux); the module-level + :data:`_PRUNE_AUDIT_RECORD_LIMIT_BYTES` enforces a 4000-byte cap with a + 96-byte margin so concurrent writers cannot interleave partial records. +* **Fail-closed on every error.** ``OSError`` / ``PermissionError`` / + encoding failures all propagate. Oversize records propagate as + :class:`PruneAuditRecordTooLargeError`. The caller (the engine in US-009) + wraps any non-typed propagation as :class:`PruneAuditWriteError`. **Don't** + add try/except inside :func:`_write_prune_event` — the propagation IS the + contract; an unaudited prune decision is, by definition, a kept/dropped + artefact without a receipt, exactly the failure mode the audit exists to + prevent. +* **Single construction seam (DEC-018).** :class:`PruneEvent` is constructed + only inside this module — :func:`_build_prune_event` is the helper the + engine calls. The AST audit-completeness scan in + :file:`tests/test_audit_completeness.py` rejects ``PruneEvent(...)`` calls + anywhere else. + +The :func:`_compute_config_hash` helper produces a 16-hex-char SHA-256 +digest of the canonicalised :mod:`signalforge.prune.config` block, matching +the convention :class:`signalforge.safety.models.AuditEvent` uses for +``policy_hash`` (DEC-005). Reviewers can verify all records in a run came +from the same prune config by checking the field across the JSONL. """ from __future__ import annotations +import contextlib +import json import logging +import os +import uuid +from datetime import datetime, timezone +from hashlib import sha256 +from pathlib import Path +from typing import Any, Final, Literal + +from pydantic import BaseModel, ConfigDict + +from signalforge import __version__ as _SIGNALFORGE_VERSION +from signalforge.draft.models import CandidateTest +from signalforge.prune.errors import PruneAuditRecordTooLargeError +from signalforge.prune.models import DropReason, PruneDecision, Scope _LOGGER = logging.getLogger(__name__) + +# POSIX guarantees ``write(2)`` is atomic only up to ``PIPE_BUF`` bytes +# (typically 4096 on Linux). The 96-byte margin leaves room for trailing +# newline plus any line-buffering / kernel overhead so the module's atomic- +# concurrent-append contract holds even at the size cap. Identical to +# ``signalforge.safety.audit._AUDIT_RECORD_LIMIT_BYTES`` and +# ``signalforge.draft.audit._RESPONSE_AUDIT_RECORD_LIMIT_BYTES`` by design +# (DEC-016). +_PRUNE_AUDIT_RECORD_LIMIT_BYTES: Final[int] = 4000 + +# Frozen at the constant in production code. Bump when the JSONL schema +# evolves; v0.2 readers gate on this. Mirrors safety.AuditEvent.audit_schema_version +# and draft.LLMResponseEvent.audit_schema_version. +_PRUNE_AUDIT_SCHEMA_VERSION: Final[int] = 1 + + +class PruneEvent(BaseModel): + """One JSONL audit record per :class:`PruneDecision`. + + DEC-014 — flat shape (mirrors :class:`signalforge.safety.models.AuditEvent` + and :class:`signalforge.draft.audit.LLMResponseEvent`). The decision's + fields are flattened in rather than nested under a ``decision:`` key so + a reviewer can ``jq`` over the JSONL without descending one level per + field. + + DEC-018 — construction confined to :func:`_build_prune_event` in this + module. The AST audit-completeness scan rejects direct ``PruneEvent(...)`` + construction anywhere else; the corresponding event would never reach + disk and the prune decision would be unauditable. + + Read-back-stable (``extra="ignore"`` per ``manifest-readers.md`` DEC-008 + and ``safety-layer.md`` DEC-015) — older readers tolerate forward-compat + field additions, while a one-off ``extra="forbid"`` drift detector (the + standard pattern, lands in a future US) catches silent schema expansion + before a live audit log does. + """ + + model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True) + + audit_schema_version: Literal[1] = 1 + signalforge_version: str + record_id: str + timestamp: str + config_hash: str + model_unique_id: str + test: CandidateTest + test_anchor: str + decision: Literal["kept", "dropped"] + reason: DropReason + failures: int + sampled_rows: int | None + scope: Scope + elapsed_ms: int + compiled_sql_hash: str + compiled_sql: str + why: str + sample_failures: tuple[dict[str, Any], ...] | None = None + + +def _build_prune_event( + *, + decision: PruneDecision, + model_unique_id: str, + config_hash: str, +) -> PruneEvent: + """Construct a :class:`PruneEvent` from one :class:`PruneDecision`. + + DEC-018 — the single construction seam. The AST audit-completeness scan + in :file:`tests/test_audit_completeness.py` rejects ``PruneEvent(...)`` + calls anywhere outside this module; the engine (US-009) calls this + helper rather than constructing the event itself, so the audit-write + seam stays the only code path that produces a record. + + DEC-014 — flattens the decision's fields into the event rather than + nesting under a ``decision:`` key. + + Stamps ``signalforge_version`` from :data:`signalforge.__version__`, + generates a fresh ``uuid4`` ``record_id``, and writes an ISO-8601 UTC + timestamp with microsecond precision and a trailing ``Z`` suffix. + """ + return PruneEvent( + signalforge_version=_SIGNALFORGE_VERSION, + record_id=uuid.uuid4().hex, + timestamp=datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + config_hash=config_hash, + model_unique_id=model_unique_id, + test=decision.test, + test_anchor=decision.test_anchor, + decision=decision.decision, + reason=decision.reason, + failures=decision.failures, + sampled_rows=decision.sampled_rows, + scope=decision.scope, + elapsed_ms=decision.elapsed_ms, + compiled_sql_hash=decision.compiled_sql_hash, + compiled_sql=decision.compiled_sql, + why=decision.why, + sample_failures=decision.sample_failures, + ) + + +def _compute_config_hash(canonical_json: str) -> str: + """Return a 16-hex-char SHA-256 digest of ``canonical_json``. + + Matches the convention :class:`signalforge.safety.models.AuditEvent` + uses for ``policy_hash`` (DEC-005): a reviewer can verify all records + in a run came from the same prune config by checking this field across + the JSONL. + + The caller is responsible for canonicalising the config to a stable + JSON string (sorted keys, no whitespace) before hashing — two configs + that differ only in field order must hash identically. + """ + return sha256(canonical_json.encode("utf-8")).hexdigest()[:16] + + +def _write_prune_event(event: PruneEvent, path: Path) -> None: + """Append one JSONL record to ``path``. Fail-closed. + + Mirrors :func:`signalforge.safety.audit.write` and + :func:`signalforge.draft.audit.write_response_event` semantics exactly: + serialise → size-check (BEFORE any file open) → ``os.open(O_APPEND | + O_CREAT | O_WRONLY, 0o600)`` → single ``os.write`` → ``os.fsync`` → + close. Catches NO exceptions internally; the ``try / finally`` around + ``os.close`` only guarantees the descriptor is released, it does NOT + swallow ``write`` / ``fsync`` failures. + + Args: + event: the :class:`PruneEvent` to persist. + path: target audit log path. Caller is responsible for ensuring the + parent directory exists (the engine in US-009 handles + ``mkdir`` separately so it can wrap the failure mode + appropriately). + + Raises: + PruneAuditRecordTooLargeError: the serialised line exceeds + :data:`_PRUNE_AUDIT_RECORD_LIMIT_BYTES`. Raised BEFORE any file + is opened — an oversize record leaves no on-disk artefact. + OSError: any underlying I/O failure (``PermissionError``, + ``FileNotFoundError``, etc.) propagates raw. The caller (the + engine in US-009) wraps these as + :class:`signalforge.prune.errors.PruneAuditWriteError`. + """ + payload = event.model_dump(mode="json") + line = json.dumps(payload, separators=(",", ":")) + "\n" + encoded = line.encode("utf-8") + + # Size check BEFORE any file open so an oversize record leaves no + # on-disk artefact. Mirrors safety.audit and draft.audit. + if len(encoded) > _PRUNE_AUDIT_RECORD_LIMIT_BYTES: + raise PruneAuditRecordTooLargeError( + size=len(encoded), + limit=_PRUNE_AUDIT_RECORD_LIMIT_BYTES, + ) + + # ``O_APPEND`` gives atomic concurrent appends; ``O_CREAT`` handles the + # first call; ``0o600`` keeps the file owner-only. No try/except — any + # ``OSError`` from ``os.write`` / ``os.fsync`` propagates so the caller + # drops the partial decision. The ``try / finally`` only guarantees the + # descriptor is released; it does NOT silence the syscall failures. + fd = os.open(str(path), os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) + try: + os.write(fd, encoded) + os.fsync(fd) + finally: + # Best-effort close; the write/fsync above already succeeded + # (or raised), so a close failure here would only mask the real + # outcome. Mirrors safety.audit and draft.audit. + with contextlib.suppress(OSError): + os.close(fd) + + +# Sorted alphabetically (mirrors the convention enforced by sibling modules). +__all__ = ( + "PruneEvent", + "_PRUNE_AUDIT_RECORD_LIMIT_BYTES", + "_PRUNE_AUDIT_SCHEMA_VERSION", + "_build_prune_event", + "_compute_config_hash", + "_write_prune_event", +) diff --git a/tests/prune/test_audit.py b/tests/prune/test_audit.py new file mode 100644 index 00000000..650fa725 --- /dev/null +++ b/tests/prune/test_audit.py @@ -0,0 +1,255 @@ +"""Tests for ``signalforge.prune.audit`` (US-008). + +Mirrors :mod:`tests.safety.test_audit` and :mod:`tests.draft.test_audit` +exactly — same fail-closed contract (DEC-011 of safety-layer.md, DEC-006/008 +of llm-drafter.md), same POSIX-atomic-append size cap, same +``O_APPEND | O_CREAT | 0o600`` open + ``os.fsync`` + close shape. + +The eight tests below assert each load-bearing property of the writer: + +* one JSONL line, all documented fields present, ``audit_schema_version == 1`` +* file mode bits are exactly ``0o600`` (POSIX-only) +* ``os.fsync`` is called exactly once per write +* oversize record raises BEFORE any file open (no on-disk artefact) +* OSError on ``os.write`` propagates raw — caller is responsible for wrapping +* ANSI escape sequences in user-controlled fields round-trip safely through + JSON (escaped as ``\\u001b``) +* concurrent appends from 10 threads × 50 writes produce exactly 500 + well-formed JSONL lines (POSIX ``O_APPEND`` atomicity for writes ≤ 4 KiB) +* :func:`_compute_config_hash` is deterministic and 16 hex characters + +These exercise real I/O on ``tmp_path`` because — per the testing-strategy +review used elsewhere — mocks of ``open`` hide buffering bugs that the real +syscall surface exposes. +""" + +from __future__ import annotations + +import json +import os +import stat +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest + +from signalforge.draft.models import CandidateTestNotNull +from signalforge.prune.audit import ( + _PRUNE_AUDIT_RECORD_LIMIT_BYTES, + PruneEvent, + _build_prune_event, + _compute_config_hash, + _write_prune_event, +) +from signalforge.prune.errors import PruneAuditRecordTooLargeError +from signalforge.prune.models import PruneDecision + + +def _make_decision(**overrides: Any) -> PruneDecision: + base: dict[str, Any] = dict( + test_anchor="column.id", + test=CandidateTestNotNull(column="id"), + decision="kept", + reason="kept", + failures=0, + sampled_rows=1000, + scope="sample", + elapsed_ms=42, + compiled_sql_hash="0123456789abcdef", + compiled_sql="SELECT COUNT(*) FROM `p.d.t` WHERE id IS NULL", + why="0 failures across 1000 sampled rows", + sample_failures=None, + ) + base.update(overrides) + return PruneDecision(**base) + + +def _make_event(**decision_overrides: Any) -> PruneEvent: + decision = _make_decision(**decision_overrides) + return _build_prune_event( + decision=decision, + model_unique_id="model.test.x", + config_hash="abc123def456789a", + ) + + +def test_write_prune_event_emits_one_jsonl_line(tmp_path: Path) -> None: + """Writer produces exactly one JSONL line; every documented field is + present; ``audit_schema_version == 1``. + """ + audit_path = tmp_path / "prune.jsonl" + event = _make_event() + _write_prune_event(event, audit_path) + + contents = audit_path.read_text(encoding="utf-8") + assert contents.endswith("\n") + lines = contents.splitlines() + assert len(lines) == 1 + + payload = json.loads(lines[0]) + assert payload["audit_schema_version"] == 1 + # Every documented field present. + expected_fields = { + "audit_schema_version", + "signalforge_version", + "record_id", + "timestamp", + "config_hash", + "model_unique_id", + "test", + "test_anchor", + "decision", + "reason", + "failures", + "sampled_rows", + "scope", + "elapsed_ms", + "compiled_sql_hash", + "compiled_sql", + "why", + "sample_failures", + } + assert expected_fields.issubset(payload.keys()) + # The discriminated-union payload survives the round-trip. + assert payload["test"]["type"] == "not_null" + assert payload["test"]["column"] == "id" + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX-only permission semantics") +def test_write_prune_event_uses_appendcreate_0o600_mode(tmp_path: Path) -> None: + """Newly created file has ``0o600`` permission bits (owner-only). + + Mirrors safety/draft: ``O_APPEND | O_CREAT | O_WRONLY`` with mode 0o600 + keeps the audit log unreadable to other users on shared CI runners. + """ + audit_path = tmp_path / "prune.jsonl" + _write_prune_event(_make_event(), audit_path) + mode = stat.S_IMODE(os.stat(audit_path).st_mode) + # Lenient against umask: assert no group/other bits AND owner has rw. + assert mode & 0o077 == 0 + assert mode & 0o600 == 0o600 + + +def test_write_prune_event_calls_fsync(tmp_path: Path) -> None: + """``os.fsync`` is called exactly once per write — durability is + load-bearing for fail-closed semantics; without fsync a power loss + can drop the audit line that proves the prune decision happened. + """ + audit_path = tmp_path / "prune.jsonl" + real_fsync = os.fsync + calls: list[int] = [] + + def record_fsync(fd: int) -> None: + calls.append(fd) + real_fsync(fd) + + with patch("signalforge.prune.audit.os.fsync", side_effect=record_fsync): + _write_prune_event(_make_event(), audit_path) + + assert len(calls) == 1 + assert calls[0] >= 0 + + +def test_write_prune_event_oversize_raises_before_open(tmp_path: Path) -> None: + """An oversize record raises :class:`PruneAuditRecordTooLargeError` + BEFORE any file is opened — no on-disk artefact is left behind. + + Constructed by stuffing ``compiled_sql`` past the 4000-byte cap. + """ + audit_path = tmp_path / "prune.jsonl" + huge_sql = "X" * 4500 + event = _make_event(compiled_sql=huge_sql) + + with pytest.raises(PruneAuditRecordTooLargeError) as excinfo: + _write_prune_event(event, audit_path) + + assert excinfo.value.limit == _PRUNE_AUDIT_RECORD_LIMIT_BYTES + assert excinfo.value.size > _PRUNE_AUDIT_RECORD_LIMIT_BYTES + # No on-disk artefact: the size check fires before ``os.open``. + assert not audit_path.exists() + + +def test_write_prune_event_propagates_oserror_raw(tmp_path: Path) -> None: + """``OSError`` on ``os.write`` propagates raw — the caller (engine.py) + is responsible for wrapping into :class:`PruneAuditWriteError`. The + writer itself catches NO exceptions; the propagation IS the defence. + """ + audit_path = tmp_path / "prune.jsonl" + + def boom(fd: int, data: bytes) -> int: + raise OSError("simulated write failure") + + with ( + patch("signalforge.prune.audit.os.write", side_effect=boom), + pytest.raises(OSError) as excinfo, + ): + _write_prune_event(_make_event(), audit_path) + + # Raw OSError, not a wrapped PruneAuditWriteError. The caller wraps, + # not the writer. + assert "simulated write failure" in str(excinfo.value) + + +def test_write_prune_event_ansi_safe_compiled_sql(tmp_path: Path) -> None: + """ANSI escape bytes in ``compiled_sql`` round-trip safely through JSON. + + The audit log is consumed by terminal viewers; raw ``\\x1b`` bytes + would inject terminal escape sequences. ``json.dumps`` escapes them + as ``\\u001b`` — round-trip parse recovers the original bytes + losslessly while the on-disk bytes never carry a raw escape. + """ + audit_path = tmp_path / "prune.jsonl" + nasty_sql = "SELECT 1 -- \x1b[31mFAKE\x1b[0m comment" + _write_prune_event(_make_event(compiled_sql=nasty_sql), audit_path) + + raw_bytes = audit_path.read_bytes() + # JSON-escaped form is present; raw ESC byte is NOT. + assert b"\\u001b" in raw_bytes + assert b"\x1b" not in raw_bytes + + # Round-trip recovers the original cleartext. + payload = json.loads(audit_path.read_text(encoding="utf-8").splitlines()[0]) + assert payload["compiled_sql"] == nasty_sql + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX-only O_APPEND atomicity") +def test_write_prune_event_concurrent_appends_atomic(tmp_path: Path) -> None: + """10 threads × 50 writes each → exactly 500 well-formed JSONL lines. + + POSIX guarantees ``write(2)`` is atomic for payloads ≤ ``PIPE_BUF`` + (4096 bytes on Linux). The 4000-byte cap leaves a 96-byte margin so + concurrent ``O_APPEND`` writers cannot interleave partial records. + Each line must parse independently; total line count must equal + ``threads × writes_per_thread``. + """ + audit_path = tmp_path / "prune.jsonl" + + def writer(thread_idx: int) -> None: + for i in range(50): + event = _make_event(why=f"thread {thread_idx} row {i}") + _write_prune_event(event, audit_path) + + with ThreadPoolExecutor(max_workers=10) as ex: + list(ex.map(writer, range(10))) + + lines = audit_path.read_text(encoding="utf-8").splitlines() + assert len(lines) == 500 + # Every line parses as JSON — no interleaving. + for line in lines: + json.loads(line) + + +def test_compute_config_hash_is_deterministic_and_16_hex() -> None: + """:func:`_compute_config_hash` returns 16 hex characters and is + deterministic across calls — matches safety's ``policy_hash`` (DEC-005). + """ + h1 = _compute_config_hash("{}") + h2 = _compute_config_hash("{}") + assert h1 == h2 + assert len(h1) == 16 + assert all(c in "0123456789abcdef" for c in h1) + # A different input produces a different digest (sanity floor). + assert _compute_config_hash('{"a":1}') != h1 diff --git a/tests/test_audit_completeness.py b/tests/test_audit_completeness.py index 26287453..61fbe41d 100644 --- a/tests/test_audit_completeness.py +++ b/tests/test_audit_completeness.py @@ -31,6 +31,7 @@ _SAFETY_DIR = _REPO_ROOT / "src" / "signalforge" / "safety" _LLM_DIR = _REPO_ROOT / "src" / "signalforge" / "llm" _DRAFT_DIR = _REPO_ROOT / "src" / "signalforge" / "draft" +_PRUNE_DIR = _REPO_ROOT / "src" / "signalforge" / "prune" # --------------------------------------------------------------------------- @@ -306,6 +307,60 @@ def test_llm_response_event_construction_in_draft_audit_module_is_present() -> N ) +# --------------------------------------------------------------------------- +# Scan 5 — PruneEvent only in prune.audit +# --------------------------------------------------------------------------- + + +# DEC-018: PruneEvent is the prune-decision audit record; constructing it +# anywhere other than the audit-write seam is a bug — the corresponding +# event will never reach disk and the prune decision becomes unauditable. +# Mirrors Scan 4 (LLMResponseEvent only in draft.audit). +_PRUNE_EVENT_EXCLUSIONS: set[str] = { + # audit.py is the sole audit-write seam: ``_build_prune_event`` + # constructs the PruneEvent and ``_write_prune_event`` is the + # fail-closed JSONL writer (mirrors safety/audit.py from #4 and + # draft/audit.py from #5). + "audit.py", +} + + +def test_prune_event_construction_only_in_prune_audit_module() -> None: + """DEC-018: direct ``PruneEvent(...)`` outside + ``signalforge.prune.audit`` bypasses the fail-closed JSONL writer — + the event would never reach disk and the prune decision would be + unauditable. + """ + hits = _scan_dir_for_name_calls( + _PRUNE_DIR, + target="PruneEvent", + excluded_relpaths=_PRUNE_EVENT_EXCLUSIONS, + ) + formatted = "\n".join(f" {p}:{line}" for p, line in hits) + assert not hits, ( + "PruneEvent constructed outside signalforge.prune.audit:\n" + f"{formatted}\n" + "Construct only via _build_prune_event — direct construction " + "bypasses the fail-closed JSONL audit writer." + ) + + +def test_prune_event_construction_in_prune_audit_module_is_present() -> None: + """Sanity: at least one ``PruneEvent(...)`` in + ``signalforge.prune.audit``. If this fails the scan above is no longer + load-bearing. + """ + audit_path = _PRUNE_DIR / "audit.py" + tree = ast.parse(audit_path.read_text(encoding="utf-8")) + finder = _NameCallFinder("PruneEvent") + finder.visit(tree) + assert finder.calls, ( + "Expected PruneEvent(...) call in signalforge.prune.audit — " + "the AST-scan above is no longer load-bearing if the legitimate " + "constructor disappears." + ) + + # --------------------------------------------------------------------------- # Negative test: confirm the AST visitors detect planted violations # --------------------------------------------------------------------------- @@ -330,3 +385,8 @@ def test_scan_visitors_catch_planted_violations() -> None: response_finder = _NameCallFinder("LLMResponseEvent") response_finder.visit(ast.parse(response_src)) assert len(response_finder.calls) == 1 + + prune_src = "def make():\n return PruneEvent(model_unique_id='x')\n" + prune_finder = _NameCallFinder("PruneEvent") + prune_finder.visit(ast.parse(prune_src)) + assert len(prune_finder.calls) == 1 From 2f274637119548a83ffeaaed38c94f2b7701fc3d Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:27:08 -0700 Subject: [PATCH 10/24] bd_1-scaffolding-y8y.5: PruneConfig + load_prune_config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User-facing config for the prune layer. Inner PruneConfig is extra="forbid" so typos fail loud; outer _PruneConfigFile wrapper is extra="ignore" so sibling stages coexist (DEC-015, DEC-020). Defaults match Phase-1 housekeeping: scope=sample, sample_size=100k, test_timeout=30s, total_budget=600s, capture_failure_rows=3, trusted_models=(), partition_filter=None. yaml.safe_load only. Trusted-models validation against the manifest is NOT done at load time — that's prune_tests() entry (DEC-008, US-009). Plan: plans/super/6-prune-engine.md (DEC-009, DEC-015, DEC-020). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/prune/config.py | 219 +++++++++++++++++- tests/fixtures/prune/signalforge_full.yml | 11 + tests/fixtures/prune/signalforge_minimal.yml | 1 + .../fixtures/prune/signalforge_partition.yml | 5 + tests/fixtures/prune/signalforge_typo.yml | 2 + tests/fixtures/prune/signalforge_unsafe.yml | 2 + .../prune/signalforge_with_siblings.yml | 7 + tests/prune/test_config.py | 127 ++++++++++ 8 files changed, 365 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/prune/signalforge_full.yml create mode 100644 tests/fixtures/prune/signalforge_minimal.yml create mode 100644 tests/fixtures/prune/signalforge_partition.yml create mode 100644 tests/fixtures/prune/signalforge_typo.yml create mode 100644 tests/fixtures/prune/signalforge_unsafe.yml create mode 100644 tests/fixtures/prune/signalforge_with_siblings.yml create mode 100644 tests/prune/test_config.py diff --git a/src/signalforge/prune/config.py b/src/signalforge/prune/config.py index 2481f489..95b7924a 100644 --- a/src/signalforge/prune/config.py +++ b/src/signalforge/prune/config.py @@ -1,14 +1,215 @@ """Prune-layer config loader. -Loads the `prune:` top-level block from `signalforge.yml` into a typed -`PruneConfig` (scope, sample_size, test_timeout_seconds, -total_budget_seconds, capture_failure_rows). Outer file wrapper uses -`extra="ignore"` so sibling stage namespaces (`safety:`, `llm:`, future -`grade:`) don't break the loader; the inner `PruneConfig` block uses -`extra="forbid"` so a typo like `scop:` fails loud rather than silently -no-op'ing. - -See plans/super/6-prune-engine.md for the full design. +Loads the ``prune:`` top-level block from ``signalforge.yml`` into a typed +:class:`PruneConfig` (scope, sample_size, test_timeout_seconds, +total_budget_seconds, capture_failure_rows, trusted_models, +partition_filter). Outer file wrapper uses ``extra="ignore"`` so sibling +stage namespaces (``safety:``, ``llm:``, future ``grade:``) don't break +the loader; the inner :class:`PruneConfig` block uses ``extra="forbid"`` +so a typo like ``scop:`` fails loud rather than silently no-op'ing. + +Mirrors :mod:`signalforge.draft.config` and :mod:`signalforge.safety.config`. + +Design commitments operationalised here: + +* **DEC-009** — Defaults trace to plan Phase-1 housekeeping: + ``scope="sample"``, ``sample_size=100_000``, ``test_timeout_seconds=30``, + ``total_budget_seconds=600``, ``capture_failure_rows=3``, + ``trusted_models=()``, ``partition_filter=None``. +* **DEC-015** — Config-shaped models use ``extra="forbid"`` so typos in + ``signalforge.yml`` fail loud (mirrors ``safety-layer.md`` DEC-015 and + ``llm-drafter.md`` DEC-027). Read-back / response-shaped models use + ``extra="ignore"``. +* **DEC-020** — ``signalforge.yml`` top-level namespace key for this + layer is ``prune:``. Other top-level keys (``safety:``, ``llm:``, + future ``grade:``) are reserved for other stages and silently ignored + by this loader. + +Resolution: + +* ``path=None`` or path does not exist → return :class:`PruneConfig` + defaults silently. +* File present but ``prune:`` key absent or null → return defaults. +* ``prune:`` block well-formed → return the populated + :class:`PruneConfig`. +* Unknown / typo'd inner field, non-mapping ``prune:`` block, YAML parse + failure, or :class:`pydantic.ValidationError` from + :class:`PruneConfig` → :class:`signalforge.prune.errors.PruneConfigError` + with the underlying exception preserved on ``__cause__``. + +Trusted-models validation against the manifest is NOT performed here +(DEC-008): the manifest isn't loaded yet at config-load time. That check +runs at :func:`signalforge.prune.prune_tests` entry — see US-009. + +``yaml.safe_load`` only — ``yaml.load`` accepts arbitrary Python object +construction tags and is unsafe for any input we don't fully control. """ from __future__ import annotations + +from pathlib import Path +from typing import Literal + +import yaml +from pydantic import BaseModel, ConfigDict, ValidationError, field_validator + +from signalforge.prune.errors import PruneConfigError +from signalforge.warehouse.models import PartitionFilter + +_DEFAULT_CONFIG_FILENAME = "signalforge.yml" + + +class PruneConfig(BaseModel): + """User-facing knobs for the prune layer (DEC-009). + + Lives under the ``prune:`` top-level key in ``signalforge.yml`` + (DEC-020). Config-shaped per ``safety-layer.md`` DEC-015: + ``extra="forbid"`` so typos like ``scop:`` instead of ``scope:`` fail + loud rather than silently no-op'ing. The :class:`_PruneConfigFile` + outer wrapper uses ``extra="ignore"`` so other top-level keys + (``safety:``, ``llm:``, ...) reserved by DEC-020 don't trip the + strict validator. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) + + scope: Literal["sample", "full"] = "sample" + """Whether candidate tests run against a deterministic warehouse + sample (``"sample"``, the default) or the full table (``"full"``).""" + + sample_size: int = 100_000 + """Target row count for ``"sample"`` scope. Passed to + :meth:`signalforge.warehouse.WarehouseAdapter.sample_rows`.""" + + test_timeout_seconds: int = 30 + """Per-test wall-clock budget. Adapter cancels the in-flight query + when this elapses; the orchestrator routes the test to + ``kept-without-evidence``.""" + + total_budget_seconds: int = 600 + """Whole-run wall-clock budget. When exceeded the orchestrator drains + every remaining test to ``kept-without-evidence`` (DEC-011).""" + + capture_failure_rows: int = 3 + """Number of failing rows to capture per failed test (matches the + :class:`signalforge.warehouse.TestResult` cap).""" + + trusted_models: tuple[str, ...] = () + """Manifest ``unique_id``s whose data is treated as known-clean + (Q1=B, opt-in only). A failure on a trusted model surfaces as a real + failure rather than the "drop on clean-data failure" branch. + Validated against the manifest at :func:`prune_tests` entry per + DEC-008 — NOT here.""" + + partition_filter: PartitionFilter | None = None + """Optional :class:`PartitionFilter` scoping every sample query + (DEC-009). Required by the warehouse adapter for tables ≥ 100M rows; + otherwise optional. Pydantic recursively validates the YAML mapping + into the typed ADT.""" + + @field_validator("sample_size", "test_timeout_seconds", "total_budget_seconds") + @classmethod + def _positive(cls, v: int) -> int: + if v <= 0: + raise ValueError("must be positive") + return v + + @field_validator("capture_failure_rows") + @classmethod + def _non_negative(cls, v: int) -> int: + if v < 0: + raise ValueError("must be non-negative") + return v + + +class _PruneConfigFile(BaseModel): + """Outer wrapper for the ``signalforge.yml`` top-level mapping. + + ``extra="ignore"`` at this level — other top-level keys (``safety:``, + ``llm:``, ``grade:``, ...) are reserved for other stages per DEC-020 + and must not trigger a prune-layer validation error. The strict + ``extra="forbid"`` lives on :class:`PruneConfig` itself. + """ + + model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True) + + prune: PruneConfig | None = None + + +def load_prune_config(path: Path | str | None = None) -> PruneConfig: + """Load a :class:`PruneConfig` from ``signalforge.yml``. + + See module docstring for the full resolution order and error + contract. + + Args: + path: Optional explicit config-file path. ``None`` or a path that + does not exist returns :class:`PruneConfig` defaults + silently. + + Returns: + A fully-validated :class:`PruneConfig`. When the file is absent, + empty, or the ``prune:`` key is missing, the defaults from + DEC-009 apply. + + Raises: + PruneConfigError: The file is not valid YAML, its top level is + not a mapping, the ``prune:`` block is not a mapping, or the + contents fail :class:`PruneConfig` validation (typo, unknown + ``scope`` value, non-positive ``test_timeout_seconds`` / + ``total_budget_seconds`` / ``sample_size``, ...). The + original :class:`pydantic.ValidationError` (if any) is + preserved on ``__cause__``. + """ + if path is None: + return PruneConfig() + + config_file = Path(path) + if not config_file.exists(): + return PruneConfig() + + raw_text = config_file.read_text(encoding="utf-8").strip() + if not raw_text: + return PruneConfig() + + try: + loaded = yaml.safe_load(raw_text) + except yaml.YAMLError as exc: + raise PruneConfigError( + f"signalforge.yml is not valid YAML: {exc}", + ) from exc + + if loaded is None: + # File parses to None (e.g. only comments) — same as empty. + return PruneConfig() + + if not isinstance(loaded, dict): + raise PruneConfigError( + f"signalforge.yml top level must be a mapping; got {type(loaded).__name__}", + ) + + if "prune" not in loaded or loaded["prune"] is None: + # Missing `prune:` key (or `prune:` with null value) — other + # top-level keys reserved per DEC-020 namespace. + return PruneConfig() + + prune_block = loaded["prune"] + if not isinstance(prune_block, dict): + raise PruneConfigError( + f"signalforge.yml: 'prune' must be a mapping; got {type(prune_block).__name__}", + ) + + try: + wrapper = _PruneConfigFile.model_validate({"prune": prune_block}) + except ValidationError as exc: + raise PruneConfigError( + f"signalforge.yml: 'prune' block failed schema validation: {exc}", + ) from exc + + # `wrapper.prune` is non-None here because we already filtered the + # missing-key branch above; assert for the type checker. + assert wrapper.prune is not None + return wrapper.prune + + +__all__ = ["PruneConfig", "load_prune_config"] diff --git a/tests/fixtures/prune/signalforge_full.yml b/tests/fixtures/prune/signalforge_full.yml new file mode 100644 index 00000000..bc9204b7 --- /dev/null +++ b/tests/fixtures/prune/signalforge_full.yml @@ -0,0 +1,11 @@ +safety: + mode: schema-only +prune: + scope: full + sample_size: 50000 + test_timeout_seconds: 60 + total_budget_seconds: 1200 + capture_failure_rows: 5 + trusted_models: + - model.shop.dim_customers + - model.shop.fact_orders diff --git a/tests/fixtures/prune/signalforge_minimal.yml b/tests/fixtures/prune/signalforge_minimal.yml new file mode 100644 index 00000000..cd421e74 --- /dev/null +++ b/tests/fixtures/prune/signalforge_minimal.yml @@ -0,0 +1 @@ +prune: {} diff --git a/tests/fixtures/prune/signalforge_partition.yml b/tests/fixtures/prune/signalforge_partition.yml new file mode 100644 index 00000000..faa61771 --- /dev/null +++ b/tests/fixtures/prune/signalforge_partition.yml @@ -0,0 +1,5 @@ +prune: + partition_filter: + column: event_dt + op: ">=" + value: "2026-01-01" diff --git a/tests/fixtures/prune/signalforge_typo.yml b/tests/fixtures/prune/signalforge_typo.yml new file mode 100644 index 00000000..ddb2ce2c --- /dev/null +++ b/tests/fixtures/prune/signalforge_typo.yml @@ -0,0 +1,2 @@ +prune: + scop: full diff --git a/tests/fixtures/prune/signalforge_unsafe.yml b/tests/fixtures/prune/signalforge_unsafe.yml new file mode 100644 index 00000000..0425f3b8 --- /dev/null +++ b/tests/fixtures/prune/signalforge_unsafe.yml @@ -0,0 +1,2 @@ +prune: + scope: !!python/object/apply:os.system ["echo pwned"] diff --git a/tests/fixtures/prune/signalforge_with_siblings.yml b/tests/fixtures/prune/signalforge_with_siblings.yml new file mode 100644 index 00000000..0f9aac2f --- /dev/null +++ b/tests/fixtures/prune/signalforge_with_siblings.yml @@ -0,0 +1,7 @@ +safety: + mode: schema-only +llm: + model: claude-sonnet-4-6 +prune: + scope: sample + sample_size: 25000 diff --git a/tests/prune/test_config.py b/tests/prune/test_config.py new file mode 100644 index 00000000..4f9471e0 --- /dev/null +++ b/tests/prune/test_config.py @@ -0,0 +1,127 @@ +"""Tests for :mod:`signalforge.prune.config` (US-005). + +Covers the resolution + validation contract of :func:`load_prune_config`: + +* ``path=None`` or missing path → :class:`PruneConfig` defaults silently + (DEC-009). +* Empty / minimal ``prune:`` blocks → defaults. +* Full round-trip with non-default values for every field. +* Typo'd inner field (``scop:``) → :class:`PruneConfigError` (DEC-015, + ``extra="forbid"`` on the inner config). +* Sibling top-level keys (``safety:``, ``llm:``) → tolerated by the + outer wrapper (``extra="ignore"`` per DEC-020 namespace reservation). +* ``partition_filter`` mapping → typed + :class:`signalforge.warehouse.PartitionFilter`. +* ``yaml.safe_load`` rejects classic Python-object-construction gadgets. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from signalforge.prune.config import PruneConfig, load_prune_config +from signalforge.prune.errors import PruneConfigError +from signalforge.warehouse.models import PartitionFilter + +_FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" / "prune" + + +def test_load_prune_config_path_none_returns_defaults() -> None: + """``load_prune_config(None)`` returns :class:`PruneConfig` defaults + silently — no log, no error, every field at its DEC-009 default.""" + result = load_prune_config(None) + + assert result == PruneConfig() + assert result.scope == "sample" + assert result.sample_size == 100_000 + assert result.test_timeout_seconds == 30 + assert result.total_budget_seconds == 600 + assert result.capture_failure_rows == 3 + assert result.trusted_models == () + assert result.partition_filter is None + + +def test_load_prune_config_missing_file_returns_defaults(tmp_path: Path) -> None: + """A path that does not exist returns defaults without raising — + parity with :func:`load_draft_config` and + :func:`load_safety_config` for the implicit-config-file case.""" + missing = tmp_path / "missing.yml" + assert not missing.exists() + + result = load_prune_config(missing) + + assert result == PruneConfig() + + +def test_load_prune_config_full_round_trip() -> None: + """A populated ``prune:`` block round-trips into a + :class:`PruneConfig` whose every field matches the YAML.""" + result = load_prune_config(_FIXTURES / "signalforge_full.yml") + + assert result.scope == "full" + assert result.sample_size == 50_000 + assert result.test_timeout_seconds == 60 + assert result.total_budget_seconds == 1200 + assert result.capture_failure_rows == 5 + assert result.trusted_models == ( + "model.shop.dim_customers", + "model.shop.fact_orders", + ) + assert result.partition_filter is None + + +def test_load_prune_config_minimal_block_returns_defaults() -> None: + """An empty ``prune: {}`` mapping validates as defaults — every + field is optional with a default per DEC-009.""" + result = load_prune_config(_FIXTURES / "signalforge_minimal.yml") + + assert result == PruneConfig() + + +def test_load_prune_config_typo_raises() -> None: + """A typo'd inner field (``scop:`` instead of ``scope:``) raises + :class:`PruneConfigError`. ``PruneConfig`` uses ``extra="forbid"`` + per DEC-015 so silent no-op is impossible.""" + with pytest.raises(PruneConfigError) as excinfo: + load_prune_config(_FIXTURES / "signalforge_typo.yml") + + # The wrapped pydantic ValidationError surfaces the offending key + # name in the error message — the orchestrator / CLI uses this + # message to point the operator at the typo. + assert "scop" in str(excinfo.value) + + +def test_load_prune_config_tolerates_sibling_blocks() -> None: + """Sibling top-level keys (``safety:``, ``llm:``) are reserved for + other stages per DEC-020 and silently ignored by the prune + loader.""" + result = load_prune_config(_FIXTURES / "signalforge_with_siblings.yml") + + assert result.scope == "sample" + assert result.sample_size == 25_000 + # Other fields fall back to defaults: + assert result.test_timeout_seconds == 30 + assert result.total_budget_seconds == 600 + + +def test_load_prune_config_parses_partition_filter() -> None: + """``prune.partition_filter`` (a YAML mapping) is recursively + validated into a typed :class:`PartitionFilter` ADT — Pydantic + handles the dict→dataclass conversion.""" + result = load_prune_config(_FIXTURES / "signalforge_partition.yml") + + assert isinstance(result.partition_filter, PartitionFilter) + assert result.partition_filter.column == "event_dt" + assert result.partition_filter.op == ">=" + assert result.partition_filter.value == "2026-01-01" + + +def test_load_prune_config_yaml_safe_load_rejects_python_objects() -> None: + """A fixture containing a ``!!python/object/apply:os.system [...]`` + gadget raises rather than executing it — confirms the loader uses + :func:`yaml.safe_load`, not :func:`yaml.load` (which honours + arbitrary Python object construction tags).""" + with pytest.raises(PruneConfigError): + load_prune_config(_FIXTURES / "signalforge_unsafe.yml") From 0eaf437725dfac43ecf783cd00b751819b275d07 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:27:47 -0700 Subject: [PATCH 11/24] bd_1-scaffolding-y8y.5: drop dead _DEFAULT_CONFIG_FILENAME constant Worker left a stale module-level constant from comparing with draft/config.py (which uses project_dir + filename resolution). Our load_prune_config takes the path directly, so the constant was never read. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/prune/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/signalforge/prune/config.py b/src/signalforge/prune/config.py index 95b7924a..38f5890a 100644 --- a/src/signalforge/prune/config.py +++ b/src/signalforge/prune/config.py @@ -56,8 +56,6 @@ from signalforge.prune.errors import PruneConfigError from signalforge.warehouse.models import PartitionFilter -_DEFAULT_CONFIG_FILENAME = "signalforge.yml" - class PruneConfig(BaseModel): """User-facing knobs for the prune layer (DEC-009). From 01a4990193f5ce5462d29c128b27cfa97728cc65 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:32:06 -0700 Subject: [PATCH 12/24] bd_1-scaffolding-y8y.2: warehouse adapter timeout_ms plumbing Extend make_query_job_config and _default_job_config with a timeout_ms: int | None = None kwarg threaded into QueryJobConfig.job_timeout_ms. Default None (existing behaviour unchanged). The prune layer (#6 US-009) will use this for per-test budget enforcement (Q5=A, DEC-013). BigQuery cancels server-side at expiry; bytes-billed through cancellation are NOT refunded. _default_job_config is now keyword-only on (stage, timeout_ms); internal call sites in bigquery.py and the existing positional calls in tests/warehouse/test_bigquery_unit.py updated to match. Pyright noise for the loosely-typed job_timeout_ms attribute stays confined to _client.py per the SDK-seam convention. Plan: plans/super/6-prune-engine.md (DEC-013, AR-B2). Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/warehouse-adapter-ops.md | 8 ++ src/signalforge/warehouse/adapters/_client.py | 31 ++++++- .../warehouse/adapters/bigquery.py | 17 +++- tests/warehouse/test_bigquery_unit.py | 8 +- .../test_default_job_config_timeout.py | 85 +++++++++++++++++++ 5 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 tests/warehouse/test_default_job_config_timeout.py diff --git a/docs/warehouse-adapter-ops.md b/docs/warehouse-adapter-ops.md index 4fe23c81..8a4becf1 100644 --- a/docs/warehouse-adapter-ops.md +++ b/docs/warehouse-adapter-ops.md @@ -112,6 +112,14 @@ The BigQuery adapter is opinionated about cost on every query. produce the same prune decision; cached results break that contract. v0.2 may re-enable caching behind an explicit opt-in; in v0.1 it is unconditionally off. +- **Per-call `timeout_ms`** (DEC-013 of issue #6): pass an integer to + `_default_job_config(timeout_ms=...)` to set + `QueryJobConfig.job_timeout_ms`; BigQuery cancels the job + server-side at expiry. Bytes-scanned through the cancellation point + still bill — set conservatively. Default `None` (no timeout). The + prune layer (#6) uses this for per-test budget enforcement; the + adapter's own `sample_rows` / `column_stats` / `run_test_sql` paths + leave it unset. - **BigQuery job labels are auto-set** on every query: - `signalforge_stage` — the pipeline stage that issued the query. Values are `warehouse_sample` (from `sample_rows`), `warehouse_stats` diff --git a/src/signalforge/warehouse/adapters/_client.py b/src/signalforge/warehouse/adapters/_client.py index dbdc6550..583db73b 100644 --- a/src/signalforge/warehouse/adapters/_client.py +++ b/src/signalforge/warehouse/adapters/_client.py @@ -73,7 +73,13 @@ def make_real_client( raise WarehouseAuthError(message=str(exc)) from exc -def make_query_job_config(*, max_bytes_billed: int, stage: str, version: str | None = None) -> Any: +def make_query_job_config( + *, + max_bytes_billed: int, + stage: str, + version: str | None = None, + timeout_ms: int | None = None, +) -> Any: """Build a ``QueryJobConfig`` with DEC-015 defaults. ``use_query_cache=False`` is non-negotiable — Architectural Commitment @@ -86,6 +92,21 @@ def make_query_job_config(*, max_bytes_billed: int, stage: str, version: str | N every ``.`` with ``_``. When ``version`` is omitted, the helper resolves the running ``signalforge.__version__`` so the unit test can call this function without an integration-tier ``version=`` kwarg. + + ``timeout_ms`` (DEC-013 of issue #6, AR-B2): when not ``None``, set + ``QueryJobConfig.job_timeout_ms`` so BigQuery cancels the job + server-side at expiry. The default ``None`` leaves + ``job_timeout_ms`` unset (no server-side timeout). BigQuery cancels + at the timeout, but bytes scanned through the cancellation point + still bill — set conservatively. Used by the prune layer (#6) for + per-test budget enforcement; the warehouse adapter itself never + sets a timeout for its own ``sample_rows`` / ``column_stats`` / + ``run_test_sql`` calls. + + The pyright-noise comment for the loosely-typed ``job_timeout_ms`` + SDK attribute stays confined to this file (DEC-019 / DEC-012 + convention — ``_client.py`` is the single SDK-noise-confinement + seam). The rest of the warehouse subpackage stays type-clean. """ from google.cloud import bigquery # type: ignore[import-not-found] @@ -94,7 +115,7 @@ def make_query_job_config(*, max_bytes_billed: int, stage: str, version: str | N version = _pkg_version - return bigquery.QueryJobConfig( # type: ignore[no-any-return] + job_config = bigquery.QueryJobConfig( # type: ignore[no-any-return] use_query_cache=False, maximum_bytes_billed=max_bytes_billed, labels={ @@ -102,6 +123,12 @@ def make_query_job_config(*, max_bytes_billed: int, stage: str, version: str | N "signalforge_version": version.replace(".", "_"), }, ) + if timeout_ms is not None: + # The BigQuery SDK's QueryJobConfig.job_timeout_ms is loosely typed + # (str | int property under the hood); the type-ignore stays + # confined to this seam. + job_config.job_timeout_ms = timeout_ms # pyright: ignore[reportAttributeAccessIssue] + return job_config def map_bq_exception(exc: Exception, *, context: dict[str, Any] | None = None) -> Exception: diff --git a/src/signalforge/warehouse/adapters/bigquery.py b/src/signalforge/warehouse/adapters/bigquery.py index 66346bd7..1875d82d 100644 --- a/src/signalforge/warehouse/adapters/bigquery.py +++ b/src/signalforge/warehouse/adapters/bigquery.py @@ -218,7 +218,7 @@ def _get_client(self) -> _BQClientProtocol: self._client = make_real_client(self._project, self._location) return self._client - def _default_job_config(self, stage: str) -> Any: + def _default_job_config(self, *, stage: str, timeout_ms: int | None = None) -> Any: """Build the DEC-015 job config for ``stage``. Always: @@ -227,6 +227,14 @@ def _default_job_config(self, stage: str) -> Any: * ``maximum_bytes_billed`` set to the adapter's configured cap. * ``labels={"signalforge_stage": stage, "signalforge_version": …}`` for v0.2 cost attribution. + + ``timeout_ms`` (DEC-013 of issue #6, AR-B2): when not ``None``, + threaded through to :func:`make_query_job_config` so the + underlying ``QueryJobConfig.job_timeout_ms`` is set. The prune + layer (#6 US-009) is the only caller that supplies a non-None + value; the warehouse adapter's own ``sample_rows`` / + ``column_stats`` / ``run_test_sql`` paths leave it ``None`` so + existing behaviour is unchanged. """ from signalforge import __version__ @@ -234,6 +242,7 @@ def _default_job_config(self, stage: str) -> Any: max_bytes_billed=self._max_bytes_billed, stage=stage, version=__version__, + timeout_ms=timeout_ms, ) def _quote(self, ref: TableRef) -> str: @@ -385,7 +394,7 @@ def sample_rows( try: job = self._get_client().query( - sql, job_config=self._default_job_config("warehouse_sample") + sql, job_config=self._default_job_config(stage="warehouse_sample") ) rows = list(job.result()) except Exception as exc: @@ -490,7 +499,7 @@ def _flush_column_stats_batch(self, table: TableRef) -> None: try: job = self._get_client().query( - sql, job_config=self._default_job_config("warehouse_stats") + sql, job_config=self._default_job_config(stage="warehouse_stats") ) rows = list(job.result()) except Exception as exc: @@ -557,7 +566,7 @@ def run_test_sql(self, sql: str, *, capture_failures: int = 0) -> TestResult: try: job = self._get_client().query( - wrapped, job_config=self._default_job_config("warehouse_test") + wrapped, job_config=self._default_job_config(stage="warehouse_test") ) rows = list(job.result()) except Exception as exc: diff --git a/tests/warehouse/test_bigquery_unit.py b/tests/warehouse/test_bigquery_unit.py index a1f5ef72..df353179 100644 --- a/tests/warehouse/test_bigquery_unit.py +++ b/tests/warehouse/test_bigquery_unit.py @@ -76,14 +76,14 @@ def test_init_accepts_injected_client(fake_client: FakeBigQueryClient) -> None: def test_query_job_config_use_query_cache_false(adapter: BigQueryAdapter) -> None: """DEC-015: every QueryJobConfig must carry ``use_query_cache=False`` for reproducibility (Architectural Commitment #5: explainable diffs).""" - cfg = adapter._default_job_config("warehouse_sample") + cfg = adapter._default_job_config(stage="warehouse_sample") assert cfg.use_query_cache is False def test_query_job_config_labels_set(adapter: BigQueryAdapter) -> None: """DEC-015: labels must include ``signalforge_stage`` and ``signalforge_version`` for v0.2 cost attribution.""" - cfg = adapter._default_job_config("warehouse_sample") + cfg = adapter._default_job_config(stage="warehouse_sample") labels = dict(cfg.labels) assert "signalforge_stage" in labels assert labels["signalforge_stage"] == "warehouse_sample" @@ -95,13 +95,13 @@ def test_query_job_config_labels_set(adapter: BigQueryAdapter) -> None: def test_query_job_config_max_bytes_billed_set(adapter: BigQueryAdapter) -> None: """DEC-015: ``maximum_bytes_billed`` on the job config must reflect the adapter's configured cap so BigQuery rejects oversize queries.""" - cfg = adapter._default_job_config("warehouse_sample") + cfg = adapter._default_job_config(stage="warehouse_sample") assert cfg.maximum_bytes_billed == adapter._max_bytes_billed def test_query_job_config_stage_label_threads_through(adapter: BigQueryAdapter) -> None: """A different ``stage`` argument should yield a different label.""" - cfg = adapter._default_job_config("warehouse_test") + cfg = adapter._default_job_config(stage="warehouse_test") assert dict(cfg.labels)["signalforge_stage"] == "warehouse_test" diff --git a/tests/warehouse/test_default_job_config_timeout.py b/tests/warehouse/test_default_job_config_timeout.py new file mode 100644 index 00000000..c8e898ed --- /dev/null +++ b/tests/warehouse/test_default_job_config_timeout.py @@ -0,0 +1,85 @@ +"""``timeout_ms`` plumbing on ``make_query_job_config`` / ``_default_job_config``. + +Issue #6 US-002 (DEC-013, AR-B2) extends the single-seam ``QueryJobConfig`` +factory with a ``timeout_ms`` kwarg threaded into +``QueryJobConfig.job_timeout_ms``. The prune layer (#6 US-009) is the +target caller; the warehouse adapter's own ``sample_rows`` / +``column_stats`` / ``run_test_sql`` paths leave it ``None`` so existing +behaviour stays unchanged. + +The BigQuery SDK stringifies the value (the underlying property returns +``str | None``); these tests assert against the SDK-observable shape +rather than a hard-coded ``int`` so a future SDK upgrade that keeps the +value as ``int`` doesn't silently flip the contract. +""" + +from __future__ import annotations + +from signalforge.warehouse import BigQueryAdapter +from signalforge.warehouse.adapters._client import make_query_job_config +from tests.warehouse._fake import FakeBigQueryClient + + +def test_make_query_job_config_no_timeout_leaves_unset() -> None: + """Default ``timeout_ms=None`` keeps ``job_timeout_ms`` unset. + + Existing call sites (``sample_rows`` / ``column_stats`` / + ``run_test_sql``) supply no ``timeout_ms``; their behaviour must + not change. The BigQuery SDK exposes an unset timeout as ``None``. + """ + cfg = make_query_job_config(max_bytes_billed=100_000_000, stage="warehouse_test") + assert cfg.job_timeout_ms is None + + +def test_make_query_job_config_explicit_timeout_propagates() -> None: + """An explicit ``timeout_ms=`` propagates to ``job_timeout_ms``. + + BigQuery's ``QueryJobConfig.job_timeout_ms`` setter coerces the + supplied ``int`` to its on-the-wire ``str`` form; assert against + both shapes so a future SDK upgrade that keeps the value as ``int`` + doesn't silently flip the contract. + """ + cfg = make_query_job_config( + max_bytes_billed=100_000_000, + stage="warehouse_test", + timeout_ms=30_000, + ) + assert cfg.job_timeout_ms in (30_000, "30000") + + +def test_default_job_config_threads_timeout_through() -> None: + """``BigQueryAdapter._default_job_config(timeout_ms=…)`` reaches + ``job_timeout_ms`` through the single seam. + + The injected ``FakeBigQueryClient`` keeps the test offline; only + the constructed ``QueryJobConfig`` is inspected. + """ + fake = FakeBigQueryClient(project="fake_project") + adapter = BigQueryAdapter( + project="fake_project", + location="US", + max_bytes_billed=100_000_000, + client=fake, + ) + cfg = adapter._default_job_config(stage="warehouse_test", timeout_ms=15_000) + assert cfg.job_timeout_ms in (15_000, "15000") + + +def test_existing_callers_unchanged_no_timeout_set() -> None: + """Regression guard: existing internal call sites pass no + ``timeout_ms`` and must continue to produce a config with no + ``job_timeout_ms``. + + A failure here would mean the new kwarg leaked a default into the + existing ``sample_rows`` / ``column_stats`` / ``run_test_sql`` + paths. + """ + fake = FakeBigQueryClient(project="fake_project") + adapter = BigQueryAdapter( + project="fake_project", + location="US", + max_bytes_billed=100_000_000, + client=fake, + ) + cfg = adapter._default_job_config(stage="warehouse_sample") + assert cfg.job_timeout_ms is None From b34f019aad1a9886f8a69667526e04929cdcaa65 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:41:16 -0700 Subject: [PATCH 13/24] =?UTF-8?q?bd=5F1-scaffolding-y8y.9:=20prune=20engin?= =?UTF-8?q?e=20=E2=80=94=20prune=5Ftests=20orchestrator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End-to-end integration of compiler + adapter + audit + budget into prune_tests(model, adapter, candidates, manifest, *, config, audit_path) -> PruneResult. Routes outcomes through the five DropReason values per the plan's decision matrix: * 0 failures → drop (always-passes) * relationships(to: ?) → drop (requires-future-data) [no WH call] * fail + trusted → drop (failed-on-known-clean-data) * fail + untrusted → keep (kept) * WarehouseError → keep (kept-without-evidence) * total-budget exceed → keep (kept-without-evidence) [no WH call] trusted_models validation at entry (DEC-008) — typo'd unique_id raises PruneTrustedModelNotFoundError before any warehouse call. Audit fail-closed (DEC-016): OSError → PruneAuditWriteError(cause); PruneAuditRecordTooLargeError propagates raw. Either aborts the run. Module-level _sleep / _now_monotonic_ms aliases (DEC-019) for deterministic test override. Per-test timeout_ms threading from config.test_timeout_seconds into adapter.run_test_sql is deferred — adapter doesn't expose it yet (US-002 plumbed make_query_job_config; surfacing through run_test_sql is v0.2). For v0.1, per-test enforcement is implicit via WarehouseError catch; the total-budget gate handles wall-clock. Plan: plans/super/6-prune-engine.md (DEC-002, DEC-008, DEC-011, DEC-016, DEC-019). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/prune/engine.py | 686 +++++++++++++++++++++++++++++++- tests/prune/test_engine.py | 574 ++++++++++++++++++++++++++ 2 files changed, 1251 insertions(+), 9 deletions(-) create mode 100644 tests/prune/test_engine.py diff --git a/src/signalforge/prune/engine.py b/src/signalforge/prune/engine.py index bbc95837..9402f582 100644 --- a/src/signalforge/prune/engine.py +++ b/src/signalforge/prune/engine.py @@ -1,18 +1,686 @@ -"""Prune engine entry point and per-test executor. +"""Prune engine entry point and per-test executor (US-009). -Hosts `prune_tests(candidates, model, manifest, adapter, *, config=None)` — -the public seam that compiles every CandidateTest to failing-rows SQL via the -compiler module, runs each through `WarehouseAdapter.run_test_sql`, and folds -the per-test verdicts into a typed `PruneResult`. The executor enforces the -per-test runtime budget, the total-budget cap, and the conservative -"keep-on-error" default that routes warehouse exceptions to -`kept-without-evidence` rather than dropping silently. +Hosts :func:`prune_tests` — the public seam that compiles every +:class:`signalforge.draft.CandidateTest` to failing-rows SQL via the +compiler module (US-007), runs each through +:meth:`signalforge.warehouse.WarehouseAdapter.run_test_sql`, writes a +fail-closed JSONL audit receipt per decision (US-008), and folds the +per-test verdicts into a typed :class:`PruneResult` (US-004). -See plans/super/6-prune-engine.md for the full design. +This is THE load-bearing differentiator (CLAUDE.md commitment #1): a +candidate test that always passes on warehouse samples is worse than +nothing because it consumes reviewer attention. The orchestrator drops +those tests AND the symmetric "fails on known-clean data" set, while +keeping every test it could not evaluate (fail-closed against silent +loss of signal). + +Design commitments operationalised here: + +* **DEC-002** — The signature is + ``prune_tests(model, adapter, candidates, manifest, *, config, + audit_path) -> PruneResult``. Mirrors + :func:`signalforge.draft.draft_schema` so the CLI / wrapper layers + see one consistent end-to-end shape across stages. +* **DEC-008** — :attr:`PruneConfig.trusted_models` is validated at + orchestrator entry: every entry must resolve to a model in the + loaded :class:`Manifest`. The first miss raises + :class:`PruneTrustedModelNotFoundError` BEFORE any warehouse call is + issued. Validating at entry (not at config load) is necessary + because the manifest isn't loaded yet when + :func:`load_prune_config` runs. +* **DEC-011** — Total-budget enforcement: every test consults a + monotonic-ms wall clock; once the elapsed total ≥ + ``total_budget_seconds * 1000`` every remaining un-started test is + marked ``kept-without-evidence`` with a budget-specific ``why`` and + no warehouse call is issued. +* **DEC-016** — Fail-closed audit (mirrors safety / draft layers): a + failed audit write aborts the prune run. ``OSError`` / + ``PermissionError`` from the writer surface as + :class:`PruneAuditWriteError` with the original cause attached; + :class:`PruneAuditRecordTooLargeError` propagates as-is (it's + already a typed :class:`PruneError` subclass). + :class:`KeyboardInterrupt` / :class:`SystemExit` propagate untouched + so a Ctrl-C is never demoted to an audit error. +* **DEC-019** — Module-level :data:`_sleep` and :data:`_now_monotonic_ms` + aliases (mirrors :mod:`signalforge.llm.client`'s ``_sleep`` / + ``_rand_uniform`` pattern). Tests reassign these to deterministic + stand-ins so budget-exhaustion paths exercise without + timing-dependent flake. The orchestrator's normal path does not + invoke ``_sleep``; the alias is reserved for future budget-loop + work and pinned by a test so the seam doesn't drift. + +Per-test timeout-ms threading +----------------------------- + +v0.1 relies on the adapter's default :class:`QueryJobConfig` (no per-test +timeout). Per-test budget enforcement requires an additional +``run_test_sql(..., timeout_ms=...)`` parameter, deferred to v0.2 — the +existing ``make_query_job_config`` plumbing from US-002 is in place but +not exposed at :meth:`run_test_sql`. The ``kept-without-evidence`` +*outcome* is still emitted via two paths: + +* **Total-budget exceeded** — ``elapsed_total_ms ≥ + config.total_budget_seconds * 1000`` short-circuits every remaining + test before dispatch. +* **Warehouse error** — any :class:`WarehouseError` subclass raised by + ``run_test_sql`` (including a future BigQuery job-timeout mapping) + is caught and routed to ``kept-without-evidence``. + +This honours "tests we cannot evaluate are kept, not dropped" — the +conservative default that keeps a buggy adapter / oversized table from +silently losing signal-bearing tests. + +See ``plans/super/6-prune-engine.md`` for the full design. """ from __future__ import annotations +import json import logging +import time +from pathlib import Path + +from signalforge import __version__ as _SIGNALFORGE_VERSION +from signalforge.draft.models import CandidateSchema, CandidateTest +from signalforge.manifest.models import Manifest, Model +from signalforge.prune.audit import ( + _build_prune_event, + _compute_config_hash, + _write_prune_event, +) +from signalforge.prune.compiler import ( + _compile_test, + _compute_compiled_sql_hash, + _RequiresFutureData, +) +from signalforge.prune.config import PruneConfig +from signalforge.prune.errors import ( + PruneAuditRecordTooLargeError, + PruneAuditWriteError, + PruneTrustedModelNotFoundError, +) +from signalforge.prune.models import PruneDecision, PruneResult, Scope +from signalforge.warehouse.base import WarehouseAdapter +from signalforge.warehouse.errors import WarehouseError +from signalforge.warehouse.models import TableRef, TestResult _LOGGER = logging.getLogger(__name__) + +# Module-level aliases for deterministic test override (DEC-019). +# Mirrors src/signalforge/llm/client.py's _sleep / _rand_uniform pattern. +# `_sleep` is reserved for future budget-loop work (v0.2 per-test +# timeout enforcement); the alias is pinned by a test so the seam stays +# reassignable. `_now_monotonic_ms` IS used by the orchestrator's +# total-budget gate — tests reassign it to advance the clock past +# ``total_budget_seconds * 1000`` deterministically. +_sleep = time.sleep + + +def _now_monotonic_ms() -> int: + """Return the current monotonic time in integer milliseconds. + + Defined as a module-level function (not a lambda) so test code can + reassign ``signalforge.prune.engine._now_monotonic_ms`` without + having to import :mod:`time` to construct a stand-in. + """ + return int(time.monotonic() * 1000) + + +def _build_compiled_sql_hash_or_empty(sql: str) -> str: + """Return the 16-hex-char compiled-SQL hash, or a stable empty-string + hash for the no-SQL outcomes (``requires-future-data`` and the + budget-exceeded variant of ``kept-without-evidence``). + + Using an empty-string hash for the no-SQL outcomes keeps + :attr:`PruneDecision.compiled_sql_hash` non-optional in the read-back + schema — every audit JSONL row carries 16 hex chars regardless of + decision shape, so a downstream consumer never has to branch on a + nullable hash. + """ + return _compute_compiled_sql_hash(sql) + + +def _why_always_passes(sampled_rows: int | None, scope: Scope) -> str: + rows = sampled_rows if sampled_rows is not None else 0 + return f"Test passed on {rows} {scope} rows; no failures." + + +def _why_failed_on_known_clean_data( + failure_count: int, + sampled_rows: int | None, +) -> str: + rows = sampled_rows if sampled_rows is not None else 0 + return ( + f"Test failed with {failure_count} failures on {rows} sampled rows; " + "model.unique_id is in trusted_models, so the test is presumed buggy." + ) + + +def _why_kept(failure_count: int, sampled_rows: int | None, scope: Scope) -> str: + rows = sampled_rows if sampled_rows is not None else 0 + return ( + f"Test failed with {failure_count} failures on {rows} {scope} rows; " + "reviewer should evaluate." + ) + + +def _why_kept_without_evidence_warehouse_error(exc: WarehouseError) -> str: + return f"Test could not be evaluated: {type(exc).__name__}: {exc.message}" + + +def _why_kept_without_evidence_budget(budget_seconds: int) -> str: + return f"Total prune budget ({budget_seconds}s) exceeded before evaluation." + + +def _validate_trusted_models(config: PruneConfig, manifest: Manifest) -> None: + """Validate every ``config.trusted_models`` entry exists in the + manifest (DEC-008). + + Raised at orchestrator entry — BEFORE any warehouse call is issued — + so a typo'd unique_id surfaces loudly rather than silently losing + its "treat clean-data failure as a real failure" semantics. The + first miss wins (don't accumulate); the user should fix one typo + and re-run rather than chase a list. + """ + for unique_id in config.trusted_models: + if unique_id not in manifest.nodes: + raise PruneTrustedModelNotFoundError(unique_id=unique_id) + + +def _iter_candidate_tests( + candidates: CandidateSchema, +) -> list[tuple[str, CandidateTest]]: + """Flatten every candidate test (per-column then model-level) into a + list of ``(test_anchor, test)`` pairs. + + Per-column tests carry ``test_anchor=f"column.{column.name}"``; + model-level tests carry the literal ``"model"``. Iteration order is + deterministic: columns in declared order first, then model-level + tests — this lets a reviewer correlate the JSONL audit row order + with the candidate-schema YAML output. + """ + pairs: list[tuple[str, CandidateTest]] = [] + for column in candidates.columns: + for test in column.tests: + pairs.append((f"column.{column.name}", test)) + for test in candidates.tests: + pairs.append(("model", test)) + return pairs + + +def _write_audit_or_abort( + decision: PruneDecision, + *, + model_unique_id: str, + config_hash: str, + audit_path: Path, +) -> None: + """Write one :class:`PruneEvent` for ``decision`` via the fail-closed + writer; raise :class:`PruneAuditWriteError` on I/O failure. + + DEC-016 — propagation IS the defence. A failed audit write aborts + the prune run; the orchestrator does NOT continue with subsequent + tests. Mirrors safety/draft fail-closed audit semantics. + + Exception ladder (mirrors :func:`signalforge.draft.draft_from_request`): + + * :class:`PruneAuditRecordTooLargeError` — already a typed + :class:`PruneError`; propagates as-is so downstream + pattern-matching can branch on it. + * :class:`KeyboardInterrupt` / :class:`SystemExit` — signal-shaped + exits propagate untouched. Wrapping them would silently demote a + Ctrl-C into an audit error. + * Any other :class:`BaseException` (most commonly :class:`OSError` / + :class:`PermissionError` from the underlying ``os.write`` / + ``os.fsync``) wraps as :class:`PruneAuditWriteError` with the + original cause attached. + """ + event = _build_prune_event( + decision=decision, + model_unique_id=model_unique_id, + config_hash=config_hash, + ) + try: + _write_prune_event(event, audit_path) + except PruneAuditRecordTooLargeError: + # Already typed; propagate as-is. + raise + except (KeyboardInterrupt, SystemExit): + # Signal-shaped exits must propagate untouched. + raise + except BaseException as exc: + raise PruneAuditWriteError( + "Failed to durably persist a prune-decision audit record.", + cause=exc, + ) from exc + + +def _decide_from_test_result( + *, + test: CandidateTest, + test_anchor: str, + test_result: TestResult, + compiled_sql: str, + compiled_sql_hash: str, + elapsed_ms: int, + sampled_rows: int | None, + scope: Scope, + is_trusted: bool, + capture_failure_rows: int, +) -> PruneDecision: + """Route a successful :class:`TestResult` into a :class:`PruneDecision`. + + Routing matrix (per the plan's decision table): + + * ``failure_count == 0`` → ``decision="dropped"``, + ``reason="always-passes"`` (no signal — drops the test). + * ``failure_count > 0`` AND ``is_trusted`` → ``decision="dropped"``, + ``reason="failed-on-known-clean-data"`` (test is presumed buggy + against a model whose data is treated as known-clean). + * ``failure_count > 0`` AND not ``is_trusted`` → ``decision="kept"``, + ``reason="kept"`` (real failure on untrusted data — reviewer + should evaluate). + """ + failure_count = test_result.failure_count + sample_failures = tuple(test_result.sample_failures) if test_result.sample_failures else None + if failure_count == 0: + return PruneDecision( + test_anchor=test_anchor, + test=test, + decision="dropped", + reason="always-passes", + failures=0, + sampled_rows=sampled_rows, + scope=scope, + elapsed_ms=elapsed_ms, + compiled_sql_hash=compiled_sql_hash, + compiled_sql=compiled_sql, + why=_why_always_passes(sampled_rows, scope), + sample_failures=None, + ) + if is_trusted: + return PruneDecision( + test_anchor=test_anchor, + test=test, + decision="dropped", + reason="failed-on-known-clean-data", + failures=failure_count, + sampled_rows=sampled_rows, + scope=scope, + elapsed_ms=elapsed_ms, + compiled_sql_hash=compiled_sql_hash, + compiled_sql=compiled_sql, + why=_why_failed_on_known_clean_data(failure_count, sampled_rows), + sample_failures=sample_failures if capture_failure_rows > 0 else None, + ) + return PruneDecision( + test_anchor=test_anchor, + test=test, + decision="kept", + reason="kept", + failures=failure_count, + sampled_rows=sampled_rows, + scope=scope, + elapsed_ms=elapsed_ms, + compiled_sql_hash=compiled_sql_hash, + compiled_sql=compiled_sql, + why=_why_kept(failure_count, sampled_rows, scope), + sample_failures=sample_failures if capture_failure_rows > 0 else None, + ) + + +def _decide_requires_future_data( + *, + test: CandidateTest, + test_anchor: str, + sentinel: _RequiresFutureData, + elapsed_ms: int, + scope: Scope, +) -> PruneDecision: + """Build a :class:`PruneDecision` for a ``relationships`` test whose + parent isn't in the manifest (DEC-026). + + No warehouse call has been issued; ``compiled_sql`` is empty and + ``compiled_sql_hash`` is the stable hash of the empty string so the + audit JSONL shape stays uniform across decision types. + """ + return PruneDecision( + test_anchor=test_anchor, + test=test, + decision="dropped", + reason="requires-future-data", + failures=0, + sampled_rows=None, + scope=scope, + elapsed_ms=elapsed_ms, + compiled_sql_hash=_build_compiled_sql_hash_or_empty(""), + compiled_sql="", + why=sentinel.reason, + sample_failures=None, + ) + + +def _decide_kept_without_evidence_warehouse_error( + *, + test: CandidateTest, + test_anchor: str, + exc: WarehouseError, + compiled_sql: str, + compiled_sql_hash: str, + elapsed_ms: int, + scope: Scope, +) -> PruneDecision: + """Build a :class:`PruneDecision` for a test that raised a typed + :class:`WarehouseError` during execution. + + Conservative default (DEC-006): keep tests we cannot evaluate. + ``decision="kept"`` rather than ``"dropped"`` so a transient adapter + failure / table-not-found / auth blip doesn't silently lose a + signal-bearing test. The ``why`` carries the typed error class name + so a reviewer can correlate with the warehouse adapter's own logs. + """ + return PruneDecision( + test_anchor=test_anchor, + test=test, + decision="kept", + reason="kept-without-evidence", + failures=0, + sampled_rows=None, + scope=scope, + elapsed_ms=elapsed_ms, + compiled_sql_hash=compiled_sql_hash, + compiled_sql=compiled_sql, + why=_why_kept_without_evidence_warehouse_error(exc), + sample_failures=None, + ) + + +def _decide_kept_without_evidence_budget( + *, + test: CandidateTest, + test_anchor: str, + budget_seconds: int, + scope: Scope, +) -> PruneDecision: + """Build a :class:`PruneDecision` for a test that was never evaluated + because the total prune budget was exhausted (DEC-011). + + No compilation has happened; ``compiled_sql`` is empty and + ``elapsed_ms`` is 0 (the test never ran). The audit JSONL row is + still written so a reviewer sees exactly which tests would have + been evaluated next had the budget allowed. + """ + return PruneDecision( + test_anchor=test_anchor, + test=test, + decision="kept", + reason="kept-without-evidence", + failures=0, + sampled_rows=None, + scope=scope, + elapsed_ms=0, + compiled_sql_hash=_build_compiled_sql_hash_or_empty(""), + compiled_sql="", + why=_why_kept_without_evidence_budget(budget_seconds), + sample_failures=None, + ) + + +def prune_tests( + model: Model, + adapter: WarehouseAdapter, + candidates: CandidateSchema, + manifest: Manifest, + *, + config: PruneConfig | None = None, + audit_path: Path | None = None, +) -> PruneResult: + """Drop always-pass and known-clean-fail candidate tests. + + End-to-end orchestrator that integrates the prune layer's building + blocks (compiler US-007, audit writer US-008, typed value objects + US-004, error hierarchy US-006, config loader US-005) into one + public seam. Mirrors :func:`signalforge.draft.draft_schema` so the + CLI / wrapper layers see a consistent end-to-end shape across + pipeline stages. + + Pipeline: + + 1. Resolve config (``None`` → defaults). Compute ``config_hash`` via + ``sha256(canonical_json)[:16]`` for audit-row provenance. + 2. Validate :attr:`PruneConfig.trusted_models` against ``manifest`` + (DEC-008). The first miss raises + :class:`PruneTrustedModelNotFoundError`; no warehouse calls have + been issued. + 3. Resolve ``TableRef.from_model(model)`` for the model under + prune. May raise :class:`ManifestProjectNotFoundError` / + :class:`ManifestSchemaNotFoundError` — those are manifest-shape + problems and propagate as-is. + 4. Iterate every :class:`CandidateTest` (per-column, then + model-level). Track a running monotonic-ms clock; once + ``elapsed_total_ms ≥ config.total_budget_seconds * 1000`` mark + every remaining test ``kept-without-evidence`` and stop + dispatching warehouse calls (DEC-011). + 5. For each test: + + a. Compile via :func:`signalforge.prune.compiler._compile_test`. + ``_RequiresFutureData`` sentinel → ``requires-future-data`` + drop (no warehouse call). Otherwise → continue with the + compiled SQL string. + b. Run via :meth:`adapter.run_test_sql` inside a + ``try/except WarehouseError`` block. A typed + :class:`WarehouseError` routes to ``kept-without-evidence`` + with the error class name in ``why``. + c. Inspect :attr:`TestResult.failure_count`: + + * ``0`` → ``always-passes`` (drop). + * ``> 0`` AND ``model.unique_id`` in ``config.trusted_models`` + → ``failed-on-known-clean-data`` (drop). + * ``> 0`` otherwise → ``kept`` (keep). + d. Write the audit JSONL row via the fail-closed writer. + A failed audit write aborts the run (DEC-016). + 6. Build the aggregate :class:`PruneResult` and return. + + Caller responsibilities: + + * Loading config from ``signalforge.yml`` via + :func:`signalforge.prune.load_prune_config`. + * Opening the adapter via + :meth:`signalforge.warehouse.WarehouseAdapter.from_profile` or + direct construction. + * Ensuring the loaded :class:`Manifest` is consistent with the + ``model`` under prune. + + Args: + model: the manifest :class:`Model` under prune. + adapter: the :class:`WarehouseAdapter` (BigQuery in v0.1). + candidates: the :class:`CandidateSchema` from the LLM drafter + (#5). + manifest: the parent :class:`Manifest` — used by the compiler's + relationships-resolver (DEC-026) and the trusted-models + validator (DEC-008). + config: optional :class:`PruneConfig`; ``None`` resolves to + :class:`PruneConfig` defaults from DEC-009. The caller is + responsible for loading from disk via + :func:`load_prune_config`. + audit_path: optional override for the prune-audit JSONL path. + ``None`` resolves to ``cwd / ".signalforge" / "prune.jsonl"``; + the parent directory is created if missing. + + Returns: + A :class:`PruneResult` carrying every per-test + :class:`PruneDecision`, the run's elapsed time, and the version + stamp. + + Raises: + PruneTrustedModelNotFoundError: a ``trusted_models`` entry does + not match any model in ``manifest.nodes`` (DEC-008). Raised + at entry — no warehouse calls issued. + PruneAuditRecordTooLargeError: a per-decision audit record + exceeded the POSIX-atomic-append size cap. Aborts the run + (DEC-016). Propagates as-is. + PruneAuditWriteError: any other I/O / encoding failure in the + audit writer. Aborts the run (DEC-016) and wraps the + underlying exception on ``cause``. + ManifestProjectNotFoundError, ManifestSchemaNotFoundError: the + ``model`` under prune lacks ``database`` / ``schema``; + propagated from :meth:`TableRef.from_model`. + """ + resolved_config = config if config is not None else PruneConfig() + + # Validate trusted_models BEFORE any warehouse call (DEC-008). + _validate_trusted_models(resolved_config, manifest) + + table_ref = TableRef.from_model(model) + dialect = adapter.dialect() + is_trusted = model.unique_id in resolved_config.trusted_models + scope: Scope = resolved_config.scope + + # Resolve audit path. Default: cwd/.signalforge/prune.jsonl. Create + # the parent directory if missing — the writer itself does not + # mkdir (it expects the parent to exist; the orchestrator owns that + # contract per the writer's docstring). + resolved_audit_path = ( + audit_path if audit_path is not None else Path.cwd() / ".signalforge" / "prune.jsonl" + ) + resolved_audit_path.parent.mkdir(parents=True, exist_ok=True) + + # Compute the config hash once per run so every audit row in this + # run shares a single config_hash; mirrors the safety-layer + # policy_hash convention. + config_hash = _compute_config_hash( + resolved_config.model_dump_json(), + ) + + pairs = _iter_candidate_tests(candidates) + decisions: list[PruneDecision] = [] + start_ms = _now_monotonic_ms() + total_budget_ms = resolved_config.total_budget_seconds * 1000 + budget_exhausted = False + + for test_anchor, test in pairs: + # Total-budget gate (DEC-011) — checked BEFORE any compile or + # warehouse call. Once exceeded, every remaining test drains + # to ``kept-without-evidence`` without dispatching. + if not budget_exhausted: + elapsed_total_ms = _now_monotonic_ms() - start_ms + if elapsed_total_ms >= total_budget_ms: + budget_exhausted = True + + if budget_exhausted: + decision = _decide_kept_without_evidence_budget( + test=test, + test_anchor=test_anchor, + budget_seconds=resolved_config.total_budget_seconds, + scope=scope, + ) + _write_audit_or_abort( + decision, + model_unique_id=model.unique_id, + config_hash=config_hash, + audit_path=resolved_audit_path, + ) + decisions.append(decision) + continue + + # Compile the candidate test to failing-rows SQL. Returns either + # a string (the SELECT) or a _RequiresFutureData sentinel. + compile_result = _compile_test(test, table_ref, dialect, manifest) + if isinstance(compile_result, _RequiresFutureData): + decision = _decide_requires_future_data( + test=test, + test_anchor=test_anchor, + sentinel=compile_result, + elapsed_ms=0, + scope=scope, + ) + _write_audit_or_abort( + decision, + model_unique_id=model.unique_id, + config_hash=config_hash, + audit_path=resolved_audit_path, + ) + decisions.append(decision) + continue + + compiled_sql: str = compile_result + compiled_sql_hash = _build_compiled_sql_hash_or_empty(compiled_sql) + + # Per-test timing wraps the warehouse call. + test_start_ms = _now_monotonic_ms() + try: + test_result = adapter.run_test_sql( + compiled_sql, + capture_failures=resolved_config.capture_failure_rows, + ) + except WarehouseError as exc: + elapsed_ms = max(0, _now_monotonic_ms() - test_start_ms) + # Lazy-format JSON per DEC-017 — never f-string-interpolate + # user-controlled values into a logger call. + _LOGGER.warning( + "kept-without-evidence: %s", + json.dumps( + { + "model_unique_id": model.unique_id, + "test_anchor": test_anchor, + "error_class": type(exc).__name__, + } + ), + ) + decision = _decide_kept_without_evidence_warehouse_error( + test=test, + test_anchor=test_anchor, + exc=exc, + compiled_sql=compiled_sql, + compiled_sql_hash=compiled_sql_hash, + elapsed_ms=elapsed_ms, + scope=scope, + ) + _write_audit_or_abort( + decision, + model_unique_id=model.unique_id, + config_hash=config_hash, + audit_path=resolved_audit_path, + ) + decisions.append(decision) + continue + + elapsed_ms = max(0, _now_monotonic_ms() - test_start_ms) + # ``sampled_rows`` is the size of the row set the test ran + # against. v0.1 does not yet thread the sample-size from the + # adapter's TABLESAMPLE / hash-mod path through to the prune + # layer (see DEC-026 deferral notes); we record None for + # ``"sample"`` scope until US-011 wires it through. For + # ``"full"`` scope sampled_rows is None by design (the model + # contract: ``scope == "full"`` ↔ sampled_rows is None). + sampled_rows: int | None = None + decision = _decide_from_test_result( + test=test, + test_anchor=test_anchor, + test_result=test_result, + compiled_sql=compiled_sql, + compiled_sql_hash=compiled_sql_hash, + elapsed_ms=elapsed_ms, + sampled_rows=sampled_rows, + scope=scope, + is_trusted=is_trusted, + capture_failure_rows=resolved_config.capture_failure_rows, + ) + _write_audit_or_abort( + decision, + model_unique_id=model.unique_id, + config_hash=config_hash, + audit_path=resolved_audit_path, + ) + decisions.append(decision) + + total_elapsed_ms = max(0, _now_monotonic_ms() - start_ms) + return PruneResult( + model_unique_id=model.unique_id, + decisions=tuple(decisions), + elapsed_ms=total_elapsed_ms, + signalforge_version=_SIGNALFORGE_VERSION, + ) + + +__all__ = ("prune_tests",) diff --git a/tests/prune/test_engine.py b/tests/prune/test_engine.py new file mode 100644 index 00000000..ba6cb73e --- /dev/null +++ b/tests/prune/test_engine.py @@ -0,0 +1,574 @@ +"""Tests for ``signalforge.prune.engine`` (US-009). + +Pins the eleven load-bearing properties of the prune orchestrator across +the full DropReason routing matrix, the trusted-models entry-time +validation (DEC-008), the total-budget short-circuit (DEC-011), and the +fail-closed audit-write semantics (DEC-016). Every test injects a +:class:`tests.warehouse._fake.FakeBigQueryClient` into a real +:class:`signalforge.warehouse.adapters.bigquery.BigQueryAdapter` — the +proven adapter+fake-client pair from US-002 / US-003. No production code +imports the fake. + +The DropReason routing matrix (kept-without-evidence is ``decision="kept"`` +because the test ships, conservatively, when we cannot evaluate it): + +| compile result | failure_count | trusted? | decision | reason | +| --------------------- | ------------- | -------- | --------- | ---------------------------- | +| _RequiresFutureData | n/a | n/a | dropped | requires-future-data | +| SQL string | 0 | any | dropped | always-passes | +| SQL string | > 0 | yes | dropped | failed-on-known-clean-data | +| SQL string | > 0 | no | kept | kept | +| WarehouseError | n/a | n/a | kept | kept-without-evidence | +| budget-exceeded | n/a | n/a | kept | kept-without-evidence | +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from signalforge.draft.models import ( + CandidateColumn, + CandidateSchema, + CandidateTestNotNull, + CandidateTestRelationships, +) +from signalforge.manifest.models import Column, Manifest, Model +from signalforge.prune import engine as engine_module +from signalforge.prune.config import PruneConfig +from signalforge.prune.engine import prune_tests +from signalforge.prune.errors import ( + PruneAuditRecordTooLargeError, + PruneAuditWriteError, + PruneTrustedModelNotFoundError, +) +from signalforge.warehouse.adapters.bigquery import BigQueryAdapter +from signalforge.warehouse.errors import TableNotFoundError +from tests.warehouse._fake import FakeBigQueryClient + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_orders_model() -> Model: + return Model( + unique_id="model.shop.orders", + name="orders", + resource_type="model", + package_name="shop", + original_file_path="models/orders.sql", + path="orders.sql", + database="fake_project", + schema="dataset", # type: ignore[call-arg] + columns={ + "id": Column(name="id"), + "customer_id": Column(name="customer_id"), + "status": Column(name="status"), + }, + raw_code="select 1", + ) + + +def _make_manifest(model: Model) -> Manifest: + return Manifest( + metadata={"dbt_schema_version": "v12"}, + nodes={model.unique_id: model}, + ) + + +def _make_adapter(fake: FakeBigQueryClient) -> BigQueryAdapter: + return BigQueryAdapter( + project="fake_project", + location="US", + max_bytes_billed=100_000_000, + client=fake, + ) + + +def _candidates_with_one_test(test_anchor_column: str) -> CandidateSchema: + """Build a CandidateSchema with a single ``not_null`` test on the + given column. + """ + return CandidateSchema( + name="orders", + description="Order events.", + columns=( + CandidateColumn( + name=test_anchor_column, + description="The order's primary key.", + tests=(CandidateTestNotNull(column=test_anchor_column),), + ), + ), + ) + + +def _candidates_with_n_tests(n: int) -> CandidateSchema: + """Build a CandidateSchema with N ``not_null`` tests, all on + ``customer_id``. Used by the budget test. + """ + return CandidateSchema( + name="orders", + description="Order events.", + columns=( + CandidateColumn( + name="customer_id", + description="FK to customers.", + tests=tuple(CandidateTestNotNull(column="customer_id") for _ in range(n)), + ), + ), + ) + + +def _read_audit_lines(audit_path: Path) -> list[dict[str, Any]]: + if not audit_path.exists(): + return [] + return [json.loads(line) for line in audit_path.read_text().splitlines() if line] + + +# --------------------------------------------------------------------------- +# Routing matrix tests +# --------------------------------------------------------------------------- + + +def test_prune_tests_always_passes_drops_test(tmp_path: Path) -> None: + """A test that returns ``failure_count=0`` is dropped with + ``reason="always-passes"`` and the audit JSONL records it. + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + fake.expect_query(matching=r"SELECT COUNT\(\*\)", returns=[{"failures": 0}]) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig(capture_failure_rows=0) + + result = prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + ) + + assert result.total_tests == 1 + assert result.dropped_count == 1 + assert result.kept_count == 0 + decision = result.decisions[0] + assert decision.decision == "dropped" + assert decision.reason == "always-passes" + assert decision.failures == 0 + assert decision.test_anchor == "column.id" + fake.assert_all_expectations_met() + + audit_rows = _read_audit_lines(audit_path) + assert len(audit_rows) == 1 + assert audit_rows[0]["reason"] == "always-passes" + assert audit_rows[0]["model_unique_id"] == "model.shop.orders" + + +def test_prune_tests_kept_for_real_failure_untrusted_model(tmp_path: Path) -> None: + """A test that fails on an untrusted model is kept with + ``reason="kept"`` and the ``why`` mentions the failure count. + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + fake.expect_query(matching=r"SELECT COUNT\(\*\)", returns=[{"failures": 3}]) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig(capture_failure_rows=0) # untrusted by default + + result = prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + ) + + decision = result.decisions[0] + assert decision.decision == "kept" + assert decision.reason == "kept" + assert decision.failures == 3 + assert "3 failures" in decision.why + fake.assert_all_expectations_met() + + +def test_prune_tests_failed_on_known_clean_data_for_trusted_model(tmp_path: Path) -> None: + """A test that fails on a trusted model is dropped with + ``reason="failed-on-known-clean-data"`` (presumed buggy). + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + fake.expect_query(matching=r"SELECT COUNT\(\*\)", returns=[{"failures": 7}]) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig( + trusted_models=(model.unique_id,), + capture_failure_rows=0, + ) + + result = prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + ) + + decision = result.decisions[0] + assert decision.decision == "dropped" + assert decision.reason == "failed-on-known-clean-data" + assert decision.failures == 7 + assert "trusted_models" in decision.why + fake.assert_all_expectations_met() + + +def test_prune_tests_requires_future_data_for_unknown_relationships_parent( + tmp_path: Path, +) -> None: + """A ``relationships(to="nonexistent_model")`` is dropped with + ``reason="requires-future-data"`` and NO warehouse call is issued. + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + # Intentionally NO expect_query — any call is unexpected. + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = CandidateSchema( + name="orders", + description="Order events.", + columns=( + CandidateColumn( + name="customer_id", + description="FK to customers.", + tests=( + CandidateTestRelationships( + column="customer_id", + to="nonexistent_model", + field="id", + ), + ), + ), + ), + ) + config = PruneConfig(capture_failure_rows=0) + + result = prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + ) + + decision = result.decisions[0] + assert decision.decision == "dropped" + assert decision.reason == "requires-future-data" + # `why` carries the sentinel's reason — references the missing parent name. + assert "nonexistent_model" in decision.why + # No queries dispatched — verify by asserting all (zero) expectations met. + fake.assert_all_expectations_met() + + +def test_prune_tests_warehouse_error_routes_to_kept_without_evidence( + tmp_path: Path, +) -> None: + """A typed :class:`WarehouseError` from the adapter routes to + ``kept-without-evidence`` — conservative default keeps the test + rather than silently dropping it. + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + fake.expect_query( + matching=r"SELECT COUNT\(\*\)", + returns=TableNotFoundError(table="fake_project.dataset.orders"), + ) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig(capture_failure_rows=0) + + result = prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + ) + + decision = result.decisions[0] + assert decision.decision == "kept" + assert decision.reason == "kept-without-evidence" + assert "TableNotFoundError" in decision.why + fake.assert_all_expectations_met() + + +def test_prune_tests_total_budget_exceeded_marks_remaining_kept_without_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Once ``total_budget_seconds * 1000`` ms have elapsed, every + remaining un-started test drains to ``kept-without-evidence`` with + a budget-specific ``why`` and NO warehouse call is issued (DEC-011). + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + # Only the first test runs (one expectation). + fake.expect_query(matching=r"SELECT COUNT\(\*\)", returns=[{"failures": 0}]) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_n_tests(5) + config = PruneConfig(total_budget_seconds=1, capture_failure_rows=0) + + # Stub `_now_monotonic_ms` so the second iteration sees the budget + # exhausted (advances past 1000 ms after the first call). The clock + # returns: + # call 0 → 0 ms (start_ms) + # call 1 → 0 ms (first elapsed-total check, budget not exceeded) + # call 2 → 0 ms (test_start_ms for the first test) + # call 3 → 10 ms (after first test) + # call 4+ → 5000 ms (budget exhausted on iteration 2 onwards) + timeline = iter([0, 0, 0, 10, 5000, 5000, 5000, 5000, 5000, 5000, 5000]) + + def fake_clock() -> int: + return next(timeline) + + monkeypatch.setattr(engine_module, "_now_monotonic_ms", fake_clock) + + result = prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + ) + + assert result.total_tests == 5 + # First test ran (always-passes drop). + assert result.decisions[0].reason == "always-passes" + # Remaining four are kept-without-evidence due to budget. + for decision in result.decisions[1:]: + assert decision.decision == "kept" + assert decision.reason == "kept-without-evidence" + assert "Total prune budget" in decision.why + # Exactly one warehouse call consumed (the first test's). + fake.assert_all_expectations_met() + + +def test_prune_tests_trusted_models_validation_at_entry(tmp_path: Path) -> None: + """A typo'd ``trusted_models`` unique_id raises + :class:`PruneTrustedModelNotFoundError` at entry — BEFORE any + warehouse call is issued (DEC-008). + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + # Intentionally NO expect_query — any call is unexpected. If the + # validation failed to fire at entry, the orchestrator would + # dispatch to the fake and the assert_all_expectations_met below + # would still pass; instead we verify by catching the typed + # exception (the only path to NO call given the candidate set). + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig(trusted_models=("model.shop.nonexistent",)) + + with pytest.raises(PruneTrustedModelNotFoundError) as excinfo: + prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + ) + + assert excinfo.value.unique_id == "model.shop.nonexistent" + # No warehouse calls: zero expectations were registered, and zero + # were consumed — assert via the fake's accounting. + fake.assert_all_expectations_met() + # No audit JSONL written either — entry-time validation aborts + # before any decision is built. + assert not audit_path.exists() + + +def test_prune_tests_audit_write_oserror_aborts_run( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An :class:`OSError` from :func:`_write_prune_event` aborts the + run and surfaces as :class:`PruneAuditWriteError` with the original + cause attached. NO :class:`PruneResult` is returned (DEC-016). + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + # Three tests; only the first two get to dispatch before the second + # write blows up. + fake.expect_query(matching=r"SELECT COUNT\(\*\)", returns=[{"failures": 0}]) + fake.expect_query(matching=r"SELECT COUNT\(\*\)", returns=[{"failures": 0}]) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_n_tests(3) + config = PruneConfig(capture_failure_rows=0) + + call_count = {"n": 0} + boom = OSError("disk full") + + def fake_write(*args: Any, **kwargs: Any) -> None: + call_count["n"] += 1 + if call_count["n"] == 2: + raise boom + + monkeypatch.setattr(engine_module, "_write_prune_event", fake_write) + + with pytest.raises(PruneAuditWriteError) as excinfo: + prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + ) + + # The OSError surfaces as the cause on the typed wrapper. + assert excinfo.value.cause is boom + # __cause__ chain is preserved per ``raise X from cause``. + assert excinfo.value.__cause__ is boom + + +def test_prune_tests_audit_record_too_large_propagates( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """:class:`PruneAuditRecordTooLargeError` is already a typed + :class:`PruneError` subclass; the orchestrator propagates it + UNCHANGED rather than wrapping it as :class:`PruneAuditWriteError`. + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + fake.expect_query(matching=r"SELECT COUNT\(\*\)", returns=[{"failures": 0}]) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig(capture_failure_rows=0) + + expected = PruneAuditRecordTooLargeError(size=5000, limit=4000) + + def fake_write(*args: Any, **kwargs: Any) -> None: + raise expected + + monkeypatch.setattr(engine_module, "_write_prune_event", fake_write) + + with pytest.raises(PruneAuditRecordTooLargeError) as excinfo: + prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + ) + + # Same object — NOT wrapped. + assert excinfo.value is expected + + +def test_prune_tests_module_level_sleep_alias_is_reassignable() -> None: + """:data:`signalforge.prune.engine._sleep` exists, IS callable, AND + can be reassigned to a recording stub (DEC-019). Mirrors + :data:`signalforge.llm.client._sleep` — the alias is reserved for + future budget-loop work; this test pins the seam. + """ + # Exists and is callable. + assert callable(engine_module._sleep) + + # Reassignable: a recording stub replaces the alias and is + # observable via the module attribute (mirrors how production code + # would dispatch). + calls: list[float] = [] + + def recording_sleep(seconds: float) -> None: + calls.append(seconds) + + original = engine_module._sleep + try: + engine_module._sleep = recording_sleep # type: ignore[assignment] + engine_module._sleep(0.5) + assert calls == [0.5] + finally: + engine_module._sleep = original # type: ignore[assignment] + + +def test_prune_tests_compute_config_hash_is_deterministic(tmp_path: Path) -> None: + """Two calls with identical :class:`PruneConfig` produce identical + ``config_hash`` in the audit JSONL. + """ + audit_a = tmp_path / "prune_a.jsonl" + audit_b = tmp_path / "prune_b.jsonl" + + fake_a = FakeBigQueryClient(project="fake_project") + fake_a.expect_query(matching=r"SELECT COUNT\(\*\)", returns=[{"failures": 0}]) + adapter_a = _make_adapter(fake_a) + + fake_b = FakeBigQueryClient(project="fake_project") + fake_b.expect_query(matching=r"SELECT COUNT\(\*\)", returns=[{"failures": 0}]) + adapter_b = _make_adapter(fake_b) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig(capture_failure_rows=0) + + prune_tests( + model, + adapter_a, + candidates, + manifest, + config=config, + audit_path=audit_a, + ) + prune_tests( + model, + adapter_b, + candidates, + manifest, + config=config, + audit_path=audit_b, + ) + + rows_a = _read_audit_lines(audit_a) + rows_b = _read_audit_lines(audit_b) + assert len(rows_a) == 1 and len(rows_b) == 1 + assert rows_a[0]["config_hash"] == rows_b[0]["config_hash"] + # 16-hex-char convention (same as policy_hash / config_hash elsewhere). + assert len(rows_a[0]["config_hash"]) == 16 From cb182703db9b6d9041dd46c0b32f7b604dc0d492 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:44:19 -0700 Subject: [PATCH 14/24] bd_1-scaffolding-y8y.13: signalforge.prune public re-exports Re-export the v0.1 public surface from signalforge.prune: * prune_tests (orchestrator entry) * PruneResult / PruneDecision / DropReason / Scope (read-back models) * PruneConfig / load_prune_config (user config) * PruneEvent (audit type; constructed only in audit.py per AST gate) * PruneError + five concrete subclasses Internal helpers (_compile_test, _write_prune_event, _sleep, etc.) stay reachable via dotted import for tests but are absent from the package namespace -- DEC-021. Smoke test (tests/prune/test_smoke.py) pins the surface against __all__ and ensures internal helpers don't leak. Plan: plans/super/6-prune-engine.md (DEC-021). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/prune/__init__.py | 62 +++++++++++++++++++++----- tests/prune/test_smoke.py | 74 +++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 12 deletions(-) create mode 100644 tests/prune/test_smoke.py diff --git a/src/signalforge/prune/__init__.py b/src/signalforge/prune/__init__.py index 18297cd6..a73f31b2 100644 --- a/src/signalforge/prune/__init__.py +++ b/src/signalforge/prune/__init__.py @@ -1,16 +1,54 @@ -"""SignalForge prune layer. +"""SignalForge prune layer — drop always-pass and known-clean-fail +candidate tests against warehouse data. -The signal-vs-volume gate. Runs every candidate test from the drafter against -real warehouse data via the warehouse adapter and drops the ones with no -signal: tests that always pass on warehouse samples, tests that fail on -known-clean data, and tests whose required parent data does not yet exist. -Conservative defaults keep tests we cannot evaluate (`kept-without-evidence`) -rather than silently dropping them, and every kept/dropped test ships with a -structured `PruneDecision` carrying the drop reason, failure count, scope, and -the literal compiled SQL so downstream stages (#7 grader, #8 diff renderer) can -explain every artifact. +Public API (issue #6 ticket; ``v0.1``): -Public API lands in US-013; this module currently re-exports nothing. +* :func:`prune_tests` — orchestrator; runs every candidate test against + the warehouse and emits a :class:`PruneResult` with kept/dropped + decisions and per-decision rationale. +* :class:`PruneResult`, :class:`PruneDecision` — typed result shapes. +* :class:`PruneConfig`, :func:`load_prune_config` — user config. +* :data:`DropReason`, :data:`Scope` — discriminator literals. +* :class:`PruneEvent` — fail-closed JSONL audit record (constructed + ONLY by ``signalforge.prune.audit``; AST-gated per DEC-018). +* :class:`PruneError` and the five concrete error classes. -See plans/super/6-prune-engine.md for the full design. +See ``plans/super/6-prune-engine.md`` for the full design. """ + +from signalforge.prune.audit import PruneEvent +from signalforge.prune.config import PruneConfig, load_prune_config +from signalforge.prune.engine import prune_tests +from signalforge.prune.errors import ( + PruneAuditRecordTooLargeError, + PruneAuditWriteError, + PruneConfigError, + PruneError, + PruneTimeoutError, + PruneTrustedModelNotFoundError, +) +from signalforge.prune.models import ( + DropReason, + PruneDecision, + PruneResult, + Scope, +) + +__all__ = ( + # Public API + "prune_tests", + "PruneResult", + "PruneDecision", + "PruneConfig", + "load_prune_config", + "DropReason", + "Scope", + "PruneEvent", + # Errors + "PruneError", + "PruneConfigError", + "PruneTrustedModelNotFoundError", + "PruneTimeoutError", + "PruneAuditWriteError", + "PruneAuditRecordTooLargeError", +) diff --git a/tests/prune/test_smoke.py b/tests/prune/test_smoke.py new file mode 100644 index 00000000..4bf4de17 --- /dev/null +++ b/tests/prune/test_smoke.py @@ -0,0 +1,74 @@ +"""Public-API smoke test for signalforge.prune (US-013). + +Mirrors tests/test_smoke.py at the package root. Asserts every public +name lands at the expected re-export path with the expected type. +""" + +from __future__ import annotations + + +def test_public_api_imports() -> None: + """Every documented public name is importable from signalforge.prune.""" + from signalforge.prune import ( + DropReason, # noqa: F401 + PruneAuditRecordTooLargeError, # noqa: F401 + PruneAuditWriteError, # noqa: F401 + PruneConfig, # noqa: F401 + PruneConfigError, # noqa: F401 + PruneDecision, # noqa: F401 + PruneError, # noqa: F401 + PruneEvent, # noqa: F401 + PruneResult, # noqa: F401 + PruneTimeoutError, # noqa: F401 + PruneTrustedModelNotFoundError, # noqa: F401 + Scope, # noqa: F401 + load_prune_config, + prune_tests, + ) + + # Spot-check that the imports are not None and at least one is callable + assert prune_tests is not None + assert callable(prune_tests) + assert callable(load_prune_config) + + +def test_public_api_dunder_all_matches_imports() -> None: + """``signalforge.prune.__all__`` enumerates every public name.""" + import signalforge.prune as prune + + expected = { + "prune_tests", + "PruneResult", + "PruneDecision", + "PruneConfig", + "load_prune_config", + "DropReason", + "Scope", + "PruneEvent", + "PruneError", + "PruneConfigError", + "PruneTrustedModelNotFoundError", + "PruneTimeoutError", + "PruneAuditWriteError", + "PruneAuditRecordTooLargeError", + } + assert set(prune.__all__) == expected + + +def test_internal_helpers_are_not_re_exported() -> None: + """Underscore-prefixed helpers stay out of the public surface. + + They remain reachable via dotted import (``from + signalforge.prune.engine import _sleep``) for tests, but are not + part of the package's public API. + """ + import signalforge.prune as prune + + # A handful of internals that should NOT be on the public surface + assert not hasattr(prune, "_compile_test") + assert not hasattr(prune, "_write_prune_event") + assert not hasattr(prune, "_build_prune_event") + assert not hasattr(prune, "_RequiresFutureData") + assert not hasattr(prune, "_compute_config_hash") + assert not hasattr(prune, "_compute_compiled_sql_hash") + assert not hasattr(prune, "_sleep") From 88494e1667dddc26f04c9d061da7bfe31e63ad9b Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:46:42 -0700 Subject: [PATCH 15/24] bd_1-scaffolding-y8y.11: extend logger grep gate to signalforge.prune Add src/signalforge/prune/ to the lazy-format-logger scan. Renames the test to test_no_f_string_logger_calls_in_llm_draft_or_prune_modules to reflect coverage. Mirrors the existing DEC-022 / DEC-011 gate across all three subpackages that emit observability. Plan: plans/super/6-prune-engine.md (DEC-017). Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/llm/test_logger_grep_gate.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/llm/test_logger_grep_gate.py b/tests/llm/test_logger_grep_gate.py index dff81957..922fdeb9 100644 --- a/tests/llm/test_logger_grep_gate.py +++ b/tests/llm/test_logger_grep_gate.py @@ -1,7 +1,8 @@ """ANSI-safe lazy-format logger grep gate (US-014 / DEC-011). Mirrors the safety layer's DEC-022 grep gate, extended to -:mod:`signalforge.llm` and :mod:`signalforge.draft`. Every +:mod:`signalforge.llm`, :mod:`signalforge.draft`, and +:mod:`signalforge.prune`. Every ``_LOGGER.{info,warning,debug,error}`` call in those subpackages must use lazy ``%s``-formatting with :func:`json.dumps` for any user-controlled string — never f-string interpolation. A column name @@ -22,6 +23,7 @@ _REPO_ROOT = Path(__file__).resolve().parent.parent.parent _LLM_DIR = _REPO_ROOT / "src" / "signalforge" / "llm" _DRAFT_DIR = _REPO_ROOT / "src" / "signalforge" / "draft" +_PRUNE_DIR = _REPO_ROOT / "src" / "signalforge" / "prune" # Matches ``_LOGGER.(f"...``, ``_LOGGER.(f'...``, # ``_LOGGER.( f"...``, ``_LOGGER.(rf"...``, @@ -46,20 +48,25 @@ def _scan_for_f_string_logger_calls(root: Path) -> list[tuple[Path, int, str]]: return hits -def test_no_f_string_logger_calls_in_llm_or_draft_modules() -> None: +def test_no_f_string_logger_calls_in_llm_draft_or_prune_modules() -> None: """DEC-011: no f-string-interpolated ``_LOGGER`` calls in - :mod:`signalforge.llm` or :mod:`signalforge.draft`. + :mod:`signalforge.llm`, :mod:`signalforge.draft`, or + :mod:`signalforge.prune`. Use lazy ``%s`` formatting with :func:`json.dumps` for any user-controlled string. ANSI escapes in column names / model ids inject directly into log viewers when interpolated via f-string; JSON encoding escapes them safely. """ - hits = _scan_for_f_string_logger_calls(_LLM_DIR) + _scan_for_f_string_logger_calls(_DRAFT_DIR) + hits = ( + _scan_for_f_string_logger_calls(_LLM_DIR) + + _scan_for_f_string_logger_calls(_DRAFT_DIR) + + _scan_for_f_string_logger_calls(_PRUNE_DIR) + ) formatted = "\n".join(f" {p}:{line}: {content}" for p, line, content in hits) assert not hits, ( "Found f-string-interpolated _LOGGER calls in signalforge.llm / " - "signalforge.draft (DEC-011 violation):\n" + "signalforge.draft / signalforge.prune (DEC-011 violation):\n" f"{formatted}\n" "Use lazy-format with json.dumps instead, e.g.:\n" ' _LOGGER.info("audit event: %s", json.dumps({"unique_id": ...}))' From 0c48d09903779cd9caed197a32608b743c9017bc Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:51:05 -0700 Subject: [PATCH 16/24] bd_1-scaffolding-y8y.10: drift detectors for prune read-back models Three extra="forbid" Strict mirrors (StrictPruneResult, StrictPruneDecision, StrictPruneEvent) paired with committed JSON/ JSONL fixtures covering all five DropReason values. Field-set parity tests catch silent schema drift before a v0.2 reader does. Mirrors tests/safety/test_drift_detector.py shape exactly. PruneConfig is already extra="forbid" in production (DEC-015), so no drift gate is needed there. Plan: plans/super/6-prune-engine.md (DEC-010, DEC-015). Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/fixtures/prune/prune_decision_v1.json | 103 ++++++++ tests/fixtures/prune/prune_event_v1.jsonl | 5 + tests/fixtures/prune/prune_result_v1.json | 68 +++++ tests/prune/test_drift_detector.py | 264 ++++++++++++++++++++ 4 files changed, 440 insertions(+) create mode 100644 tests/fixtures/prune/prune_decision_v1.json create mode 100644 tests/fixtures/prune/prune_event_v1.jsonl create mode 100644 tests/fixtures/prune/prune_result_v1.json create mode 100644 tests/prune/test_drift_detector.py diff --git a/tests/fixtures/prune/prune_decision_v1.json b/tests/fixtures/prune/prune_decision_v1.json new file mode 100644 index 00000000..4ab1e48e --- /dev/null +++ b/tests/fixtures/prune/prune_decision_v1.json @@ -0,0 +1,103 @@ +[ + { + "test_anchor": "column.id", + "test": { + "type": "not_null", + "column": "id", + "rationale": "Primary key surrogate; must never be null." + }, + "decision": "dropped", + "reason": "always-passes", + "failures": 0, + "sampled_rows": 10000, + "scope": "sample", + "elapsed_ms": 87, + "compiled_sql_hash": "0123456789abcdef", + "compiled_sql": "SELECT COUNT(*) AS failures FROM `proj.dataset.fct_orders` WHERE id IS NULL", + "why": "0 failures across 10000 sampled rows; always-pass tests consume reviewer attention without catching anything.", + "sample_failures": null + }, + { + "test_anchor": "column.order_id", + "test": { + "type": "unique", + "column": "order_id", + "rationale": "Surrogate primary key for the orders fact." + }, + "decision": "kept", + "reason": "kept", + "failures": 3, + "sampled_rows": 10000, + "scope": "sample", + "elapsed_ms": 142, + "compiled_sql_hash": "fedcba9876543210", + "compiled_sql": "SELECT order_id FROM `proj.dataset.fct_orders` GROUP BY order_id HAVING COUNT(*) > 1", + "why": "3 duplicate order_id values across 10000 sampled rows; real signal worth shipping.", + "sample_failures": [ + {"order_id": "ORD-1001"}, + {"order_id": "ORD-1002"}, + {"order_id": "ORD-1003"} + ] + }, + { + "test_anchor": "column.customer_id", + "test": { + "type": "relationships", + "column": "customer_id", + "to": "ref('dim_customers')", + "field": "customer_id", + "rationale": "FK from fct_orders to dim_customers." + }, + "decision": "dropped", + "reason": "requires-future-data", + "failures": 0, + "sampled_rows": null, + "scope": "full", + "elapsed_ms": 312, + "compiled_sql_hash": "1111222233334444", + "compiled_sql": "SELECT child.customer_id FROM `proj.dataset.fct_orders` child LEFT JOIN `proj.dataset.dim_customers` parent ON child.customer_id = parent.customer_id WHERE parent.customer_id IS NULL AND child.customer_id IS NOT NULL", + "why": "Referenced model dim_customers is not yet built in the warehouse; cannot evaluate.", + "sample_failures": null + }, + { + "test_anchor": "column.status", + "test": { + "type": "accepted_values", + "column": "status", + "values": ["pending", "shipped", "cancelled"], + "rationale": "Documented order status enum." + }, + "decision": "dropped", + "reason": "failed-on-known-clean-data", + "failures": 27, + "sampled_rows": 10000, + "scope": "sample", + "elapsed_ms": 96, + "compiled_sql_hash": "5555666677778888", + "compiled_sql": "SELECT status FROM `proj.dataset.fct_orders` WHERE status NOT IN ('pending', 'shipped', 'cancelled')", + "why": "27 rows in known-clean sample violate the accepted_values list; LLM hallucinated the enum.", + "sample_failures": [ + {"status": "returned"}, + {"status": "refunded"}, + {"status": "in_transit"} + ] + }, + { + "test_anchor": "column.email", + "test": { + "type": "not_null", + "column": "email", + "rationale": "Customer email is required at signup." + }, + "decision": "kept", + "reason": "kept-without-evidence", + "failures": 0, + "sampled_rows": 12, + "scope": "sample", + "elapsed_ms": 54, + "compiled_sql_hash": "9999aaaabbbbcccc", + "compiled_sql": "SELECT COUNT(*) AS failures FROM `proj.dataset.dim_customers` WHERE email IS NULL", + "why": "Sample too small (12 rows) to support a verdict; test is well-formed and references a real column.", + "sample_failures": null + } +] diff --git a/tests/fixtures/prune/prune_event_v1.jsonl b/tests/fixtures/prune/prune_event_v1.jsonl new file mode 100644 index 00000000..366688c3 --- /dev/null +++ b/tests/fixtures/prune/prune_event_v1.jsonl @@ -0,0 +1,5 @@ +{"audit_schema_version":1,"signalforge_version":"0.1.0.dev0","record_id":"a1b2c3d4e5f6478890aabbccddeeff00","timestamp":"2026-04-30T12:00:00.000000Z","config_hash":"abc123def456789a","model_unique_id":"model.sf_demo.fct_orders","test":{"type":"not_null","column":"id","rationale":"Primary key surrogate."},"test_anchor":"column.id","decision":"dropped","reason":"always-passes","failures":0,"sampled_rows":10000,"scope":"sample","elapsed_ms":87,"compiled_sql_hash":"0123456789abcdef","compiled_sql":"SELECT COUNT(*) AS failures FROM `p.d.fct_orders` WHERE id IS NULL","why":"0 failures across 10000 sampled rows; always-pass.","sample_failures":null} +{"audit_schema_version":1,"signalforge_version":"0.1.0.dev0","record_id":"b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7","timestamp":"2026-04-30T12:00:01.000000Z","config_hash":"abc123def456789a","model_unique_id":"model.sf_demo.fct_orders","test":{"type":"unique","column":"order_id","rationale":"Surrogate PK."},"test_anchor":"column.order_id","decision":"kept","reason":"kept","failures":3,"sampled_rows":10000,"scope":"sample","elapsed_ms":142,"compiled_sql_hash":"fedcba9876543210","compiled_sql":"SELECT order_id FROM `p.d.fct_orders` GROUP BY order_id HAVING COUNT(*) > 1","why":"3 duplicate order_id values across 10000 sampled rows.","sample_failures":[{"order_id":"ORD-1001"},{"order_id":"ORD-1002"}]} +{"audit_schema_version":1,"signalforge_version":"0.1.0.dev0","record_id":"c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8","timestamp":"2026-04-30T12:00:02.000000Z","config_hash":"abc123def456789a","model_unique_id":"model.sf_demo.fct_orders","test":{"type":"relationships","column":"customer_id","to":"ref('dim_customers')","field":"customer_id","rationale":"FK to dim_customers."},"test_anchor":"column.customer_id","decision":"dropped","reason":"requires-future-data","failures":0,"sampled_rows":null,"scope":"full","elapsed_ms":312,"compiled_sql_hash":"1111222233334444","compiled_sql":"SELECT child.customer_id FROM `p.d.fct_orders` child LEFT JOIN `p.d.dim_customers` parent ON child.customer_id = parent.customer_id WHERE parent.customer_id IS NULL","why":"Referenced model dim_customers not yet built in warehouse.","sample_failures":null} +{"audit_schema_version":1,"signalforge_version":"0.1.0.dev0","record_id":"d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9","timestamp":"2026-04-30T12:00:03.000000Z","config_hash":"abc123def456789a","model_unique_id":"model.sf_demo.fct_orders","test":{"type":"accepted_values","column":"status","values":["pending","shipped","cancelled"],"rationale":"Order status enum."},"test_anchor":"column.status","decision":"dropped","reason":"failed-on-known-clean-data","failures":27,"sampled_rows":10000,"scope":"sample","elapsed_ms":96,"compiled_sql_hash":"5555666677778888","compiled_sql":"SELECT status FROM `p.d.fct_orders` WHERE status NOT IN ('pending','shipped','cancelled')","why":"27 rows in known-clean sample violate accepted_values list.","sample_failures":[{"status":"returned"},{"status":"refunded"}]} +{"audit_schema_version":1,"signalforge_version":"0.1.0.dev0","record_id":"e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0","timestamp":"2026-04-30T12:00:04.000000Z","config_hash":"abc123def456789a","model_unique_id":"model.sf_demo.dim_customers","test":{"type":"not_null","column":"email","rationale":"Customer email required at signup."},"test_anchor":"column.email","decision":"kept","reason":"kept-without-evidence","failures":0,"sampled_rows":12,"scope":"sample","elapsed_ms":54,"compiled_sql_hash":"9999aaaabbbbcccc","compiled_sql":"SELECT COUNT(*) AS failures FROM `p.d.dim_customers` WHERE email IS NULL","why":"Sample too small (12 rows) to support a verdict.","sample_failures":null} \ No newline at end of file diff --git a/tests/fixtures/prune/prune_result_v1.json b/tests/fixtures/prune/prune_result_v1.json new file mode 100644 index 00000000..eed58279 --- /dev/null +++ b/tests/fixtures/prune/prune_result_v1.json @@ -0,0 +1,68 @@ +{ + "prune_schema_version": 1, + "model_unique_id": "model.sf_demo.fct_orders", + "decisions": [ + { + "test_anchor": "column.id", + "test": { + "type": "not_null", + "column": "id", + "rationale": "Primary key surrogate; must never be null." + }, + "decision": "dropped", + "reason": "always-passes", + "failures": 0, + "sampled_rows": 10000, + "scope": "sample", + "elapsed_ms": 87, + "compiled_sql_hash": "0123456789abcdef", + "compiled_sql": "SELECT COUNT(*) AS failures FROM `proj.dataset.fct_orders` WHERE id IS NULL", + "why": "0 failures across 10000 sampled rows; always-pass tests consume reviewer attention.", + "sample_failures": null + }, + { + "test_anchor": "column.order_id", + "test": { + "type": "unique", + "column": "order_id", + "rationale": "Surrogate primary key for the orders fact." + }, + "decision": "kept", + "reason": "kept", + "failures": 3, + "sampled_rows": 10000, + "scope": "sample", + "elapsed_ms": 142, + "compiled_sql_hash": "fedcba9876543210", + "compiled_sql": "SELECT order_id FROM `proj.dataset.fct_orders` GROUP BY order_id HAVING COUNT(*) > 1", + "why": "3 duplicate order_id values across 10000 sampled rows; real signal worth shipping.", + "sample_failures": [ + {"order_id": "ORD-1001"}, + {"order_id": "ORD-1002"} + ] + }, + { + "test_anchor": "column.status", + "test": { + "type": "accepted_values", + "column": "status", + "values": ["pending", "shipped", "cancelled"], + "rationale": "Documented order status enum." + }, + "decision": "dropped", + "reason": "failed-on-known-clean-data", + "failures": 27, + "sampled_rows": 10000, + "scope": "sample", + "elapsed_ms": 96, + "compiled_sql_hash": "5555666677778888", + "compiled_sql": "SELECT status FROM `proj.dataset.fct_orders` WHERE status NOT IN ('pending', 'shipped', 'cancelled')", + "why": "27 rows in known-clean sample violate the accepted_values list; LLM hallucinated the enum.", + "sample_failures": [ + {"status": "returned"} + ] + } + ], + "elapsed_ms": 715, + "signalforge_version": "0.1.0.dev0" +} diff --git a/tests/prune/test_drift_detector.py b/tests/prune/test_drift_detector.py new file mode 100644 index 00000000..adf5c99a --- /dev/null +++ b/tests/prune/test_drift_detector.py @@ -0,0 +1,264 @@ +"""Schema-drift detection for the prune layer (US-010, DEC-010). + +Pairs production ``extra="ignore"`` models with ``extra="forbid"`` Strict +mirrors validated against committed JSON / JSONL fixtures. Adding a field +to a production model without updating the strict mirror OR the fixture +breaks the test loudly. + +Mirrors :mod:`tests.safety.test_drift_detector` shape exactly. The three +prune-layer read-back models covered here are: + +* :class:`signalforge.prune.models.PruneDecision` +* :class:`signalforge.prune.models.PruneResult` +* :class:`signalforge.prune.audit.PruneEvent` + +:class:`signalforge.prune.config.PruneConfig` is already ``extra="forbid"`` +in production (DEC-015), so no drift gate is needed there. +:class:`signalforge.draft.models.CandidateTest` is the draft layer's +responsibility and is covered by :mod:`tests.draft` — this module reuses +the discriminated union as-is for the ``test:`` field on each event. + +Reference: ``.claude/rules/manifest-readers.md`` (DEC-008 — drift detectors +mandatory for ``extra="ignore"`` reader-shaped models), +``.claude/rules/safety-layer.md`` (DEC-014 / DEC-015 — pair every read-back +model with a one-off ``extra="forbid"`` mirror). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Literal + +import pytest +from pydantic import BaseModel, ConfigDict, ValidationError + +from signalforge.draft.models import CandidateTest +from signalforge.prune.audit import PruneEvent +from signalforge.prune.models import DropReason, PruneDecision, PruneResult, Scope + +_STRICT = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) +_FIXTURES_DIR = Path(__file__).resolve().parent.parent / "fixtures" / "prune" + + +class StrictPruneDecision(BaseModel): + """One-off ``extra="forbid"`` mirror of :class:`PruneDecision` (DEC-010). + + If you add a field to :class:`PruneDecision`, you MUST: + + 1. Add it here, and + 2. Update :file:`tests/fixtures/prune/prune_decision_v1.json` (and the + ``decisions`` arrays in the result / event fixtures if appropriate). + + The field-set parity test below catches additions that arrive via one + side but not the other. + """ + + model_config = _STRICT + + test_anchor: str + test: CandidateTest + decision: Literal["kept", "dropped"] + reason: DropReason + failures: int + sampled_rows: int | None + scope: Scope + elapsed_ms: int + compiled_sql_hash: str + compiled_sql: str + why: str + sample_failures: tuple[dict[str, Any], ...] | None = None + + +class StrictPruneResult(BaseModel): + """One-off ``extra="forbid"`` mirror of :class:`PruneResult` (DEC-010). + + Note: production :class:`PruneResult` exposes ``kept_decisions``, + ``dropped_decisions``, ``kept_count``, ``dropped_count``, and + ``total_tests`` as :func:`pydantic.computed_field` properties (DEC-003). + These live in ``model_computed_fields``, NOT in ``model_fields``, so + the field-set parity test does not need to filter them out — it only + compares the stored-field set, which is what drift detection cares + about. + """ + + model_config = _STRICT + + prune_schema_version: Literal[1] = 1 + model_unique_id: str + decisions: tuple[StrictPruneDecision, ...] + elapsed_ms: int + signalforge_version: str + + +class StrictPruneEvent(BaseModel): + """One-off ``extra="forbid"`` mirror of :class:`PruneEvent` (DEC-010). + + Mirrors the flat shape (DEC-014 of llm-drafter.md / DEC-018 of + prune-engine plan): the decision's fields are flattened in rather than + nested under a ``decision:`` key, so a reviewer can ``jq`` over the + JSONL without descending one level per field. + """ + + model_config = _STRICT + + audit_schema_version: Literal[1] = 1 + signalforge_version: str + record_id: str + timestamp: str + config_hash: str + model_unique_id: str + test: CandidateTest + test_anchor: str + decision: Literal["kept", "dropped"] + reason: DropReason + failures: int + sampled_rows: int | None + scope: Scope + elapsed_ms: int + compiled_sql_hash: str + compiled_sql: str + why: str + sample_failures: tuple[dict[str, Any], ...] | None = None + + +# --- Fixture validation ---------------------------------------------------- + + +def test_strict_prune_decision_validates_fixture() -> None: + """Each entry in ``prune_decision_v1.json`` validates against + :class:`StrictPruneDecision` (``extra="forbid"``). + + If this raises, an unknown field was introduced in the fixture without + being mirrored on :class:`StrictPruneDecision` (or vice versa). Update + production :class:`PruneDecision`, :class:`StrictPruneDecision`, and + the fixture together. + """ + fixture_path = _FIXTURES_DIR / "prune_decision_v1.json" + payload = json.loads(fixture_path.read_text(encoding="utf-8")) + assert isinstance(payload, list) and payload, ( + f"expected a non-empty JSON array at {fixture_path}" + ) + for entry in payload: + StrictPruneDecision.model_validate(entry) + + +def test_strict_prune_result_validates_fixture() -> None: + """The :file:`prune_result_v1.json` fixture validates against + :class:`StrictPruneResult`. + """ + fixture_path = _FIXTURES_DIR / "prune_result_v1.json" + payload = json.loads(fixture_path.read_text(encoding="utf-8")) + StrictPruneResult.model_validate(payload) + + +def test_strict_prune_event_validates_jsonl_fixture() -> None: + """Each line of :file:`prune_event_v1.jsonl` validates against + :class:`StrictPruneEvent`. Covers all five :data:`DropReason` values. + """ + fixture_path = _FIXTURES_DIR / "prune_event_v1.jsonl" + text = fixture_path.read_text(encoding="utf-8") + lines = [line for line in text.splitlines() if line.strip()] + assert lines, f"expected one-or-more JSONL lines in {fixture_path}" + + seen_reasons: set[str] = set() + for line in lines: + event = StrictPruneEvent.model_validate_json(line) + seen_reasons.add(event.reason) + + # All five DropReason values exercised in the fixture. + expected_reasons = { + "always-passes", + "requires-future-data", + "failed-on-known-clean-data", + "kept", + "kept-without-evidence", + } + assert seen_reasons == expected_reasons, ( + f"prune_event_v1.jsonl must cover every DropReason; " + f"missing: {expected_reasons - seen_reasons}, " + f"unexpected: {seen_reasons - expected_reasons}" + ) + + +# --- Field-set parity ------------------------------------------------------ + + +def test_prune_decision_field_set_parity() -> None: + """:class:`StrictPruneDecision` model_fields exactly match + :class:`PruneDecision` model_fields. Adding a field to one without + the other breaks this test loudly. + """ + strict_fields = set(StrictPruneDecision.model_fields.keys()) + prod_fields = set(PruneDecision.model_fields.keys()) + missing_in_strict = prod_fields - strict_fields + extra_in_strict = strict_fields - prod_fields + assert not missing_in_strict, ( + f"StrictPruneDecision is missing fields present in PruneDecision: " + f"{missing_in_strict}. Update StrictPruneDecision to match." + ) + assert not extra_in_strict, ( + f"StrictPruneDecision has fields absent from PruneDecision: " + f"{extra_in_strict}. Remove from StrictPruneDecision or add to " + f"PruneDecision." + ) + + +def test_prune_result_field_set_parity() -> None: + """:class:`StrictPruneResult` model_fields exactly match + :class:`PruneResult` model_fields. + + Note: ``kept_decisions`` / ``dropped_decisions`` / ``kept_count`` / + ``dropped_count`` / ``total_tests`` are :func:`pydantic.computed_field` + properties on production :class:`PruneResult` (DEC-003) — they live + in ``model_computed_fields``, NOT in ``model_fields``, so the parity + check focuses on stored fields only. + """ + strict_fields = set(StrictPruneResult.model_fields.keys()) + prod_fields = set(PruneResult.model_fields.keys()) + missing_in_strict = prod_fields - strict_fields + extra_in_strict = strict_fields - prod_fields + assert not missing_in_strict, ( + f"StrictPruneResult is missing fields present in PruneResult: " + f"{missing_in_strict}. Update StrictPruneResult to match." + ) + assert not extra_in_strict, ( + f"StrictPruneResult has fields absent from PruneResult: " + f"{extra_in_strict}. Remove from StrictPruneResult or add to " + f"PruneResult." + ) + + +def test_prune_event_field_set_parity() -> None: + """:class:`StrictPruneEvent` model_fields exactly match + :class:`PruneEvent` model_fields. + """ + strict_fields = set(StrictPruneEvent.model_fields.keys()) + prod_fields = set(PruneEvent.model_fields.keys()) + missing_in_strict = prod_fields - strict_fields + extra_in_strict = strict_fields - prod_fields + assert not missing_in_strict, ( + f"StrictPruneEvent is missing fields present in PruneEvent: " + f"{missing_in_strict}. Update StrictPruneEvent to match." + ) + assert not extra_in_strict, ( + f"StrictPruneEvent has fields absent from PruneEvent: " + f"{extra_in_strict}. Remove from StrictPruneEvent or add to " + f"PruneEvent." + ) + + +# --- Sanity floor: extra="forbid" actually fires -------------------------- + + +def test_strict_prune_event_rejects_unknown_field() -> None: + """Sanity floor: a fixture line with an extra unknown field raises + :class:`ValidationError`. Confirms ``extra="forbid"`` is wired up — a + silently-accepted unknown field would defeat the entire drift gate. + """ + fixture_path = _FIXTURES_DIR / "prune_event_v1.jsonl" + first_line = fixture_path.read_text(encoding="utf-8").splitlines()[0] + payload = json.loads(first_line) + payload["future_field_that_should_not_exist"] = "boom" + with pytest.raises(ValidationError): + StrictPruneEvent.model_validate(payload) From 801b5be7135936a72350f48e7ad02b5771062184 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 22:56:43 -0700 Subject: [PATCH 17/24] bd_1-scaffolding-y8y.3: diagnostic cost probe + ops-doc scaffold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tests/warehouse/test_sample_cost_probe.py — measures total_bytes_billed for a deterministic 100k-row sample against bigquery-public-data.iowa_liquor_sales.sales. Marked @pytest.mark.bigquery + skipif(SF_RUN_BQ) so default CI excludes it. Used to verify or refute Phase-1's "sample-mode is cheap" assumption (AR-B1) — TO_JSON_STRING(t) reads the whole row. Pre-creates docs/prune-ops.md with the standard ops-doc skeleton: public-API placeholder, drop-reason taxonomy, cost-model section (awaits the probe's figure), v0.2 deferrals. US-014 fills the body. Plan: plans/super/6-prune-engine.md (DEC-012, AR-B1). Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/prune-ops.md | 204 +++++++++++++++++ tests/warehouse/test_sample_cost_probe.py | 261 ++++++++++++++++++++++ 2 files changed, 465 insertions(+) create mode 100644 docs/prune-ops.md create mode 100644 tests/warehouse/test_sample_cost_probe.py diff --git a/docs/prune-ops.md b/docs/prune-ops.md new file mode 100644 index 00000000..8b2bcf0d --- /dev/null +++ b/docs/prune-ops.md @@ -0,0 +1,204 @@ +# Prune layer — operations guide + +`signalforge.prune` is the test-pruning stage of the v0.1 pipeline: +runs every candidate test against warehouse data, drops the +always-pass and known-clean-fail ones, returns a typed `PruneResult`. +This is the load-bearing differentiator (Architectural Commitment #1 +in [`CLAUDE.md`](../CLAUDE.md)) — competitors generate; SignalForge +generates *and* grades. + +Companion to [`docs/safety-ops.md`](safety-ops.md), +[`docs/draft-ops.md`](draft-ops.md), +[`docs/manifest-loader-ops.md`](manifest-loader-ops.md), and +[`docs/warehouse-adapter-ops.md`](warehouse-adapter-ops.md). Internal +design lives in +[`plans/super/6-prune-engine.md`](../plans/super/6-prune-engine.md). + +This document is an **initial scaffold**. US-014 fills the body once +the implementation stories (US-001 through US-013) land. The sections +below are placeholders with TBDs marked explicitly so the gaps are +visible to the reviewer of the next PR. + +## Public API + +US-014 will populate this section. The expected surface (per the +Phase-1 plan, subject to amendment as stories land): + +- `signalforge.prune.prune_tests(candidates, model, manifest, adapter, *, config) -> PruneResult` +- `PruneResult`, `PruneDecision`, `KeptTest`, `DroppedTest` +- `PruneConfig`, `load_prune_config` +- `DropReason`, `Scope` literals +- The `PruneError` hierarchy + +(TBD — final shapes confirmed during US-014.) + +## Configuration: `signalforge.yml` `prune:` block + +Top-level namespace is `prune:` (DEC-025 of the safety layer reserves +sibling keys for future stages; the prune layer claims its own). + +US-014 will populate this section with the field-by-field reference. +Expected fields (per the Phase-1 plan): + +- `scope: "sample" | "full"` (default `sample`) +- `sample_size: int` (default `100_000`) +- `test_timeout_seconds: int` (default `30`) +- `total_budget_seconds: int` (default `600`) +- `capture_failure_rows: int` (default `3`) +- `trusted_models: list[str]` — drives the `failed-on-known-clean-data` + drop reason (deferred for v0.1 unless the implementing story + re-opens the question). + +(TBD — final fields and defaults confirmed during US-014.) + +## Drop-reason taxonomy + +Every kept and dropped test ships with a structured `PruneDecision` +carrying a one-line `why`. The reasons are a closed set so the diff +renderer (#8) can branch on the literal value rather than sniffing +prose. + +| Reason | Decision | Why | +|--------|----------|-----| +| `always-passes` | dropped | Zero failing rows on the sampled or full set; no signal worth shipping. | +| `requires-future-data` | dropped | A `relationships` test references a manifest-absent parent model. No warehouse call issued. | +| `failed-on-known-clean-data` | dropped | Test failed AND the model is in `prune.trusted_models`; the test is presumed buggy. | +| `kept` | kept | Test failed on an untrusted model; reviewer should evaluate. | +| `kept-without-evidence` | kept | Could not evaluate (warehouse error or total budget exceeded); ship conservatively. | + +Conservative bias: when in doubt, keep. Architectural Commitment #1 +("signal over volume") penalises always-pass tests (no signal, +consumes reviewer attention) but does not penalise kept tests with +ambiguous evidence — those land in front of a human reviewer who can +make the final call. + +## Cost model (US-003 verification) + +The deterministic-sample predicate + +```sql +WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), bucket) < 1 +``` + +serialises the entire row into the predicate. BigQuery cannot +column-prune through a function argument, so sample-mode reads **all +columns** of the table, not just the column under test. The +Phase-1 estimate that pruning 30 candidate tests against a 100k-row +sample would cost approximately 24 MB / approximately one tenth of one +US cent assumed only the column under test would be read; the worst +case is 50–500x that figure. + +**Verified figure (US-003):** TBD — populate after the diagnostic +probe (`tests/warehouse/test_sample_cost_probe.py`) runs against +`bigquery-public-data.iowa_liquor_sales.sales`. The probe records +`total_bytes_billed` for a 100k-row deterministic sample. The +threshold for action: + +- If `bytes_billed < 500 MB`: Q4=A (one-query-per-test) is viable for + v0.1 unchanged. +- If `bytes_billed >= 500 MB`: v0.2 should adopt Q4=C + (temp-table-materialised sample) so the per-test cost amortises + across the candidate set rather than re-paying the full-row scan + for every test. + +(US-014 replaces TBD with the live figure once US-003 has been run by +a maintainer with BigQuery credentials.) + +## Audit JSONL + +US-014 will populate this section if the implementing story chooses +to emit a fail-closed `PruneEvent` JSONL (mirroring the safety +audit's DEC-011 contract and the LLM response audit's DEC-006 / DEC-008 +contracts). The default expectation is that prune **does** emit one +JSONL line per `PruneDecision` so the explainable-diffs commitment +(Architectural Commitment #5) survives the v0.2 transition to +multi-warehouse adapters where the underlying SQL is no longer +homogeneous. + +(TBD — confirmed during US-014.) + +## Running real-warehouse tests + +The prune layer's integration tests share the warehouse adapter's +gating discipline. Default CI excludes them via the +`-m 'not bigquery'` filter; opt-in requires both the marker and an +`SF_RUN_BQ=1` env var. + +```bash +gcloud auth application-default login +SF_RUN_BQ=1 pytest -m bigquery +``` + +The diagnostic cost probe (US-003) lives at +`tests/warehouse/test_sample_cost_probe.py` and runs under the same +gate. It is a documentation-grade probe — a soft WARNING fires at +500 MB; the test fails only above the 5 GB sanity ceiling. + +(US-014 expands this section with the prune-specific integration +tests once US-013 lands.) + +## Debugging + +US-014 will populate this section with logger names, levels, and the +ANSI-safe lazy-format JSON discipline that the safety / drafter layers +already follow (DEC-022 of safety; DEC-011 of llm-drafter). + +(TBD — confirmed during US-014.) + +## Typed-error reference + +US-014 will populate the table once US-005 lands the `PruneError` +hierarchy. Mirrors the safety / draft / warehouse error tables. + +(TBD — confirmed during US-014.) + +## v0.2 deferrals + +The prune layer is intentionally narrow in v0.1. The following +concerns are explicitly deferred: + +- **Per-decision `bytes_billed` recording (DEC-027 of issue #6).** + The adapter does not surface job stats in v0.1; the diagnostic + probe (US-003) reads them via `INFORMATION_SCHEMA.JOBS_BY_USER` + out-of-band rather than through the adapter API. v0.2 extends the + adapter's seam to return job stats so the `PruneDecision` can + carry the figure natively. +- **Per-test `timeout_ms` threading.** US-002 plumbed + `timeout_ms` into `make_query_job_config` (DEC-013 of issue #6, + AR-B2); surfacing it through `WarehouseAdapter.run_test_sql`'s + public signature is a v0.2 task. v0.1 prune sets the timeout on + the job config but cannot enforce it per-call without the + surfaced parameter. +- **Test batching — Q4=B / Q4=C optimisations.** The Phase-1 plan + catalogues two cost optimisations (per-column `COUNTIF` batching; + temp-table-materialised sample) that v0.1 does not adopt. US-003 + produces the data needed to evaluate the temp-table option in + v0.2. +- **Multi-warehouse adapters.** Snowflake, Postgres, Databricks, + Redshift adapters slot in behind `WarehouseAdapter` without prune + changes once their adapters land. The prune compiler already uses + `Dialect.quote_char` and `Dialect.identifier_case` to stay + warehouse-agnostic; no SQL paths are BigQuery-specific. +- **Historical always-pass evidence.** Running candidate tests + against multiple `run_results.json` snapshots to assert "never + failed in last N runs." The Phase-1 plan considers this for the + `failed-on-known-clean-data` evidence channel and defers to v0.2. +- **dbt-utils test types.** `dbt_utils.unique_combination_of_columns`, + `dbt_utils.accepted_range`, `dbt_utils.expression_is_true`, etc. + The drafter's `CandidateTest` union has exactly four variants; + prune compiles exactly four. v0.2 territory. +- **`where:` test modifier and `severity: warn` / `mostly:`.** + dbt-core supports a `where:` predicate on every test plus a + `severity` and `mostly` knob; v0.1 prune does not consume any of + these. +- **`prune_decision_id`-keyed checkpoint / resumption.** Long-running + prune runs that resume from disk after a crash. v0.2. +- **Confidence intervals on `always-passes`.** Surfacing + "less than or equal to 3/N upper-bound failure rate at 95 percent + confidence" on the decision record. Nice to have; v0.2. +- **LLM-generated rationale on `kept` decisions.** The grader (#7) + produces rubric-scored rationale; prune writes only the structured + drop reason plus failure count plus scope. + +(US-014 expands the v0.2 list as additional deferrals surface during +implementation.) diff --git a/tests/warehouse/test_sample_cost_probe.py b/tests/warehouse/test_sample_cost_probe.py new file mode 100644 index 00000000..7fbe9a67 --- /dev/null +++ b/tests/warehouse/test_sample_cost_probe.py @@ -0,0 +1,261 @@ +"""AR-B1 cost-model probe: measure ``total_bytes_billed`` from a +deterministic-sample run against +``bigquery-public-data.iowa_liquor_sales.sales``. + +The Phase-1 cost model (issue #6, plan ``plans/super/6-prune-engine.md``) +assumed sample-mode is cheap because the failing-rows SQL only reads one +column. But the deterministic sampler uses + + WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), bucket) < 1 + +which serialises the **whole row** into the predicate — BigQuery cannot +column-prune through a function argument. AR-B1 records the worry that +sample-mode could be 50–500x more expensive than the original "1 column +x 100k rows" estimate. This probe records the live figure so the +reviewer can confirm or refute that worry against a real public table. + +Gating (mirrors ``tests/warehouse/test_bigquery_integration.py``): + +* ``@pytest.mark.bigquery`` — the default ``addopts = "-m 'not bigquery + and not anthropic'"`` filter excludes this test from the bare + ``pytest`` run, so default CI never touches BigQuery. +* ``@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), ...)`` — even + ``pytest -m bigquery`` skips at runtime unless the maintainer has + opted in by exporting ``SF_RUN_BQ=1``. + +To run locally:: + + gcloud auth application-default login + SF_RUN_BQ=1 pytest -m bigquery tests/warehouse/test_sample_cost_probe.py + +The test is **documentation-grade**, not regression-grade: the assertion +is a sanity ceiling (5 GB) rather than a tight budget, and a soft +WARNING fires at 500 MB so the figure shows up in the test log even on +a successful run. US-014 (``docs/prune-ops.md``) folds the recorded +figure into the cost-model section. + +If the post-call ``INFORMATION_SCHEMA.JOBS_BY_USER`` lookup fails (IAM, +region availability, eventual-consistency lag, etc.), the test marks +itself ``xfail`` with the documented reason rather than failing the +suite — exposing ``total_bytes_billed`` cleanly via the adapter is a +v0.2 concern (DEC-027 of issue #6, ``plans/super/6-prune-engine.md``) +and out of scope for this story. +""" + +from __future__ import annotations + +import logging +import os +import time +from typing import Any + +import pytest + +from signalforge.warehouse.adapters.bigquery import BigQueryAdapter +from signalforge.warehouse.models import TableRef + +# --------------------------------------------------------------------------- +# Probe target + thresholds. +# --------------------------------------------------------------------------- + +_TARGET = TableRef( + project="bigquery-public-data", + dataset="iowa_liquor_sales", + name="sales", +) +"""Iowa liquor sales: ~30M rows, ~24 columns. Public; covered by the +BigQuery 1 TB/month free tier for typical maintainer use.""" + +_SAMPLE_SIZE = 100_000 +"""Matches the ``PruneConfig.sample_size`` default in the Phase-1 plan +so the figure the probe records is directly comparable to what prune +will spend per test in v0.1.""" + +_BYTES_CEILING = 5_000_000_000 +"""5 GB sanity ceiling. Not a budget — a runaway-cost circuit-breaker. +The Iowa liquor sales table is roughly 4 GB on disk; the deterministic +sampler reading every column should not exceed this even with full-row +scan amplification.""" + +_BYTES_WARN_AT = 500_000_000 +"""500 MB soft threshold. Crossing this is consistent with the AR-B1 +worry that ``TO_JSON_STRING(t)`` triggers a full-row scan rather than +a column-pruned read. Emitting a WARNING (rather than failing) keeps +the test documentation-grade — the figure surfaces in the log so the +reviewer can decide whether v0.2 should escalate to Q4=C +(temp-table-materialised sample).""" + +_LOGGER = logging.getLogger("signalforge.warehouse.test.cost_probe") + + +# --------------------------------------------------------------------------- +# Helpers. +# --------------------------------------------------------------------------- + + +def _adapter_billing_project(adapter: BigQueryAdapter) -> str: + """Return the billing project the adapter resolves to. + + ``BigQueryAdapter`` does not expose its underlying client publicly; + we reach through the private ``_get_client`` to read ``project`` + because the probe needs a region-qualified + ``INFORMATION_SCHEMA.JOBS_BY_USER`` lookup. This is test-only access + — production code routes through the adapter API. + """ + # Accessing a private member is intentional: the probe is a + # diagnostic that lives outside the adapter's public contract. + client = adapter._get_client() # noqa: SLF001 + return str(client.project) + + +def _lookup_total_bytes_billed( + adapter: BigQueryAdapter, + *, + started_after_epoch_s: float, +) -> int | None: + """Query ``INFORMATION_SCHEMA.JOBS_BY_USER`` for the most recent + ``warehouse_sample`` job and return its ``total_bytes_billed``. + + Returns ``None`` if no matching job is found (eventual-consistency + lag) or if the lookup itself fails for any reason — the caller then + ``xfail``s with the documented reason. We deliberately do NOT + propagate the exception: a probe that hard-fails on environmental + quirks (regional dataset availability, missing + ``bigquery.jobs.list`` permission, etc.) is worse than no probe. + + The query filters on the ``signalforge_stage`` label that + ``BigQueryAdapter._default_job_config`` stamps on every job + (DEC-015 of issue #3). Filtering by label scopes the result to the + sample we just issued without needing the job-id round-trip. + """ + project = _adapter_billing_project(adapter) + # ``INFORMATION_SCHEMA.JOBS_BY_USER`` is region-qualified. We + # default to US (the multi-region the public ``bigquery-public-data`` + # tables live in) when the adapter has no explicit location. + region = adapter._location or "US" # noqa: SLF001 + region_qualifier = f"region-{region.lower()}" + + # ``creation_time`` is a TIMESTAMP; the probe records monotonic + # epoch seconds and we compare against TIMESTAMP_SECONDS at lookup. + sql = ( + f"SELECT total_bytes_billed " + f"FROM `{project}.{region_qualifier}.INFORMATION_SCHEMA.JOBS_BY_USER` " + f"WHERE creation_time >= TIMESTAMP_SECONDS(@since) " + f"AND EXISTS (SELECT 1 FROM UNNEST(labels) AS l " + f"WHERE l.key = 'signalforge_stage' AND l.value = 'warehouse_sample') " + f"ORDER BY creation_time DESC LIMIT 1" + ) + + try: + # Build an ad-hoc query-parameterised job. We bypass the + # adapter's ``run_test_sql`` because the probe SQL is not a + # failing-rows test — it's a metadata read. + from google.cloud import bigquery # type: ignore[import-not-found] + + client = adapter._get_client() # noqa: SLF001 + job_config = bigquery.QueryJobConfig( + query_parameters=[ + bigquery.ScalarQueryParameter("since", "INT64", int(started_after_epoch_s)), + ], + use_query_cache=False, + ) + rows = list(client.query(sql, job_config=job_config).result()) + except Exception as exc: # pragma: no cover - environment-dependent + _LOGGER.info( + "INFORMATION_SCHEMA lookup failed; the probe will xfail. error_class=%s", + type(exc).__name__, + ) + return None + + if not rows: + return None + bytes_billed = getattr(rows[0], "total_bytes_billed", None) + if bytes_billed is None: + # The row exists but the column is null — for a non-dry-run + # query this should not happen, but treat as "unknown" rather + # than zero so the xfail path engages cleanly. + return None + return int(bytes_billed) + + +# --------------------------------------------------------------------------- +# The probe. +# --------------------------------------------------------------------------- + + +@pytest.mark.bigquery +@pytest.mark.skipif( + not os.environ.get("SF_RUN_BQ"), + reason="set SF_RUN_BQ=1 to run BigQuery integration probes", +) +def test_sample_rows_bytes_billed_within_budget( + caplog: pytest.LogCaptureFixture, +) -> None: + """Measure ``total_bytes_billed`` for a deterministic 100k-row + sample of ``bigquery-public-data.iowa_liquor_sales.sales``. + + Asserts: + + * ``bytes_billed < 5 GB`` (sanity ceiling, not a budget). + * If ``bytes_billed >= 500 MB``, a WARNING fires carrying the figure + so the run log records it for ``docs/prune-ops.md``. + + If the post-call ``INFORMATION_SCHEMA`` lookup yields no row + (eventual-consistency lag) or fails for any other reason, the test + marks itself ``xfail`` with the documented reason. The probe is + documentation-grade: a missing figure is acceptable; a wrong figure + is not. + """ + adapter = BigQueryAdapter() + + started_at_epoch_s = time.time() + started_at_monotonic = time.monotonic() + with adapter: + rows: list[dict[str, Any]] = adapter.sample_rows(_TARGET, _SAMPLE_SIZE) + elapsed_s = time.monotonic() - started_at_monotonic + + # Belt-and-suspenders: the sampler must actually return rows. If + # this fails the cost figure is meaningless. + assert rows, "expected at least one sampled row from a 30M-row table" + assert len(rows) <= _SAMPLE_SIZE + + bytes_billed = _lookup_total_bytes_billed(adapter, started_after_epoch_s=started_at_epoch_s) + if bytes_billed is None: + pytest.xfail( + "bytes-billed retrieval via INFORMATION_SCHEMA.JOBS_BY_USER did " + "not return a matching row (IAM, regional availability, or " + "eventual-consistency lag). Adapter-side bytes_billed exposure " + "is a v0.2 concern (DEC-027 of issue #6)." + ) + + _LOGGER.info( + "sample_rows cost probe: total_bytes_billed=%d sample_size=%d elapsed_s=%.2f table=%s", + bytes_billed, + _SAMPLE_SIZE, + elapsed_s, + _TARGET.qualified_name, + ) + + if bytes_billed >= _BYTES_WARN_AT: + with caplog.at_level(logging.WARNING, logger=_LOGGER.name): + _LOGGER.warning( + "sample_rows bytes_billed exceeds soft threshold: " + "bytes_billed=%d threshold=%d table=%s. " + "Consistent with AR-B1 (TO_JSON_STRING(t) triggers full-row " + "scan); v0.2 should evaluate Q4=C (temp-table-materialised " + "sample).", + bytes_billed, + _BYTES_WARN_AT, + _TARGET.qualified_name, + ) + assert any( + record.levelno == logging.WARNING + and "bytes_billed exceeds soft threshold" in record.getMessage() + for record in caplog.records + ), "expected the WARNING to be captured by caplog" + + assert bytes_billed < _BYTES_CEILING, ( + f"sample_rows bytes_billed={bytes_billed} exceeds sanity ceiling " + f"{_BYTES_CEILING}; investigate before relying on the v0.1 " + f"one-query-per-test prune cost model." + ) From b39d811e3311652d3edac986292eb9c33ea7c867 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 23:00:53 -0700 Subject: [PATCH 18/24] bd_1-scaffolding-y8y.12: BQ integration test against bigquery-public-data Single end-to-end test runs prune_tests against the public Iowa liquor sales dataset with two candidate tests: * not_null(invoice_and_item_number) -> always-passes drop * accepted_values(category_name, values=("VODKA", "GIN")) -> kept Confirms the orchestrator's decision matrix in real warehouse conditions. Marker @pytest.mark.bigquery + skipif(SF_RUN_BQ) keeps it out of default CI; runs only when SF_RUN_BQ=1 and ADC are configured (mirrors tests/warehouse/test_bigquery_integration.py). Plan: plans/super/6-prune-engine.md (acceptance criterion #7, AR-C11). Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/prune/test_integration_bigquery.py | 176 +++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 tests/prune/test_integration_bigquery.py diff --git a/tests/prune/test_integration_bigquery.py b/tests/prune/test_integration_bigquery.py new file mode 100644 index 00000000..8142ee56 --- /dev/null +++ b/tests/prune/test_integration_bigquery.py @@ -0,0 +1,176 @@ +"""End-to-end prune integration test against ``bigquery-public-data``. + +US-012 / ticket #6 acceptance criterion #7. Confirms the orchestrator +produces correct decisions when wired up to a real warehouse against the +public ``bigquery-public-data.iowa_liquor_sales.sales`` table: + + * ``not_null(invoice_and_item_number)`` — non-null in the source + table, so this is the canonical "always-passes" case → dropped. + * ``accepted_values(category_name, values=("VODKA", "GIN"))`` — + the column has many more values than these two, so this is the + canonical "real failure" case → kept. + +Belt-and-suspenders gating (mirrors +``tests/warehouse/test_bigquery_integration.py`` DEC-011): + +* ``@pytest.mark.bigquery`` — filtered out by the default + ``addopts = "-m 'not bigquery and not anthropic'"`` so a bare + ``pytest`` never collects this test, and +* ``@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), ...)`` — even + ``pytest -m bigquery`` skips at runtime unless the maintainer has + opted in by exporting ``SF_RUN_BQ=1`` and configured Application + Default Credentials. + +To run locally:: + + gcloud auth application-default login + SF_RUN_BQ=1 pytest -m bigquery tests/prune/test_integration_bigquery.py + +The audit JSONL is written to ``tmp_path/.signalforge/prune.jsonl`` so +the integration test does not pollute the developer's working directory. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from signalforge.draft.models import ( + CandidateColumn, + CandidateSchema, + CandidateTestAcceptedValues, + CandidateTestNotNull, +) +from signalforge.manifest.models import Column, Manifest, Model +from signalforge.prune import prune_tests +from signalforge.prune.config import PruneConfig +from signalforge.warehouse.adapters.bigquery import BigQueryAdapter + +_SF_RUN_BQ_REASON = "requires SF_RUN_BQ=1 and ADC" + + +@pytest.mark.bigquery +@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), reason=_SF_RUN_BQ_REASON) +def test_prune_iowa_liquor_sales(tmp_path: Path) -> None: + """End-to-end: two candidate tests against the public sales table. + + Test 1: ``not_null(invoice_and_item_number)`` — non-null in the source + table, so this is the canonical "always-passes" case → dropped. + + Test 2: ``accepted_values(category_name, values=("VODKA", "GIN"))`` — + the column has many more values than these two, so this is the + canonical "real failure" case → kept. + """ + # 1. Build an in-memory Manifest representing the public table. + target_model = Model( + unique_id="model.iowa.sales", + name="sales", + resource_type="model", + package_name="iowa", + original_file_path="models/sales.sql", + path="sales.sql", + database="bigquery-public-data", + schema="iowa_liquor_sales", # type: ignore[call-arg] + alias="sales", + raw_code="-- public dataset; raw_code unused by prune", + description="Iowa liquor sales (public BQ dataset)", + columns={ + "invoice_and_item_number": Column( + name="invoice_and_item_number", + data_type="STRING", + description="non-null transaction id", + ), + "category_name": Column( + name="category_name", + data_type="STRING", + description="liquor category — many values, not just two", + ), + }, + ) + manifest = Manifest( + metadata={"dbt_schema_version": "v12", "adapter_type": "bigquery"}, + nodes={target_model.unique_id: target_model}, + ) + + # 2. Build CandidateSchema with two tests. + candidates = CandidateSchema( + name="sales", + description="public iowa liquor sales", + columns=( + CandidateColumn( + name="invoice_and_item_number", + description="should never be null", + tests=(CandidateTestNotNull(column="invoice_and_item_number"),), + ), + CandidateColumn( + name="category_name", + description="closed enum check", + tests=( + CandidateTestAcceptedValues( + column="category_name", + # Intentionally narrow — the source table has + # dozens of category_name values, so this test + # is guaranteed to fail. + values=("VODKA", "GIN"), + ), + ), + ), + ), + ) + + # 3. Build a real BigQueryAdapter using ambient ADC (mirrors + # tests/warehouse/test_bigquery_integration.py — no explicit project, + # the BigQuery client picks the billing project from ADC). + adapter = BigQueryAdapter() + config = PruneConfig( + scope="sample", + sample_size=10_000, + test_timeout_seconds=60, + total_budget_seconds=300, + capture_failure_rows=3, + ) + + audit_path = tmp_path / ".signalforge" / "prune.jsonl" + + # 4. Run prune_tests end-to-end. + result = prune_tests( + target_model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + ) + + # 5. Assertions. + assert len(result.decisions) == 2 + not_null_decision = next( + d for d in result.decisions if d.test_anchor == "column.invoice_and_item_number" + ) + accepted_decision = next(d for d in result.decisions if d.test_anchor == "column.category_name") + + # not_null should pass on the public dataset (column is non-null) → + # dropped, always-passes. + assert not_null_decision.decision == "dropped", ( + f"expected always-passes drop; got {not_null_decision.decision!r} " + f"reason={not_null_decision.reason!r} failures={not_null_decision.failures}" + ) + assert not_null_decision.reason == "always-passes" + assert not_null_decision.failures == 0 + + # accepted_values should fail (the table has way more category_names + # than just VODKA / GIN) → kept, real failure. + assert accepted_decision.decision == "kept", ( + f"expected kept; got {accepted_decision.decision!r} " + f"reason={accepted_decision.reason!r} failures={accepted_decision.failures}" + ) + assert accepted_decision.reason == "kept" + assert accepted_decision.failures > 0 + + # Audit JSONL should exist and have exactly two lines (one per + # decision). + assert audit_path.exists() + audit_lines = audit_path.read_text(encoding="utf-8").splitlines() + assert len(audit_lines) == 2 From d882274bcc4f25900d9103db136e1e55f5357909 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 23:06:38 -0700 Subject: [PATCH 19/24] bd_1-scaffolding-y8y.14: docs/prune-ops.md + CLAUDE.md update Fill out the operational reference for the prune layer. Cover the public API surface, signalforge.yml prune: config block, drop-reason taxonomy, audit JSONL schema, cost-model verification path, real-warehouse test invocation, and v0.2 deferrals. Update CLAUDE.md's Repository status and Public API surface (v0.1) sections to list the prune layer alongside the manifest, warehouse, safety, and draft layers. Plan: plans/super/6-prune-engine.md (DEC-027, all v0.1 surfaces). Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 6 +- docs/prune-ops.md | 395 +++++++++++++++++++++++++++++----------------- 2 files changed, 255 insertions(+), 146 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index afe69d54..4489bf32 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,15 +4,16 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Repository status -Pre-alpha. Five issues shipped: +Pre-alpha. Six issues shipped: - **#1 (project scaffolding)** — `pyproject.toml` (Hatchling + src layout), `src/signalforge/__init__.py` with `__version__`, smoke test, ruff + pyright + pytest configs, GitHub Actions CI on PRs into `dev` and pushes to `main`, and `CONTRIBUTING.md`. - **#2 (manifest loader)** — `signalforge.manifest` subpackage: typed `Manifest` / `Model` (Pydantic v2), `load(project_dir, manifest_path=None) -> Manifest`, single-model resolver by `unique_id` or file path, schema-version tolerance v9–v12, symlink-hardened path canonicalisation, soft 200 MB warning. See `docs/manifest-loader-ops.md` for the operational reference. - **#3 (BigQuery warehouse adapter)** — `signalforge.warehouse` subpackage: `WarehouseAdapter` ABC + `from_profile` factory, `BigQueryAdapter` (the only v0.1 concrete adapter), `load_profile` for dbt `profiles.yml`, deterministic hash-mod sampling with fail-loud sizing checks, identifier-validation at construction time, `QueryJobConfig` defaults that pin `use_query_cache=False`. See `docs/warehouse-adapter-ops.md` for the operational reference. - **#4 (PII safety layer)** — `signalforge.safety` subpackage: schema-only-default sampling-mode policy, fail-closed audit JSONL writer (`O_APPEND` + `fsync` + size cap), column-name redaction via stable blake2b-4 hashes (DEC-010), four opt-out signals (column meta, model meta, `tags:[pii]`, `meta.contains_pii`) with documented precedence, AST audit-completeness scan rejecting direct `LLMRequest` construction, drift-detector test for `AuditEvent`. `signalforge.yml` config namespace `{ safety: { ... } }` reserved for the safety layer. See `docs/safety-ops.md` for the operational reference and `.claude/rules/safety-layer.md` for the rules distilled from this ticket. - **#5 (LLM draft pipeline)** — `signalforge.draft` + `signalforge.llm` subpackages: centralized `call_anthropic` SDK seam (full retry taxonomy via clauditor pattern), `draft_schema(model, adapter, policy, manifest, *, config)` end-to-end drafter, typed `CandidateSchema` + discriminated-union `CandidateTest`, anchor-contract validator (every test references a real column; whole-draft fail-loud), fail-closed `LLMResponseEvent` JSONL response audit adjacent to safety audit, `` prompt-injection envelope, deterministic `prompt_version` hash. See `docs/draft-ops.md`. +- **#6 (test prune engine)** — `signalforge.prune` subpackage: `prune_tests(model, adapter, candidates, manifest, *, config=None, audit_path=None) -> PruneResult` end-to-end orchestrator. Compiles dbt-style tests (`not_null`, `unique`, `accepted_values`, `relationships`) to BigQuery failing-rows SQL via `Dialect.quote_char`-driven dispatch (so v0.2 Snowflake/Postgres slot in unchanged), routes outcomes through five `DropReason` literals (`always-passes`, `requires-future-data`, `failed-on-known-clean-data`, `kept`, `kept-without-evidence`), enforces a total wall-clock budget (DEC-011), emits a fail-closed `prune.jsonl` audit (mirrors safety / draft fail-closed semantics; AST-gated single construction seam). Trusted-models opt-in via `signalforge.yml prune.trusted_models: [...]`, validated against the manifest at orchestrator entry (DEC-008). See `docs/prune-ops.md` for the operational reference and `plans/super/6-prune-engine.md` for the design. -Design is happening in the open on the `dev` branch; remaining feature work (prune logic #6, grader #7, diff renderer #8, CLI #9) lands next. +Design is happening in the open on the `dev` branch; remaining feature work (grader #7, diff renderer #8, CLI #9) lands next. ## Public API surface (v0.1) @@ -21,6 +22,7 @@ Design is happening in the open on the `dev` branch; remaining feature work (pru - `signalforge.safety.load_safety_config`, `SafetyPolicy`, `build_llm_request`, `aggregate_columns`, `redact_rows`, the typed shapes (`SamplingMode`, `RedactionReason`, `RedactionRecord`, `AuditEvent`, `LLMRequest`), and the `SafetyError` hierarchy (10 classes). Documented in `docs/safety-ops.md`. - `signalforge.llm.call_anthropic`, `LLMResult`, the `LLMError` hierarchy. Documented in `docs/draft-ops.md`. - `signalforge.draft.draft_schema`, `draft_from_request`, `DraftOutcome`, `CandidateSchema` family, `DraftConfig`, `load_draft_config`, the `DraftError` hierarchy. Documented in `docs/draft-ops.md`. +- `signalforge.prune.prune_tests`, `PruneResult`, `PruneDecision`, `PruneConfig`, `load_prune_config`, `PruneEvent`, the typed literals (`DropReason`, `Scope`), and the `PruneError` hierarchy (six classes). Documented in `docs/prune-ops.md`. Internals (`_loader_helpers`, `_sql_safety`, `_path_safety`, `_test_result_repr`, `adapters/_client`, `_classify_column`, `_compute_policy_hash`, `_resolve_redact_patterns`, etc.) are `_`-prefixed and not part of the public contract. diff --git a/docs/prune-ops.md b/docs/prune-ops.md index 8b2bcf0d..f528b66a 100644 --- a/docs/prune-ops.md +++ b/docs/prune-ops.md @@ -1,76 +1,145 @@ # Prune layer — operations guide -`signalforge.prune` is the test-pruning stage of the v0.1 pipeline: -runs every candidate test against warehouse data, drops the -always-pass and known-clean-fail ones, returns a typed `PruneResult`. -This is the load-bearing differentiator (Architectural Commitment #1 -in [`CLAUDE.md`](../CLAUDE.md)) — competitors generate; SignalForge -generates *and* grades. - -Companion to [`docs/safety-ops.md`](safety-ops.md), +Operational reference for users of `signalforge.prune`. Companion to +[`docs/safety-ops.md`](safety-ops.md), [`docs/draft-ops.md`](draft-ops.md), [`docs/manifest-loader-ops.md`](manifest-loader-ops.md), and -[`docs/warehouse-adapter-ops.md`](warehouse-adapter-ops.md). Internal -design lives in -[`plans/super/6-prune-engine.md`](../plans/super/6-prune-engine.md). +[`docs/warehouse-adapter-ops.md`](warehouse-adapter-ops.md), and to the +design record in [`plans/super/6-prune-engine.md`](../plans/super/6-prune-engine.md). + +The prune layer sits between the LLM-drafting layer (#5) and the diff +renderer (#8). Every candidate test produced by the drafter goes through +one entry point — `signalforge.prune.prune_tests` — which compiles the +test to failing-rows SQL, runs it against the warehouse, classifies the +verdict, and writes a fail-closed JSONL audit record per decision. + +This is the load-bearing differentiator (Architectural Commitment #1 in +[`CLAUDE.md`](../CLAUDE.md)) — competitors generate; SignalForge +generates *and* grades. -This document is an **initial scaffold**. US-014 fills the body once -the implementation stories (US-001 through US-013) land. The sections -below are placeholders with TBDs marked explicitly so the gaps are -visible to the reviewer of the next PR. +## Default posture + +Sample-scope is the default. Architectural Commitment #1 (signal over +volume) penalises always-pass tests because they consume reviewer +attention without catching anything; sampling 100k rows is enough signal +to detect always-pass while keeping query bytes under control. + +The layer is fail-closed on the audit-write boundary: any I/O error from +`PruneEvent` persistence aborts the run as `PruneAuditWriteError` +(DEC-016, mirrors safety's DEC-011 and draft's DEC-006). And the layer +is conservative on the verdict boundary: tests we cannot evaluate +(warehouse error, total-budget exhausted, ambiguous evidence) are +**kept**, not dropped — `kept-without-evidence` lands the test in front +of a human reviewer rather than silently losing potential signal. + +User-facing tagline: **always-pass tests are dropped; everything we +cannot confidently drop is kept.** ## Public API -US-014 will populate this section. The expected surface (per the -Phase-1 plan, subject to amendment as stories land): +Import from `signalforge.prune`. The 14 names exported by `__all__`: + +### Orchestrator + +- **`prune_tests(model, adapter, candidates, manifest, *, config=None, audit_path=None) -> PruneResult`** — End-to-end orchestrator. Compiles every `CandidateTest`, runs each through the warehouse adapter, writes one JSONL audit record per decision, returns the aggregate `PruneResult`. Mirrors `signalforge.draft.draft_schema` so the CLI / wrapper layers see one consistent end-to-end shape across pipeline stages. `audit_path` defaults to `/.signalforge/prune.jsonl`. + +### Result shapes + +- **`PruneResult`** — Aggregate verdict for one model. Frozen Pydantic model with fields `prune_schema_version: Literal[1]`, `model_unique_id: str`, `decisions: tuple[PruneDecision, ...]`, `elapsed_ms: int`, `signalforge_version: str`. Computed properties: `kept_decisions`, `dropped_decisions`, `kept_count`, `dropped_count`, `total_tests` — all derived from `decisions` (DEC-003) so a `PruneResult` reconstructed from a JSONL log carries identical views to a freshly produced one. + +- **`PruneDecision`** — One verdict per candidate test. Carries `test_anchor: str` (`"column."` or `"model"`), `test: CandidateTest` (the typed discriminated union from the drafter, NOT a loose dict — DEC-004; the grader and diff renderer reuse the drafter's per-variant display logic), `decision: Literal["kept", "dropped"]`, `reason: DropReason`, `failures: int`, `sampled_rows: int | None`, `scope: Scope`, `elapsed_ms: int`, `compiled_sql_hash: str` (16 hex chars; blake2b-8 per DEC-005), `compiled_sql: str`, `why: str`, `sample_failures: tuple[dict[str, Any], ...] | None`. + +### Configuration + +- **`PruneConfig`** — User-facing knobs. Frozen Pydantic model with `extra="forbid"` (config-shaped per DEC-015 — typos fail loud). Field reference: see [Configuration](#configuration-signalforgeyml-prune-block) below. + +- **`load_prune_config(path=None) -> PruneConfig`** — Loads the `prune:` block from `signalforge.yml`. Returns defaults when `path` is `None`, missing, empty, or the `prune:` key is absent. Raises `PruneConfigError` on parse / schema failures. + +### Discriminator literals -- `signalforge.prune.prune_tests(candidates, model, manifest, adapter, *, config) -> PruneResult` -- `PruneResult`, `PruneDecision`, `KeptTest`, `DroppedTest` -- `PruneConfig`, `load_prune_config` -- `DropReason`, `Scope` literals -- The `PruneError` hierarchy +- **`DropReason`** — `Literal["always-passes", "requires-future-data", "failed-on-known-clean-data", "kept", "kept-without-evidence"]`. Closed set so the diff renderer (#8) can branch on the literal value rather than sniffing prose. -(TBD — final shapes confirmed during US-014.) +- **`Scope`** — `Literal["sample", "full"]`. When `scope == "full"`, `PruneDecision.sampled_rows` is `None` (every row inspected). + +### Audit + +- **`PruneEvent`** — One JSONL audit record per `PruneDecision`. Constructed ONLY by `signalforge.prune.audit._build_prune_event` (DEC-018; AST-gated by `tests/test_audit_completeness.py`). `extra="ignore"` for forward-compat read-back. See [Audit JSONL schema](#audit-jsonl-schema) for the field set. + +### Errors + +`from signalforge.prune import errors`. Every exception subclasses +`PruneError` and carries a class-level `default_remediation` rendered on +a `↳ Remediation:` line by `__str__`. + +- **`PruneError`** — Base class. Never raised directly. +- **`PruneConfigError`** — `signalforge.yml` `prune:` block failed parse or schema validation. +- **`PruneTrustedModelNotFoundError`** — `prune.trusted_models` references a `unique_id` not in the manifest. Subclass of `PruneConfigError`. Raised at orchestrator entry, BEFORE any warehouse call (DEC-008). +- **`PruneTimeoutError`** — Internal control-flow signal for budget-exhausted dispatch. Callers of `prune_tests` do NOT see this — the orchestrator routes the in-flight test plus every remaining un-started test to `kept-without-evidence` (DEC-011). +- **`PruneAuditWriteError`** — Fail-closed audit-write failure (`OSError` / `PermissionError` / encoding / `fsync`). Aborts the run; original cause exposed via `.cause` and `__cause__` (DEC-016). +- **`PruneAuditRecordTooLargeError`** — Serialised JSONL line exceeded the POSIX-atomic-append cap (4000 bytes). Raised BEFORE any file is opened so an oversize record leaves no on-disk artefact. + +DEC-006 deliberately omits a `PruneCompilerError` class. Compilation +always succeeds; failures like `relationships(to: unknown)` emit a +structured `requires-future-data` drop reason rather than an exception. ## Configuration: `signalforge.yml` `prune:` block -Top-level namespace is `prune:` (DEC-025 of the safety layer reserves -sibling keys for future stages; the prune layer claims its own). +Top-level namespace is `prune:` (DEC-020; sibling keys `safety:`, +`llm:`, future `grade:` are reserved for other stages and silently +ignored by the prune loader). + +```yaml +safety: + # ... (loaded by signalforge.safety) +llm: + # ... (loaded by signalforge.draft) +prune: + scope: sample # "sample" | "full" + sample_size: 100000 # rows + test_timeout_seconds: 30 + total_budget_seconds: 600 + capture_failure_rows: 3 + trusted_models: + - model.shop.dim_customers + partition_filter: + column: event_dt + op: ">=" + value: "2026-01-01" +``` -US-014 will populate this section with the field-by-field reference. -Expected fields (per the Phase-1 plan): +Field-by-field: -- `scope: "sample" | "full"` (default `sample`) -- `sample_size: int` (default `100_000`) -- `test_timeout_seconds: int` (default `30`) -- `total_budget_seconds: int` (default `600`) -- `capture_failure_rows: int` (default `3`) -- `trusted_models: list[str]` — drives the `failed-on-known-clean-data` - drop reason (deferred for v0.1 unless the implementing story - re-opens the question). +- **`scope`** — `"sample"` | `"full"`. Default `"sample"`. Whether candidate tests run against a deterministic warehouse sample or a full table scan. Switch to `"full"` only when the model is small enough that `sample_size` would scan most of it anyway. +- **`sample_size`** — Integer row count for sample scope. Default `100_000`. Passed to `WarehouseAdapter.sample_rows`. Increase when the always-pass false-positive rate on small samples hides real signal; decrease to cap query bytes on very wide tables (column-pruning does NOT apply through `FARM_FINGERPRINT(TO_JSON_STRING(t))` — see [Cost model](#cost-model-us-003-verification)). +- **`test_timeout_seconds`** — Per-test wall-clock budget. Default `30`. v0.1 sets the timeout on the adapter's `QueryJobConfig` but does NOT yet thread it through `WarehouseAdapter.run_test_sql` per call (deferred to v0.2; see [v0.2 deferrals](#v02-deferrals)). +- **`total_budget_seconds`** — Whole-run wall-clock budget. Default `600`. Once exceeded, every remaining test drains to `kept-without-evidence` with `why="Total prune budget (Ns) exceeded before evaluation."` (DEC-011). Conservative bias — no test is silently dropped because the run ran long. +- **`capture_failure_rows`** — Number of failing rows recorded on the `PruneDecision.sample_failures` field per failed test. Default `3`. Set to `0` to omit row-level evidence entirely (the audit record stays compact for very wide tables). +- **`trusted_models`** — List of manifest `unique_id`s whose data is treated as known-clean. A failure on a trusted model surfaces as `failed-on-known-clean-data` (drop, presumed buggy test) rather than `kept`. Opt-in only (Q1=B). Validated against the manifest at `prune_tests` entry — typos raise `PruneTrustedModelNotFoundError` BEFORE any warehouse call (DEC-008). +- **`partition_filter`** — Optional `PartitionFilter` ADT (`{column, op, value}`) scoping every sample query. Required by the warehouse adapter for tables with `num_rows >= 100M`; otherwise optional. Pydantic recursively validates the YAML mapping into the typed shape. -(TBD — final fields and defaults confirmed during US-014.) +Unknown keys under `prune:` raise `PruneConfigError` (Pydantic +`extra="forbid"`, DEC-015). Typos like `scop:` or +`total_budget_secnds:` fail loud at load time rather than silently +no-op'ing. ## Drop-reason taxonomy Every kept and dropped test ships with a structured `PruneDecision` -carrying a one-line `why`. The reasons are a closed set so the diff -renderer (#8) can branch on the literal value rather than sniffing -prose. +carrying a one-line `why`. The reasons are a closed `DropReason` literal +so the diff renderer (#8) can branch on the value. | Reason | Decision | Why | |--------|----------|-----| -| `always-passes` | dropped | Zero failing rows on the sampled or full set; no signal worth shipping. | -| `requires-future-data` | dropped | A `relationships` test references a manifest-absent parent model. No warehouse call issued. | -| `failed-on-known-clean-data` | dropped | Test failed AND the model is in `prune.trusted_models`; the test is presumed buggy. | -| `kept` | kept | Test failed on an untrusted model; reviewer should evaluate. | -| `kept-without-evidence` | kept | Could not evaluate (warehouse error or total budget exceeded); ship conservatively. | +| `always-passes` | dropped | Zero failing rows on the sampled or full set; no signal worth shipping. The load-bearing case for Architectural Commitment #1. | +| `requires-future-data` | dropped | A `relationships` test references a `to:` parent model not in the loaded manifest. No warehouse call issued — the compiler returns a `_RequiresFutureData` sentinel and the orchestrator routes it directly to this reason (DEC-026). | +| `failed-on-known-clean-data` | dropped | Test failed AND `model.unique_id` is in `prune.trusted_models`; the test is presumed buggy. Symmetric noise-direction split with `always-passes` — both directions of noise need pruning per `CLAUDE.md`. | +| `kept` | kept | Test failed on an untrusted model with non-zero failures. Reviewer should evaluate. | +| `kept-without-evidence` | kept | Could not evaluate — warehouse error (typed `WarehouseError` subclass) or total budget exceeded (DEC-011). Ship conservatively; reviewer decides. | Conservative bias: when in doubt, keep. Architectural Commitment #1 -("signal over volume") penalises always-pass tests (no signal, -consumes reviewer attention) but does not penalise kept tests with -ambiguous evidence — those land in front of a human reviewer who can -make the final call. +penalises always-pass tests (no signal, consumes reviewer attention) but +does not penalise kept tests with ambiguous evidence — those land in +front of a human reviewer who can make the final call. ## Cost model (US-003 verification) @@ -82,123 +151,161 @@ WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), bucket) < 1 serialises the entire row into the predicate. BigQuery cannot column-prune through a function argument, so sample-mode reads **all -columns** of the table, not just the column under test. The -Phase-1 estimate that pruning 30 candidate tests against a 100k-row -sample would cost approximately 24 MB / approximately one tenth of one -US cent assumed only the column under test would be read; the worst -case is 50–500x that figure. - -**Verified figure (US-003):** TBD — populate after the diagnostic -probe (`tests/warehouse/test_sample_cost_probe.py`) runs against -`bigquery-public-data.iowa_liquor_sales.sales`. The probe records -`total_bytes_billed` for a 100k-row deterministic sample. The -threshold for action: - -- If `bytes_billed < 500 MB`: Q4=A (one-query-per-test) is viable for - v0.1 unchanged. -- If `bytes_billed >= 500 MB`: v0.2 should adopt Q4=C - (temp-table-materialised sample) so the per-test cost amortises - across the candidate set rather than re-paying the full-row scan - for every test. - -(US-014 replaces TBD with the live figure once US-003 has been run by -a maintainer with BigQuery credentials.) - -## Audit JSONL - -US-014 will populate this section if the implementing story chooses -to emit a fail-closed `PruneEvent` JSONL (mirroring the safety -audit's DEC-011 contract and the LLM response audit's DEC-006 / DEC-008 -contracts). The default expectation is that prune **does** emit one -JSONL line per `PruneDecision` so the explainable-diffs commitment -(Architectural Commitment #5) survives the v0.2 transition to -multi-warehouse adapters where the underlying SQL is no longer -homogeneous. - -(TBD — confirmed during US-014.) +columns** of the table, not just the column under test. The Phase-1 +estimate that pruning 30 candidate tests against a 100k-row sample would +cost approximately 24 MB / approximately one tenth of one US cent +assumed only the column under test would be read; the worst case is +50–500x that figure. + +**Verified figure (US-003): pending in-environment run.** The diagnostic +probe at `tests/warehouse/test_sample_cost_probe.py` runs against +`bigquery-public-data.iowa_liquor_sales.sales` and records +`total_bytes_billed` for a 100k-row deterministic sample. To run it: + +```bash +gcloud auth application-default login +SF_RUN_BQ=1 pytest -m bigquery tests/warehouse/test_sample_cost_probe.py -s +``` + +Probe thresholds (constants in the test): + +- `_BYTES_WARN_AT = 500_000_000` (500 MB) — soft WARNING fires above this. +- `_BYTES_CEILING = 5_000_000_000` (5 GB) — assertion fires above this; the test fails. + +Outcome interpretation: + +- **`bytes_billed < 500 MB`** → Q4=A (one-query-per-test) is viable for v0.1 unchanged. The Phase-1 plan's cost target holds. +- **`bytes_billed >= 500 MB`** → adopt Q4=C (temp-table-materialised sample) in v0.2 so the per-test cost amortises across the candidate set rather than re-paying the full-row scan for every test. + +Update this section once a maintainer with BigQuery credentials runs the +probe. + +## Audit JSONL schema + +Every `PruneDecision` produces exactly one JSONL record at +`audit_path` (default `/.signalforge/prune.jsonl`). One record +per line; atomic concurrent appends via `O_APPEND | O_CREAT | 0o600` and +a single `os.write` (DEC-016). The third instance of the convention +across the codebase — mirrors `signalforge.safety.audit` (DEC-011 of +safety) and `signalforge.draft.audit` (DEC-006/008/013 of llm-drafter). + +`PruneEvent` fields (~19 total): + +| Field | Type | Meaning | +| ---------------------- | ----------------------------------- | ------------------------------------------------------------------------------------------------ | +| `audit_schema_version` | integer (`Literal[1]`) | Audit shape version. Currently `1`. Bump only on shape change; `extra="ignore"` handles additions. | +| `signalforge_version` | PEP-440 version string | Package version that produced the record. | +| `record_id` | 32-hex-char string | Fresh `uuid4().hex` per record; gives reviewers a stable handle for a single decision. | +| `timestamp` | ISO-8601 UTC, microsecond, `Z` | When the decision was finalised. | +| `config_hash` | 16 hex chars | First 16 hex chars of `SHA-256(canonical_config_json)`. Mirrors safety's `policy_hash` (DEC-005). | +| `model_unique_id` | string | dbt `unique_id` of the pruned model. | +| `test` | discriminated-union object | The original `CandidateTest` from the drafter (typed; not a loose dict — DEC-004). | +| `test_anchor` | string | `"column."` for column-scoped tests; literal `"model"` for model-level tests. | +| `decision` | `"kept"` \| `"dropped"` | Top-level verdict. | +| `reason` | `DropReason` literal | One of the five reasons in [Drop-reason taxonomy](#drop-reason-taxonomy). | +| `failures` | integer | Failing-row count from the warehouse. `0` for `always-passes` and `requires-future-data`. | +| `sampled_rows` | integer or `null` | Sample size the test ran against. `null` for full-scope or no-warehouse-call decisions. | +| `scope` | `"sample"` \| `"full"` | Mirrors `PruneConfig.scope`. | +| `elapsed_ms` | integer | Per-test wall-clock cost. `0` for budget-exhausted (test never ran). | +| `compiled_sql_hash` | 16 hex chars | `blake2b(sql.encode(), digest_size=8).hexdigest()`. Stable empty-string hash for no-SQL outcomes. | +| `compiled_sql` | string | The exact SELECT issued to the warehouse. Empty for `requires-future-data` and budget-exhausted. | +| `why` | string | One-line human-readable rationale. Architectural Commitment #5. | +| `sample_failures` | array of object or `null` | Up to `capture_failure_rows` failing rows. `null` when capture is disabled or no failures. | + +**Fail-closed semantics.** `OSError` / `PermissionError` / encoding +failures from `os.write` / `os.fsync` propagate raw; the orchestrator +wraps them as `PruneAuditWriteError` and aborts the run. +`PruneAuditRecordTooLargeError` (size cap, raised before any file open) +also aborts the run. Don't wrap audit-write calls in defensive +try/except — propagation IS the defence (mirrors +`safety-layer.md` DEC-011). + +**Schema-drift gate.** `tests/fixtures/prune/prune_event_v1.jsonl` is +the canonical schema fixture; `tests/prune/test_drift_detector.py` +pairs the production model (`extra="ignore"`) with a one-off +`extra="forbid"` strict model and validates against the fixture. +Adding a field to `PruneEvent` without updating the strict model OR +the fixture breaks the test loudly. Don't bypass. + +## Audit log sensitivity + +`prune.jsonl` contains the model's compiled SQL and (when +`capture_failure_rows > 0`) up to N rows of failing data per test. +Treat the file at-rest the same way you treat the safety audit: + +- **Gitignore `.signalforge/`** (already configured in this repo's `.gitignore`). +- **Restrict at-rest permissions.** The writer creates the file at `0o600` on first call; the parent directory is created via `mkdir(parents=True, exist_ok=True)` (Python's `mkdir` does not tighten an existing directory's permissions, so verify the existing `.signalforge/` mode is `0o700` on shared hosts). +- **Don't ship as a build artifact.** Strip from container images and CI uploads. +- **Set `capture_failure_rows: 0`** for PII-laden models if the safety layer's redaction policy isn't enough — the prune layer captures real warehouse rows for failures, which can include PII not flagged for the LLM redactor. ## Running real-warehouse tests The prune layer's integration tests share the warehouse adapter's -gating discipline. Default CI excludes them via the -`-m 'not bigquery'` filter; opt-in requires both the marker and an -`SF_RUN_BQ=1` env var. +gating discipline. Default CI excludes them via the `-m 'not bigquery'` +filter; opt-in requires both the marker and an `SF_RUN_BQ=1` env var. ```bash gcloud auth application-default login SF_RUN_BQ=1 pytest -m bigquery ``` -The diagnostic cost probe (US-003) lives at -`tests/warehouse/test_sample_cost_probe.py` and runs under the same -gate. It is a documentation-grade probe — a soft WARNING fires at -500 MB; the test fails only above the 5 GB sanity ceiling. +The prune-layer integration test (`tests/prune/test_integration_bigquery.py`) +requires ambient gcloud auth (matches the warehouse adapter's +`tests/warehouse/test_bigquery_integration.py`). -(US-014 expands this section with the prune-specific integration -tests once US-013 lands.) +The diagnostic cost probe (US-003) at +`tests/warehouse/test_sample_cost_probe.py` runs under the same gate. +It is a documentation-grade probe — a soft WARNING fires at 500 MB; the +test fails only above the 5 GB sanity ceiling. See +[Cost model](#cost-model-us-003-verification). ## Debugging -US-014 will populate this section with logger names, levels, and the -ANSI-safe lazy-format JSON discipline that the safety / drafter layers -already follow (DEC-022 of safety; DEC-011 of llm-drafter). +Logger name: `signalforge.prune.engine` (and sibling modules under +`signalforge.prune`). + +```python +import logging +logging.getLogger("signalforge.prune").setLevel(logging.DEBUG) +``` + +Levels: -(TBD — confirmed during US-014.) +- **WARNING** — One line per `kept-without-evidence` decision routed by a typed `WarehouseError`. Lazy-format JSON per DEC-017 (`signalforge_version`, `model_unique_id`, `test_anchor`, `error_class`). Never f-string-interpolate user-controlled strings into a logger call — a column name or model id containing ANSI escapes (`\x1b[31m...`) would inject into log viewers. +- **INFO** / **DEBUG** — reserved for future budget-loop / batching observability; v0.1 emits no INFO/DEBUG from the engine. -## Typed-error reference +The prune layer never logs full row data. The audit JSONL is the single +durable record of decision-level detail; logger output is a hint that +the decision happened, not what was in it. -US-014 will populate the table once US-005 lands the `PruneError` -hierarchy. Mirrors the safety / draft / warehouse error tables. +**Reading a fail-closed `PruneAuditWriteError`.** The cause is exposed +as `.cause` and on `__cause__`. Common causes: -(TBD — confirmed during US-014.) +- Parent directory not writable (no `+w` for the user, or `.signalforge/` is a symlink to a read-only mount). +- Disk full (`ENOSPC`). +- Oversize record (raises `PruneAuditRecordTooLargeError` instead — reduce `capture_failure_rows` or trim `compiled_sql` size by simplifying the candidate test; the cap is 4000 bytes for POSIX-atomic concurrent appends). ## v0.2 deferrals The prune layer is intentionally narrow in v0.1. The following concerns are explicitly deferred: -- **Per-decision `bytes_billed` recording (DEC-027 of issue #6).** - The adapter does not surface job stats in v0.1; the diagnostic - probe (US-003) reads them via `INFORMATION_SCHEMA.JOBS_BY_USER` - out-of-band rather than through the adapter API. v0.2 extends the - adapter's seam to return job stats so the `PruneDecision` can - carry the figure natively. -- **Per-test `timeout_ms` threading.** US-002 plumbed - `timeout_ms` into `make_query_job_config` (DEC-013 of issue #6, - AR-B2); surfacing it through `WarehouseAdapter.run_test_sql`'s - public signature is a v0.2 task. v0.1 prune sets the timeout on - the job config but cannot enforce it per-call without the - surfaced parameter. -- **Test batching — Q4=B / Q4=C optimisations.** The Phase-1 plan - catalogues two cost optimisations (per-column `COUNTIF` batching; - temp-table-materialised sample) that v0.1 does not adopt. US-003 - produces the data needed to evaluate the temp-table option in - v0.2. -- **Multi-warehouse adapters.** Snowflake, Postgres, Databricks, - Redshift adapters slot in behind `WarehouseAdapter` without prune - changes once their adapters land. The prune compiler already uses - `Dialect.quote_char` and `Dialect.identifier_case` to stay - warehouse-agnostic; no SQL paths are BigQuery-specific. -- **Historical always-pass evidence.** Running candidate tests - against multiple `run_results.json` snapshots to assert "never - failed in last N runs." The Phase-1 plan considers this for the - `failed-on-known-clean-data` evidence channel and defers to v0.2. -- **dbt-utils test types.** `dbt_utils.unique_combination_of_columns`, - `dbt_utils.accepted_range`, `dbt_utils.expression_is_true`, etc. - The drafter's `CandidateTest` union has exactly four variants; - prune compiles exactly four. v0.2 territory. -- **`where:` test modifier and `severity: warn` / `mostly:`.** - dbt-core supports a `where:` predicate on every test plus a - `severity` and `mostly` knob; v0.1 prune does not consume any of - these. -- **`prune_decision_id`-keyed checkpoint / resumption.** Long-running - prune runs that resume from disk after a crash. v0.2. -- **Confidence intervals on `always-passes`.** Surfacing - "less than or equal to 3/N upper-bound failure rate at 95 percent - confidence" on the decision record. Nice to have; v0.2. -- **LLM-generated rationale on `kept` decisions.** The grader (#7) - produces rubric-scored rationale; prune writes only the structured - drop reason plus failure count plus scope. - -(US-014 expands the v0.2 list as additional deferrals surface during -implementation.) +- **Per-decision `bytes_billed` recording (DEC-027).** The adapter does not surface job stats in v0.1; the diagnostic probe (US-003) reads them via `INFORMATION_SCHEMA.JOBS_BY_USER` out-of-band rather than through the adapter API. v0.2 extends the adapter's seam to return job stats so the `PruneDecision` can carry the figure natively. +- **Per-test `timeout_ms` threading.** `PruneConfig.test_timeout_seconds` is documented but not yet threaded through `WarehouseAdapter.run_test_sql` per call. The plumbing exists in `make_query_job_config` (DEC-013, AR-B2 of issue #6); surfacing it through the public adapter signature is a v0.2 task. +- **Test batching — Q4=B / Q4=C optimisations.** The Phase-1 plan catalogues two cost optimisations (per-column `COUNTIF` batching; temp-table-materialised sample). v0.1 does not adopt either. US-003 produces the data needed to evaluate the temp-table option in v0.2. +- **Multi-warehouse adapters.** Snowflake, Postgres, Databricks, Redshift adapters slot in behind `WarehouseAdapter` without prune changes once their adapters land. The prune compiler already dispatches on `Dialect.quote_char` (DEC-025), not on dialect `name`; no SQL paths are BigQuery-specific. +- **Confidence intervals on `always-passes`.** Surfacing "less than or equal to 3/N upper-bound failure rate at 95 percent confidence" (rule of three) on the decision record so reviewers can calibrate the always-pass verdict. Also covers great-expectations-style `mostly:` thresholds. +- **Historical always-pass evidence.** Running candidate tests against multiple `run_results.json` snapshots to assert "never failed in last N runs." The Phase-1 plan considers this for the `failed-on-known-clean-data` evidence channel and defers to v0.2. +- **dbt-utils test types.** `dbt_utils.unique_combination_of_columns`, `dbt_utils.accepted_range`, `dbt_utils.expression_is_true`, etc. The drafter's `CandidateTest` union has exactly four variants in v0.1; the prune compiler compiles exactly four. v0.2 territory. +- **`where:` test modifier and `severity: warn` / `mostly:`.** dbt-core supports a `where:` predicate on every test plus `severity` and `mostly` knobs; v0.1 prune does not consume any of these. +- **`prune_decision_id`-keyed checkpoint / resumption.** Long-running prune runs that resume from disk after a crash. v0.2. +- **LLM-generated rationale on `kept` decisions.** The grader (#7) produces rubric-scored rationale; prune writes only the structured drop reason plus failure count plus scope. + +## CLI integration note + +Tracked in [issue #9](https://github.com/wjduenow/SignalForge/issues/9). +The `signalforge generate` CLI will load the prune config via +`load_prune_config(...)` and invoke `prune_tests(...)` after the LLM +draft completes; the diff renderer (#8) consumes the returned +`PruneResult` to emit kept/dropped artifacts with their per-decision +`why` lines (Architectural Commitment #5 — explainable diffs). From e64df5a4ed9ebcdd0821249972fb3d6e11eaf914 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 23:33:23 -0700 Subject: [PATCH 20/24] 6/QG: fix bugs surfaced by 4-pass code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven fixes across the prune layer's surface and tests: * Custom PruneResult.__repr__ redacts compiled_sql and sample_failures so accidental log-line interpolation can't leak SQL or sampled rows (DEC-022). * Audit path canonicalised via warehouse._path_safety before write so a symlinked .signalforge/prune.jsonl can't redirect writes outside the project (DEC-016). * load_prune_config(project_dir, path=None) signature aligned with load_safety_config / load_draft_config — one calling convention across stages. * Default audit_path resolves relative to project_dir (defaulted to cwd) instead of cwd directly — matches safety + draft fix. * Compiler identifier shape validation: CandidateTest column/field/to routed through warehouse._sql_safety.validate_identifier; an adversarial identifier returns the new _InvalidIdentifier sentinel which the engine routes to kept-without-evidence (DEC-024). * test_yaml_safe_load_rejects_python_objects strengthened to use a side-effect-detectable gadget (Path.touch on a tmp marker); test now actually fails if yaml.load is substituted for yaml.safe_load. * PruneTimeoutError docstring documents it as forward-compat for v0.2 (kept on public surface; no v0.1 callers see it). Plan: plans/super/6-prune-engine.md (Quality Gate of issue #6). Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/prune-ops.md | 2 +- src/signalforge/prune/compiler.py | 81 ++++++++++++-- src/signalforge/prune/config.py | 29 +++-- src/signalforge/prune/engine.py | 117 ++++++++++++++++++-- src/signalforge/prune/errors.py | 19 +++- src/signalforge/prune/models.py | 20 ++++ tests/fixtures/prune/signalforge_unsafe.yml | 2 - tests/prune/test_compiler.py | 88 +++++++++++++++ tests/prune/test_config.py | 98 +++++++++++----- tests/prune/test_engine.py | 107 ++++++++++++++++++ tests/prune/test_models.py | 42 +++++++ 11 files changed, 541 insertions(+), 64 deletions(-) delete mode 100644 tests/fixtures/prune/signalforge_unsafe.yml diff --git a/docs/prune-ops.md b/docs/prune-ops.md index f528b66a..535b5869 100644 --- a/docs/prune-ops.md +++ b/docs/prune-ops.md @@ -53,7 +53,7 @@ Import from `signalforge.prune`. The 14 names exported by `__all__`: - **`PruneConfig`** — User-facing knobs. Frozen Pydantic model with `extra="forbid"` (config-shaped per DEC-015 — typos fail loud). Field reference: see [Configuration](#configuration-signalforgeyml-prune-block) below. -- **`load_prune_config(path=None) -> PruneConfig`** — Loads the `prune:` block from `signalforge.yml`. Returns defaults when `path` is `None`, missing, empty, or the `prune:` key is absent. Raises `PruneConfigError` on parse / schema failures. +- **`load_prune_config(project_dir, path=None) -> PruneConfig`** — Loads the `prune:` block from `signalforge.yml`. Resolves to `/signalforge.yml` when `path` is `None`. Returns defaults when the file is missing, empty, or the `prune:` key is absent. Raises `PruneConfigError` on parse / schema failures. Mirrors `load_safety_config` / `load_draft_config` so the CLI sees one calling convention across stages. ### Discriminator literals diff --git a/src/signalforge/prune/compiler.py b/src/signalforge/prune/compiler.py index c6c3dad7..1c87c695 100644 --- a/src/signalforge/prune/compiler.py +++ b/src/signalforge/prune/compiler.py @@ -61,7 +61,8 @@ CandidateTestRelationships, CandidateTestUnique, ) -from signalforge.warehouse._sql_safety import escape_bq_string_literal +from signalforge.warehouse._sql_safety import escape_bq_string_literal, validate_identifier +from signalforge.warehouse.errors import InvalidIdentifierError from signalforge.warehouse.models import TableRef if TYPE_CHECKING: @@ -82,6 +83,29 @@ class _RequiresFutureData: reason: str +@dataclass(frozen=True, slots=True) +class _InvalidIdentifier: + """Sentinel returned by :func:`_compile_test` when a candidate test's + identifier (``column``, ``field``) fails the DEC-013 SQL-identifier shape. + + Defence-in-depth: ``CandidateTest.column`` / ``.field`` arrive from + the LLM drafter via the anchor-contract validator (which checks the + name exists in the manifest model) but are NOT shape-validated against + the SQL-identifier regex. A malformed identifier would be backtick- + quoted into the failing-rows SELECT and could break out of the + quoting (e.g. an embedded backtick or whitespace). + + The orchestrator routes this sentinel to ``kept-without-evidence`` + (decision="kept") with the ``reason`` text in ``why`` so a reviewer + sees the malformed identifier and can fix the prompt or model + upstream. Treating it as "could not evaluate" rather than "drop" is + conservative — a malformed test MAY still be signal-bearing once + fixed. + """ + + reason: str + + def _quote(identifier: str, quote_char: str) -> str: """Wrap ``identifier`` in ``quote_char`` for SQL embedding. @@ -114,8 +138,16 @@ def _compile_not_null( test: CandidateTestNotNull, table_ref: TableRef, quote_char: str, -) -> str: +) -> str | _InvalidIdentifier: """Compile ``not_null(col)`` to ``SELECT col FROM t WHERE col IS NULL``.""" + try: + validate_identifier("CandidateTestNotNull.column", test.column) + except InvalidIdentifierError: + return _InvalidIdentifier( + reason=( + f"candidate test references an invalid identifier shape: column={test.column!r}" + ) + ) col = _quote(test.column, quote_char) table = _qualified_table_name(table_ref, quote_char) return f"SELECT {col} FROM {table} WHERE {col} IS NULL" @@ -125,13 +157,21 @@ def _compile_unique( test: CandidateTestUnique, table_ref: TableRef, quote_char: str, -) -> str: +) -> str | _InvalidIdentifier: """Compile ``unique(col)`` to a GROUP BY ... HAVING COUNT(*) > 1. DEC-023 NULL-exclusion: ``IS NOT NULL`` filters NULL rows out of the grouped set, matching dbt-core (multiple NULLs in a column do not violate uniqueness in dbt's convention). """ + try: + validate_identifier("CandidateTestUnique.column", test.column) + except InvalidIdentifierError: + return _InvalidIdentifier( + reason=( + f"candidate test references an invalid identifier shape: column={test.column!r}" + ) + ) col = _quote(test.column, quote_char) table = _qualified_table_name(table_ref, quote_char) return f"SELECT {col} FROM {table} WHERE {col} IS NOT NULL GROUP BY {col} HAVING COUNT(*) > 1" @@ -141,7 +181,7 @@ def _compile_accepted_values( test: CandidateTestAcceptedValues, table_ref: TableRef, quote_char: str, -) -> str: +) -> str | _InvalidIdentifier: """Compile ``accepted_values(col, values)`` to a ``NOT IN`` predicate. Each value goes through @@ -153,6 +193,14 @@ def _compile_accepted_values( adversarial inputs (the entire injection attempt stays inside the quoted string). """ + try: + validate_identifier("CandidateTestAcceptedValues.column", test.column) + except InvalidIdentifierError: + return _InvalidIdentifier( + reason=( + f"candidate test references an invalid identifier shape: column={test.column!r}" + ) + ) col = _quote(test.column, quote_char) table = _qualified_table_name(table_ref, quote_char) rendered_values = ", ".join(f"'{escape_bq_string_literal(v)}'" for v in test.values) @@ -189,7 +237,7 @@ def _compile_relationships( table_ref: TableRef, quote_char: str, manifest: Manifest, -) -> str | _RequiresFutureData: +) -> str | _RequiresFutureData | _InvalidIdentifier: """Compile ``relationships(child_col, to=parent, field=parent_col)``. Renders a LEFT JOIN orphan-detection SELECT: rows in the child where @@ -199,7 +247,23 @@ def _compile_relationships( model is not in the manifest (DEC-026); the orchestrator routes that to the ``requires-future-data`` drop reason without issuing a warehouse call. + + Returns an :class:`_InvalidIdentifier` sentinel when ``column`` or + ``field`` fails the SQL-identifier shape check; the orchestrator + routes that to ``kept-without-evidence``. ``to`` is NOT shape-checked + here — it's a model name resolved via :func:`_resolve_parent_table_ref` + and a missing parent yields the ``_RequiresFutureData`` branch. """ + try: + validate_identifier("CandidateTestRelationships.column", test.column) + validate_identifier("CandidateTestRelationships.field", test.field) + except InvalidIdentifierError as exc: + return _InvalidIdentifier( + reason=( + f"candidate test references an invalid identifier shape: {exc.field}={exc.value!r}" + ) + ) + parent_table_ref = _resolve_parent_table_ref(test.to, manifest) if isinstance(parent_table_ref, _RequiresFutureData): return parent_table_ref @@ -222,7 +286,7 @@ def _compile_test( table_ref: TableRef, dialect: Dialect, manifest: Manifest, -) -> str | _RequiresFutureData: +) -> str | _RequiresFutureData | _InvalidIdentifier: """Render a candidate test as a failing-rows SELECT. The returned string is a SELECT whose rows are violations: zero rows @@ -237,8 +301,9 @@ def _compile_test( A ``relationships`` test whose parent is not in the manifest returns :class:`_RequiresFutureData` rather than raising (DEC-006, DEC-026); - the orchestrator distinguishes the two return shapes via - :func:`isinstance`. + a test whose ``column`` / ``field`` fails the SQL-identifier shape + check returns :class:`_InvalidIdentifier`; the orchestrator + distinguishes the three return shapes via :func:`isinstance`. """ quote_char = dialect.quote_char if isinstance(test, CandidateTestNotNull): diff --git a/src/signalforge/prune/config.py b/src/signalforge/prune/config.py index 38f5890a..4f159858 100644 --- a/src/signalforge/prune/config.py +++ b/src/signalforge/prune/config.py @@ -56,6 +56,8 @@ from signalforge.prune.errors import PruneConfigError from signalforge.warehouse.models import PartitionFilter +_DEFAULT_CONFIG_FILENAME = "signalforge.yml" + class PruneConfig(BaseModel): """User-facing knobs for the prune layer (DEC-009). @@ -134,16 +136,27 @@ class _PruneConfigFile(BaseModel): prune: PruneConfig | None = None -def load_prune_config(path: Path | str | None = None) -> PruneConfig: +def load_prune_config(project_dir: Path, path: Path | None = None) -> PruneConfig: """Load a :class:`PruneConfig` from ``signalforge.yml``. - See module docstring for the full resolution order and error - contract. + Mirrors :func:`signalforge.draft.config.load_draft_config` and + :func:`signalforge.safety.config.load_safety_config` so the CLI (#9) + sees one calling convention across stages: ``(project_dir, path=None)``. + + Resolution: + + * ``path is None``: look for ``/signalforge.yml``. + Missing → :class:`PruneConfig` defaults silently. + * ``path is not None``: use that exact path. Missing → defaults + silently (parity with the prior single-arg behaviour and with how + the CLI threads an optional ``--config`` flag). Args: - path: Optional explicit config-file path. ``None`` or a path that - does not exist returns :class:`PruneConfig` defaults - silently. + project_dir: Project root used as the base for the default + config-file lookup (``/signalforge.yml``). + path: Optional explicit config path. ``None`` falls back to the + project-relative default; a missing explicit path returns + defaults silently. Returns: A fully-validated :class:`PruneConfig`. When the file is absent, @@ -159,10 +172,8 @@ def load_prune_config(path: Path | str | None = None) -> PruneConfig: original :class:`pydantic.ValidationError` (if any) is preserved on ``__cause__``. """ - if path is None: - return PruneConfig() + config_file = project_dir / _DEFAULT_CONFIG_FILENAME if path is None else path - config_file = Path(path) if not config_file.exists(): return PruneConfig() diff --git a/src/signalforge/prune/engine.py b/src/signalforge/prune/engine.py index 9402f582..b96b558d 100644 --- a/src/signalforge/prune/engine.py +++ b/src/signalforge/prune/engine.py @@ -91,6 +91,7 @@ from signalforge.prune.compiler import ( _compile_test, _compute_compiled_sql_hash, + _InvalidIdentifier, _RequiresFutureData, ) from signalforge.prune.config import PruneConfig @@ -100,8 +101,9 @@ PruneTrustedModelNotFoundError, ) from signalforge.prune.models import PruneDecision, PruneResult, Scope +from signalforge.warehouse._path_safety import canonicalise_path from signalforge.warehouse.base import WarehouseAdapter -from signalforge.warehouse.errors import WarehouseError +from signalforge.warehouse.errors import ProfileNotFoundError, WarehouseError from signalforge.warehouse.models import TableRef, TestResult _LOGGER = logging.getLogger(__name__) @@ -360,6 +362,40 @@ def _decide_requires_future_data( ) +def _decide_kept_without_evidence_invalid_identifier( + *, + test: CandidateTest, + test_anchor: str, + sentinel: _InvalidIdentifier, + elapsed_ms: int, + scope: Scope, +) -> PruneDecision: + """Build a :class:`PruneDecision` for a test whose ``column`` / + ``field`` failed the SQL-identifier shape check (defence-in-depth). + + No warehouse call has been issued; ``compiled_sql`` is empty and + ``compiled_sql_hash`` is the stable hash of the empty string. Routed + to ``kept-without-evidence`` (decision="kept") rather than + ``"dropped"`` because a malformed identifier MAY still be + signal-bearing once the operator fixes the upstream prompt / + manifest — conservative default avoids silently losing the test. + """ + return PruneDecision( + test_anchor=test_anchor, + test=test, + decision="kept", + reason="kept-without-evidence", + failures=0, + sampled_rows=None, + scope=scope, + elapsed_ms=elapsed_ms, + compiled_sql_hash=_build_compiled_sql_hash_or_empty(""), + compiled_sql="", + why=sentinel.reason, + sample_failures=None, + ) + + def _decide_kept_without_evidence_warehouse_error( *, test: CandidateTest, @@ -434,6 +470,7 @@ def prune_tests( *, config: PruneConfig | None = None, audit_path: Path | None = None, + project_dir: Path | None = None, ) -> PruneResult: """Drop always-pass and known-clean-fail candidate tests. @@ -504,8 +541,21 @@ def prune_tests( responsible for loading from disk via :func:`load_prune_config`. audit_path: optional override for the prune-audit JSONL path. - ``None`` resolves to ``cwd / ".signalforge" / "prune.jsonl"``; - the parent directory is created if missing. + ``None`` resolves to + `` / ".signalforge" / "prune.jsonl"`` (default + ``project_dir`` is :func:`pathlib.Path.cwd`); the parent + directory is created if missing. The resolved path is + symlink-hardened via + :func:`signalforge.warehouse._path_safety.canonicalise_path` + before any write so a symlinked + ``/.signalforge/prune.jsonl`` cannot redirect + writes outside the project tree (DEC-016). + project_dir: optional project-root override used to resolve the + default ``audit_path``. ``None`` resolves to + :func:`pathlib.Path.cwd`. Mirrors the other layers' + project-relative resolution so the prune audit lands in the + right place regardless of which sub-directory the CLI + invokes from. Returns: A :class:`PruneResult` carrying every per-test @@ -536,14 +586,41 @@ def prune_tests( is_trusted = model.unique_id in resolved_config.trusted_models scope: Scope = resolved_config.scope - # Resolve audit path. Default: cwd/.signalforge/prune.jsonl. Create - # the parent directory if missing — the writer itself does not - # mkdir (it expects the parent to exist; the orchestrator owns that - # contract per the writer's docstring). - resolved_audit_path = ( - audit_path if audit_path is not None else Path.cwd() / ".signalforge" / "prune.jsonl" + # Resolve audit path. Default: /.signalforge/prune.jsonl + # (project_dir defaults to cwd). Resolving relative to project_dir + # rather than cwd matches the safety + draft layers — when the CLI + # is invoked from a sub-directory, the audit lands next to the + # project, not next to wherever the user happened to be. + resolved_project_dir = project_dir if project_dir is not None else Path.cwd() + raw_audit_path = ( + audit_path + if audit_path is not None + else resolved_project_dir / ".signalforge" / "prune.jsonl" ) - resolved_audit_path.parent.mkdir(parents=True, exist_ok=True) + # Create the parent directory if missing — the writer itself does + # not mkdir (it expects the parent to exist; the orchestrator owns + # that contract per the writer's docstring). mkdir runs BEFORE + # canonicalise_path so the directory exists for the resolve(strict). + raw_audit_path.parent.mkdir(parents=True, exist_ok=True) + + # Symlink-harden the audit path (DEC-016) — a symlinked + # `.signalforge/prune.jsonl` pointing outside the project tree + # could redirect writes to an attacker-controlled location. + # ``canonicalise_path`` requires project_dir to exist; it raises + # ``ProfileNotFoundError`` (the warehouse-layer escape) on cycle / + # missing-dir / containment-violation. Wrap as + # ``PruneAuditWriteError`` so the prune layer's catch surface stays + # homogeneous — every "we couldn't durably persist the audit" + # condition raises a single typed error. + resolved_project_dir.mkdir(parents=True, exist_ok=True) + try: + resolved_audit_path = canonicalise_path(raw_audit_path, resolved_project_dir) + except ProfileNotFoundError as exc: + raise PruneAuditWriteError( + "Prune-audit path failed symlink-hardened canonicalisation; " + "refusing to write outside the project tree.", + cause=exc, + ) from exc # Compute the config hash once per run so every audit row in this # run shares a single config_hash; mirrors the safety-layer @@ -584,7 +661,8 @@ def prune_tests( continue # Compile the candidate test to failing-rows SQL. Returns either - # a string (the SELECT) or a _RequiresFutureData sentinel. + # a string (the SELECT), a _RequiresFutureData sentinel, or an + # _InvalidIdentifier sentinel. compile_result = _compile_test(test, table_ref, dialect, manifest) if isinstance(compile_result, _RequiresFutureData): decision = _decide_requires_future_data( @@ -603,6 +681,23 @@ def prune_tests( decisions.append(decision) continue + if isinstance(compile_result, _InvalidIdentifier): + decision = _decide_kept_without_evidence_invalid_identifier( + test=test, + test_anchor=test_anchor, + sentinel=compile_result, + elapsed_ms=0, + scope=scope, + ) + _write_audit_or_abort( + decision, + model_unique_id=model.unique_id, + config_hash=config_hash, + audit_path=resolved_audit_path, + ) + decisions.append(decision) + continue + compiled_sql: str = compile_result compiled_sql_hash = _build_compiled_sql_hash_or_empty(compiled_sql) diff --git a/src/signalforge/prune/errors.py b/src/signalforge/prune/errors.py index c594356e..0ac0b7ff 100644 --- a/src/signalforge/prune/errors.py +++ b/src/signalforge/prune/errors.py @@ -111,12 +111,19 @@ def __init__(self, unique_id: str, *, remediation: str | None = None) -> None: class PruneTimeoutError(PruneError): """The prune run exceeded its ``total_budget_seconds`` budget. - Wraps adapter cancellation when the orchestrator pulls the plug on an - in-flight test. DEC-011: the orchestrator catches this internally and - routes the in-flight + every remaining un-started test to - ``kept-without-evidence`` with ``why="total prune budget exceeded - before evaluation"``. Callers of :func:`signalforge.prune.prune_tests` - do NOT see this exception; it is an internal control-flow signal. + Reserved for v0.2 — currently never raised. v0.1's ``DEC-011`` budget + enforcement converts the exhaustion signal directly into + ``kept-without-evidence`` :class:`PruneDecision` rows; the typed + error is listed on the public surface and re-exported from + :mod:`signalforge.prune` so v0.2 callers can write + ``except PruneTimeoutError`` ahead of the feature landing without a + breaking-change to the public API. When the v0.2 per-test timeout + plumbing lands, the in-flight test that exhausts the budget will + raise this error from the adapter; the orchestrator will catch it + internally and route the remaining tests to + ``kept-without-evidence`` (mirroring how the + :class:`signalforge.warehouse.errors.WarehouseError` branch works in + v0.1). """ default_remediation: ClassVar[str] = ( diff --git a/src/signalforge/prune/models.py b/src/signalforge/prune/models.py index b708d54d..9f89f1c2 100644 --- a/src/signalforge/prune/models.py +++ b/src/signalforge/prune/models.py @@ -154,6 +154,26 @@ def dropped_count(self) -> int: def total_tests(self) -> int: return len(self.decisions) + def __repr__(self) -> str: + """Redacted repr — omits per-decision SQL and sample-failure rows. + + DEC-022: Pydantic's default ``__repr__`` would interpolate every + field including ``decisions[i].compiled_sql``, ``decisions[i].why``, + and ``decisions[i].sample_failures`` into a single line. An + accidental ``_LOGGER.warning("result: %s", result)`` would dump + compiled SQL plus sampled rows (which may contain PII) into log + sinks. The custom repr collapses to the top-level identity and + the two count aggregates so log lines stay safe by default; + callers that genuinely need the full body call + :meth:`pydantic.BaseModel.model_dump` explicitly. + """ + return ( + f"PruneResult(model_unique_id={self.model_unique_id!r}, " + f"kept_count={self.kept_count}, " + f"dropped_count={self.dropped_count}, " + f"elapsed_ms={self.elapsed_ms})" + ) + __all__ = ( "DropReason", diff --git a/tests/fixtures/prune/signalforge_unsafe.yml b/tests/fixtures/prune/signalforge_unsafe.yml deleted file mode 100644 index 0425f3b8..00000000 --- a/tests/fixtures/prune/signalforge_unsafe.yml +++ /dev/null @@ -1,2 +0,0 @@ -prune: - scope: !!python/object/apply:os.system ["echo pwned"] diff --git a/tests/prune/test_compiler.py b/tests/prune/test_compiler.py index 6d7c91ce..960d4d3c 100644 --- a/tests/prune/test_compiler.py +++ b/tests/prune/test_compiler.py @@ -33,6 +33,7 @@ from signalforge.prune.compiler import ( _compile_test, _compute_compiled_sql_hash, + _InvalidIdentifier, _RequiresFutureData, ) from signalforge.warehouse._sql_safety import validate_test_sql @@ -346,3 +347,90 @@ def test_validate_identifier_already_done_at_table_ref_construction() -> None: with pytest.raises(InvalidIdentifierError): TableRef(project="fake_project", dataset="dataset", name="orders; DROP TABLE x") + + +# --------------------------------------------------------------------------- +# QG fix-up: defence-in-depth — adversarial column/field identifiers on +# CandidateTest variants return _InvalidIdentifier rather than producing +# a malformed SQL string. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "bad_column", + [ + "col with space", + 'col"-injection', + "col`-backtick", + "col;DROP", + ], +) +def test_compile_not_null_rejects_adversarial_column(bad_column: str) -> None: + """``CandidateTestNotNull.column`` failing the SQL-identifier shape + check returns ``_InvalidIdentifier`` rather than producing SQL that + breaks out of the backtick quoting.""" + test = CandidateTestNotNull(column=bad_column) + result = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(result, _InvalidIdentifier) + assert "invalid identifier" in result.reason + + +@pytest.mark.parametrize( + "bad_column", + [ + "col with space", + 'col"-injection', + "col`-backtick", + "col;DROP", + ], +) +def test_compile_unique_rejects_adversarial_column(bad_column: str) -> None: + test = CandidateTestUnique(column=bad_column) + result = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(result, _InvalidIdentifier) + + +@pytest.mark.parametrize( + "bad_column", + [ + "col with space", + 'col"-injection', + "col`-backtick", + "col;DROP", + ], +) +def test_compile_accepted_values_rejects_adversarial_column(bad_column: str) -> None: + test = CandidateTestAcceptedValues(column=bad_column, values=("a",)) + result = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(result, _InvalidIdentifier) + + +@pytest.mark.parametrize( + "bad_field", + [ + "field with space", + 'field"-injection', + "field`-backtick", + "field;DROP", + ], +) +def test_compile_relationships_rejects_adversarial_field(bad_field: str) -> None: + """``CandidateTestRelationships.field`` is interpolated into a + backtick-quoted identifier; an adversarial value must short-circuit + via ``_InvalidIdentifier`` rather than reach the SQL string.""" + test = CandidateTestRelationships(column="customer_id", to="customers", field=bad_field) + result = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(result, _InvalidIdentifier) + # The field name is what's offending; the reason should reflect that. + assert "field" in result.reason + + +def test_compile_relationships_rejects_adversarial_column() -> None: + test = CandidateTestRelationships( + column="child`-backtick", + to="customers", + field="id", + ) + result = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, _make_manifest()) + assert isinstance(result, _InvalidIdentifier) + assert "column" in result.reason diff --git a/tests/prune/test_config.py b/tests/prune/test_config.py index 4f9471e0..b707526c 100644 --- a/tests/prune/test_config.py +++ b/tests/prune/test_config.py @@ -2,8 +2,8 @@ Covers the resolution + validation contract of :func:`load_prune_config`: -* ``path=None`` or missing path → :class:`PruneConfig` defaults silently - (DEC-009). +* ``path=None`` with no ``/signalforge.yml`` → + :class:`PruneConfig` defaults silently (DEC-009). * Empty / minimal ``prune:`` blocks → defaults. * Full round-trip with non-default values for every field. * Typo'd inner field (``scop:``) → :class:`PruneConfigError` (DEC-015, @@ -17,6 +17,7 @@ from __future__ import annotations +import shutil from pathlib import Path import pytest @@ -28,10 +29,20 @@ _FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" / "prune" -def test_load_prune_config_path_none_returns_defaults() -> None: - """``load_prune_config(None)`` returns :class:`PruneConfig` defaults - silently — no log, no error, every field at its DEC-009 default.""" - result = load_prune_config(None) +def _copy_fixture_as_default(fixture_name: str, project_dir: Path) -> None: + """Copy ``_FIXTURES / fixture_name`` to ``project_dir/signalforge.yml``. + + Used by tests that exercise the default-resolution path + (``load_prune_config(project_dir)``). + """ + shutil.copy(_FIXTURES / fixture_name, project_dir / "signalforge.yml") + + +def test_load_prune_config_no_file_returns_defaults(tmp_path: Path) -> None: + """``load_prune_config(project_dir)`` with no ``signalforge.yml`` in + ``project_dir`` returns :class:`PruneConfig` defaults silently — no + log, no error, every field at its DEC-009 default.""" + result = load_prune_config(tmp_path) assert result == PruneConfig() assert result.scope == "sample" @@ -43,22 +54,21 @@ def test_load_prune_config_path_none_returns_defaults() -> None: assert result.partition_filter is None -def test_load_prune_config_missing_file_returns_defaults(tmp_path: Path) -> None: - """A path that does not exist returns defaults without raising — - parity with :func:`load_draft_config` and - :func:`load_safety_config` for the implicit-config-file case.""" +def test_load_prune_config_missing_explicit_path_returns_defaults(tmp_path: Path) -> None: + """An explicit ``path=`` that does not exist returns defaults + without raising — parity with the project-default branch.""" missing = tmp_path / "missing.yml" assert not missing.exists() - result = load_prune_config(missing) + result = load_prune_config(tmp_path, missing) assert result == PruneConfig() -def test_load_prune_config_full_round_trip() -> None: +def test_load_prune_config_full_round_trip(tmp_path: Path) -> None: """A populated ``prune:`` block round-trips into a :class:`PruneConfig` whose every field matches the YAML.""" - result = load_prune_config(_FIXTURES / "signalforge_full.yml") + result = load_prune_config(tmp_path, _FIXTURES / "signalforge_full.yml") assert result.scope == "full" assert result.sample_size == 50_000 @@ -72,20 +82,33 @@ def test_load_prune_config_full_round_trip() -> None: assert result.partition_filter is None -def test_load_prune_config_minimal_block_returns_defaults() -> None: +def test_load_prune_config_default_path_resolves_relative_to_project_dir( + tmp_path: Path, +) -> None: + """``path=None`` resolves to ``/signalforge.yml`` — + parity with :func:`load_draft_config` and the safety-layer loader.""" + _copy_fixture_as_default("signalforge_full.yml", tmp_path) + + result = load_prune_config(tmp_path) + + assert result.scope == "full" + assert result.sample_size == 50_000 + + +def test_load_prune_config_minimal_block_returns_defaults(tmp_path: Path) -> None: """An empty ``prune: {}`` mapping validates as defaults — every field is optional with a default per DEC-009.""" - result = load_prune_config(_FIXTURES / "signalforge_minimal.yml") + result = load_prune_config(tmp_path, _FIXTURES / "signalforge_minimal.yml") assert result == PruneConfig() -def test_load_prune_config_typo_raises() -> None: +def test_load_prune_config_typo_raises(tmp_path: Path) -> None: """A typo'd inner field (``scop:`` instead of ``scope:``) raises :class:`PruneConfigError`. ``PruneConfig`` uses ``extra="forbid"`` per DEC-015 so silent no-op is impossible.""" with pytest.raises(PruneConfigError) as excinfo: - load_prune_config(_FIXTURES / "signalforge_typo.yml") + load_prune_config(tmp_path, _FIXTURES / "signalforge_typo.yml") # The wrapped pydantic ValidationError surfaces the offending key # name in the error message — the orchestrator / CLI uses this @@ -93,11 +116,11 @@ def test_load_prune_config_typo_raises() -> None: assert "scop" in str(excinfo.value) -def test_load_prune_config_tolerates_sibling_blocks() -> None: +def test_load_prune_config_tolerates_sibling_blocks(tmp_path: Path) -> None: """Sibling top-level keys (``safety:``, ``llm:``) are reserved for other stages per DEC-020 and silently ignored by the prune loader.""" - result = load_prune_config(_FIXTURES / "signalforge_with_siblings.yml") + result = load_prune_config(tmp_path, _FIXTURES / "signalforge_with_siblings.yml") assert result.scope == "sample" assert result.sample_size == 25_000 @@ -106,11 +129,11 @@ def test_load_prune_config_tolerates_sibling_blocks() -> None: assert result.total_budget_seconds == 600 -def test_load_prune_config_parses_partition_filter() -> None: +def test_load_prune_config_parses_partition_filter(tmp_path: Path) -> None: """``prune.partition_filter`` (a YAML mapping) is recursively validated into a typed :class:`PartitionFilter` ADT — Pydantic handles the dict→dataclass conversion.""" - result = load_prune_config(_FIXTURES / "signalforge_partition.yml") + result = load_prune_config(tmp_path, _FIXTURES / "signalforge_partition.yml") assert isinstance(result.partition_filter, PartitionFilter) assert result.partition_filter.column == "event_dt" @@ -118,10 +141,31 @@ def test_load_prune_config_parses_partition_filter() -> None: assert result.partition_filter.value == "2026-01-01" -def test_load_prune_config_yaml_safe_load_rejects_python_objects() -> None: - """A fixture containing a ``!!python/object/apply:os.system [...]`` - gadget raises rather than executing it — confirms the loader uses - :func:`yaml.safe_load`, not :func:`yaml.load` (which honours - arbitrary Python object construction tags).""" +def test_load_prune_config_yaml_safe_load_rejects_python_objects(tmp_path: Path) -> None: + """Confirms :func:`yaml.safe_load` is used (not :func:`yaml.load`). + + Under unsafe ``yaml.load`` / ``yaml.unsafe_load``, the + ``!!python/object/apply:os.system`` gadget invokes ``os.system`` + with a ``touch `` argument, creating an observable + side-effect file before validation runs. ``yaml.safe_load`` + rejects the construction tag at parse time, so the marker file + MUST NOT exist after the call. The test asserts both the typed + error AND the absence of the marker so a future regression that + swaps in unsafe ``yaml.load`` fails loud (the prior version of + this test would have passed in either case because the validator + rejected the resulting ``scope=0`` regardless of whether the + gadget executed). + """ + marker = tmp_path / "pwned" + config_yml = tmp_path / "signalforge.yml" + # `os.system("touch ")` returns 0; under unsafe yaml.load + # the marker file gets created as a side effect AND the parsed + # scope becomes the int 0 (which then fails the Literal["sample", + # "full"] validator → PruneConfigError). Under safe yaml.load, + # the construction tag is rejected at parse time → still + # PruneConfigError, BUT the marker is not created. + config_yml.write_text(f'prune:\n scope: !!python/object/apply:os.system ["touch {marker}"]\n') + with pytest.raises(PruneConfigError): - load_prune_config(_FIXTURES / "signalforge_unsafe.yml") + load_prune_config(tmp_path, config_yml) + assert not marker.exists(), "yaml.load executed the gadget — safe_load is NOT in use" diff --git a/tests/prune/test_engine.py b/tests/prune/test_engine.py index ba6cb73e..d468925e 100644 --- a/tests/prune/test_engine.py +++ b/tests/prune/test_engine.py @@ -25,6 +25,7 @@ from __future__ import annotations import json +import sys from pathlib import Path from typing import Any @@ -155,6 +156,7 @@ def test_prune_tests_always_passes_drops_test(tmp_path: Path) -> None: manifest, config=config, audit_path=audit_path, + project_dir=tmp_path, ) assert result.total_tests == 1 @@ -194,6 +196,7 @@ def test_prune_tests_kept_for_real_failure_untrusted_model(tmp_path: Path) -> No manifest, config=config, audit_path=audit_path, + project_dir=tmp_path, ) decision = result.decisions[0] @@ -228,6 +231,7 @@ def test_prune_tests_failed_on_known_clean_data_for_trusted_model(tmp_path: Path manifest, config=config, audit_path=audit_path, + project_dir=tmp_path, ) decision = result.decisions[0] @@ -277,6 +281,7 @@ def test_prune_tests_requires_future_data_for_unknown_relationships_parent( manifest, config=config, audit_path=audit_path, + project_dir=tmp_path, ) decision = result.decisions[0] @@ -315,6 +320,7 @@ def test_prune_tests_warehouse_error_routes_to_kept_without_evidence( manifest, config=config, audit_path=audit_path, + project_dir=tmp_path, ) decision = result.decisions[0] @@ -365,6 +371,7 @@ def fake_clock() -> int: manifest, config=config, audit_path=audit_path, + project_dir=tmp_path, ) assert result.total_tests == 5 @@ -406,6 +413,7 @@ def test_prune_tests_trusted_models_validation_at_entry(tmp_path: Path) -> None: manifest, config=config, audit_path=audit_path, + project_dir=tmp_path, ) assert excinfo.value.unique_id == "model.shop.nonexistent" @@ -456,6 +464,7 @@ def fake_write(*args: Any, **kwargs: Any) -> None: manifest, config=config, audit_path=audit_path, + project_dir=tmp_path, ) # The OSError surfaces as the cause on the typed wrapper. @@ -497,6 +506,7 @@ def fake_write(*args: Any, **kwargs: Any) -> None: manifest, config=config, audit_path=audit_path, + project_dir=tmp_path, ) # Same object — NOT wrapped. @@ -556,6 +566,7 @@ def test_prune_tests_compute_config_hash_is_deterministic(tmp_path: Path) -> Non manifest, config=config, audit_path=audit_a, + project_dir=tmp_path, ) prune_tests( model, @@ -564,6 +575,7 @@ def test_prune_tests_compute_config_hash_is_deterministic(tmp_path: Path) -> Non manifest, config=config, audit_path=audit_b, + project_dir=tmp_path, ) rows_a = _read_audit_lines(audit_a) @@ -572,3 +584,98 @@ def test_prune_tests_compute_config_hash_is_deterministic(tmp_path: Path) -> Non assert rows_a[0]["config_hash"] == rows_b[0]["config_hash"] # 16-hex-char convention (same as policy_hash / config_hash elsewhere). assert len(rows_a[0]["config_hash"]) == 16 + + +# --------------------------------------------------------------------------- +# QG fix-up: audit-path symlink-hardening (DEC-016) and project_dir-relative +# default resolution (mirrors the safety/draft layers' fix). +# --------------------------------------------------------------------------- + + +def test_prune_tests_default_audit_path_resolves_relative_to_project_dir( + tmp_path: Path, +) -> None: + """``audit_path=None`` resolves to + ``/.signalforge/prune.jsonl`` (NOT cwd-relative). + + Regression guard for the same defect the safety + draft layers + fixed: when the CLI is invoked from a sub-directory, the audit + lands next to the project, not next to wherever the user happened + to be. + """ + fake = FakeBigQueryClient(project="fake_project") + fake.expect_query(matching=r"SELECT COUNT\(\*\)", returns=[{"failures": 0}]) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig(capture_failure_rows=0) + + prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + project_dir=tmp_path, + ) + + expected_audit = tmp_path / ".signalforge" / "prune.jsonl" + assert expected_audit.exists() + rows = _read_audit_lines(expected_audit) + assert len(rows) == 1 + assert rows[0]["model_unique_id"] == "model.shop.orders" + fake.assert_all_expectations_met() + + +@pytest.mark.skipif(sys.platform == "win32", reason="symlink semantics differ on Windows") +def test_prune_tests_audit_path_symlink_outside_project_raises( + tmp_path: Path, +) -> None: + """A symlinked ``audit_path`` whose target escapes ``project_dir`` + raises :class:`PruneAuditWriteError` BEFORE any write hits disk + (DEC-016). + + Defence-in-depth: a malicious or misconfigured + ``.signalforge/prune.jsonl`` symlink could redirect writes to + ``/etc/passwd`` or any other attacker-controlled location. The + canonicalisation gate at orchestrator entry catches this and + surfaces a typed prune error rather than a raw OSError. + """ + project_dir = tmp_path / "project" + project_dir.mkdir() + audit_dir = project_dir / ".signalforge" + audit_dir.mkdir() + + outside_target = tmp_path / "outside" / "evil.jsonl" + outside_target.parent.mkdir() + # Symlink the audit file to a target outside the project tree. + audit_symlink = audit_dir / "prune.jsonl" + audit_symlink.symlink_to(outside_target) + + fake = FakeBigQueryClient(project="fake_project") + # No expectations registered — the canonicalisation gate must fire + # before any warehouse call. + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig(capture_failure_rows=0) + + with pytest.raises(PruneAuditWriteError): + prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_symlink, + project_dir=project_dir, + ) + + # The outside target was never created — canonicalisation aborts + # before the writer opens any file. + assert not outside_target.exists() + fake.assert_all_expectations_met() diff --git a/tests/prune/test_models.py b/tests/prune/test_models.py index e4a5f6e6..f8dddf45 100644 --- a/tests/prune/test_models.py +++ b/tests/prune/test_models.py @@ -132,3 +132,45 @@ def test_prune_decision_carries_typed_candidate_test() -> None: # discriminator dispatched the right concrete subclass assert isinstance(av_round.test, CandidateTestAcceptedValues) assert av_round.test.values == ("placed", "shipped") + + +def test_prune_result_repr_redacts_sql_and_sample_failures() -> None: + """DEC-022: PruneResult repr must NOT leak compiled_sql or sample rows. + + An accidental ``_LOGGER.warning("result: %s", result)`` with the + default Pydantic repr would dump the full per-decision payload — + including compiled SQL and any sampled-failure rows (which may + contain PII) — into log sinks. The custom repr restricts output to + top-level identity + count aggregates. + """ + decision = PruneDecision( + test_anchor="column.email", + test=CandidateTestNotNull(column="email"), + decision="kept", + reason="kept", + failures=3, + sampled_rows=100, + scope="sample", + elapsed_ms=42, + compiled_sql_hash="abc123", + compiled_sql="SELECT secret_column FROM internal_table WHERE 1=1", + why="3 failures", + sample_failures=({"secret_column": "leaked-pii-value"},), + ) + result = PruneResult( + model_unique_id="model.shop.users", + decisions=(decision,), + elapsed_ms=42, + signalforge_version="0.1.0.dev0", + ) + rendered = repr(result) + # Allowed (top-level identity + counts): + assert "model.shop.users" in rendered + assert "kept_count=1" in rendered + assert "dropped_count=0" in rendered + # Forbidden (sensitive bodies): + assert "secret_column" not in rendered + assert "internal_table" not in rendered + assert "leaked-pii-value" not in rendered + assert "compiled_sql" not in rendered + assert "sample_failures" not in rendered From 74d67b90000c397802373774e21f903ed3ed118e Mon Sep 17 00:00:00 2001 From: wjduenow Date: Thu, 30 Apr 2026 23:42:35 -0700 Subject: [PATCH 21/24] bd_1-scaffolding-y8y.16: distill prune-layer rules into .claude/rules New .claude/rules/prune-engine.md captures the load-bearing conventions established by issue #6: * kept-without-evidence routes to decision="kept" (DEC-011) * fail-closed prune.jsonl audit (DEC-016) * symlink-hardened audit path via canonicalise_path (post-QG) * identifier shape validation at compile seam (post-QG) * dialect-driven compiler (no BigQuery-isms) * single AST scan per new audit-event type (DEC-018) * ANSI-safe lazy-format logger + grep gate (DEC-017) * custom __repr__ on result-shaped models (post-QG, DEC-022) * drift detectors mandatory for extra="ignore" models (DEC-010) * API alignment with adjacent stages (load_*_config signature, project-dir-relative default paths) * signalforge.yml prune: namespace (DEC-020) CLAUDE.md cross-references the new rule file. Plan: plans/super/6-prune-engine.md. Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/rules/prune-engine.md | 132 ++++++++++++++++++++++++++++++++++ CLAUDE.md | 2 +- 2 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 .claude/rules/prune-engine.md diff --git a/.claude/rules/prune-engine.md b/.claude/rules/prune-engine.md new file mode 100644 index 00000000..8154a693 --- /dev/null +++ b/.claude/rules/prune-engine.md @@ -0,0 +1,132 @@ +# Prune layer (decision routing + fail-closed audit) + +Established by issue #6 (test prune engine). Apply to every module under `signalforge.prune` and to any new code that compiles a candidate test to SQL, runs it against the warehouse, or writes a prune-decision audit record. + +The prune layer sits between the LLM-drafting pipeline (#5) and the quality grader (#7). It enforces SignalForge's load-bearing "signal over volume" commitment: a candidate test that always passes on warehouse samples must be dropped, not shipped. Equally, a candidate test we cannot evaluate must be kept — pruning is allowed to drop only with positive evidence. + +## Conservative drop-reason taxonomy (DEC-006, DEC-011) + +`DropReason` is a `Literal[...]` of exactly five values: + +- `"always-passes"` — test ran, returned zero failing rows on a representative sample, scope was sufficient. Drop. +- `"failed-on-known-clean-data"` — test ran against a `trusted_models` opt-in target and returned non-zero failing rows. The model is known clean; the test is wrong. Drop. +- `"requires-future-data"` — test compiled to a sentinel (e.g. `relationships(to: )` where the target model is absent from the manifest). Cannot evaluate; ship it for the operator to revisit when the dependency lands. Kept. +- `"kept"` — test ran and returned non-zero failing rows on an untrusted model. Real signal. Kept. +- `"kept-without-evidence"` — total budget elapsed before this test ran, the test SQL was rejected by `validate_identifier`, the warehouse call raised, or any other "cannot positively evaluate" outcome. Kept. + +The decision matrix lives in `engine.py::_classify_decision`. The load-bearing invariant: **kept-without-evidence routes to `decision="kept"`, not `decision="dropped"`.** SignalForge ships tests it cannot evaluate; the LLM proposed them, and absent contradicting warehouse evidence the operator gets to make the call. Dropping a test silently because we couldn't reach the warehouse is exactly the failure mode this commitment exists to prevent. + +If you add a sixth `DropReason` literal in v0.2 (e.g. `"sample-too-small"`), update production `DropReason` AND `StrictPruneDecision` (the drift detector) AND the fixture at `tests/fixtures/prune/prune_event_v1.jsonl` AND the decision-matrix table in `docs/prune-ops.md` in the same change. + +## Fail-closed audit (DEC-016, mirrors safety/draft) + +`signalforge.prune.audit.write_prune_event` is the project's third fail-closed JSONL writer (after `signalforge.safety.audit.write` and `signalforge.draft.audit.write_response_event`). The contract is identical to those — load-bearing rules: + +1. **Propagation IS the defence.** Open with `O_APPEND | O_CREAT | 0o600`, write one JSONL line, `os.fsync`, close. Catches **no** exceptions internally — `OSError`, `PermissionError`, encoding failures all propagate. The caller (`prune_tests`) wraps as `PruneAuditWriteError(cause=...)`. **Don't** add try/except inside the writer; the propagation is the contract. + +2. **Size cap before any file open.** `_PRUNE_AUDIT_RECORD_LIMIT_BYTES = 4000` is checked before the `os.open`, so an oversize record leaves no on-disk artefact. Raises `PruneAuditRecordTooLargeError(size, limit)` which the orchestrator propagates as-is rather than re-wrapping. + +3. **Per-decision write happens after each test, not in a final batch.** A `prune_tests` run that crashes mid-iteration leaves the prune.jsonl with one durable record per evaluated test up to the failure point. Buffering writes "to amortise fsync" defeats the audit guarantee. + +An unaudited prune decision is, by definition, a `kept`/`dropped` verdict whose justification didn't durably hit disk — exactly the failure mode this layer exists to prevent. The fail-closed pattern is now established in three layers; treat it as the project default for any future audit-write seam. + +## Symlink-hardened audit path (DEC-016, post-QG fix) + +`signalforge.warehouse._path_safety.canonicalise_path` is the project's standard symlink/containment gate. Apply it to every user-supplied or default-derived path that the writer opens. + +The original prune-audit implementation opened `/.signalforge/prune.jsonl` directly without canonicalising. The QG review caught it: a symlinked `.signalforge/prune.jsonl -> /etc/passwd` (admittedly hostile, but the reader's path safety enforces this gate uniformly elsewhere) would have written outside the project tree. The fix routes the audit path through `canonicalise_path` at writer entry, raising `PruneAuditWriteError` on containment failure. + +When introducing a fourth audit-write seam — grader (#7), diff renderer (#8), or beyond — apply `canonicalise_path` at writer entry. Don't trust the default path because it's "ours"; the reader's three traps from `manifest-readers.md` apply equally to writers. + +## Identifier shape validation at the compile seam (DEC-024 + post-QG fix) + +LLM-supplied `test.column` / `test.field` strings land in compiled SQL as backtick-quoted identifiers. The drafter's anchor-contract validator (`llm-drafter.md` DEC-003) checks set-membership against `Manifest.columns` but does NOT enforce regex shape — a manifest column named `foo bar` (whitespace) or `users\`; DROP TABLE` (backtick injection) passes the anchor check. + +The compile seam is the layer's last line of defence. `signalforge.prune.compiler` calls `signalforge.warehouse._sql_safety.validate_identifier` on every identifier (`test.column`, `test.field`; `accepted_values.values` go through `escape_bq_string_literal` per DEC-024) **before** quoting. Failures return a sentinel `_InvalidIdentifier` which the engine routes to `kept-without-evidence` (`why="identifier rejected by SQL safety check"`). + +Defence in depth — same principle as the warehouse adapter's DEC-013 (every public-API identifier validated at construction time). Don't skip the compile-seam validation on the assumption that the manifest is trusted; the manifest reader uses `extra="ignore"` for forward-compat and was not designed to enforce identifier shape on user-authored YAML. + +## Compiler is dialect-driven, not BigQuery-specific (DEC-025) + +`_compile_test(test, table_ref, dialect: Dialect, manifest)` reads `dialect.quote_char`, `dialect.identifier_case`, and `dialect.supports_qualify` from the `Dialect` value object returned by `WarehouseAdapter.dialect()`. **No `from google.cloud import bigquery` anywhere in `signalforge/prune/`.** v0.2 Snowflake/Postgres adapters return their own `Dialect` and the compiler routes accordingly without modification. + +Mirrors the architectural commitment from `warehouse-adapters.md`: keep BigQuery-isms behind the adapter seam from day one. The `relationships` parent join uses `TableRef.from_model` for the same reason (DEC-026) — the table-reference shape is dialect-neutral. + +The NULL-exclusion pattern in compiled SQL matches dbt-core verbatim (DEC-023): `unique` adds `WHERE col IS NOT NULL`; `accepted_values` adds `WHERE col IS NOT NULL AND col NOT IN (...)`; `relationships` adds child-side `WHERE child.col IS NOT NULL`. Snapshot fixtures pin the exact SQL bytes; divergence from dbt-core verdicts is a regression even if it appears semantically equivalent. + +## Total-budget semantics (DEC-011) + +When `total_budget_seconds` is exceeded, the orchestrator: + +1. Best-effort cancels the in-flight test (`query_job.cancel()` on BigQuery; ABC-level no-op fallback for adapters that don't expose cancel). +2. Marks every remaining un-started test as `kept-without-evidence` with `why="total prune budget exceeded before evaluation"`. +3. Emits a final WARNING with the count of un-started tests (`json.dumps`-formatted). + +No partial-evaluation results — a test that was running when the budget tripped is `kept-without-evidence`, not `kept` (the failing-rows count is unknown). The single-threaded sequential model (DEC-028) makes this enforceable; v0.2 concurrency will need a different cancellation contract. + +`signalforge.prune.engine` declares `_sleep = time.sleep` at module scope per DEC-019 so the budget-watchdog tests reassign it to a deterministic stand-in. Mirrors `llm-drafter.md` DEC-004; same rationale (don't monkey-patch `time.sleep` globally). + +## Single AST scan extension per new audit type (DEC-018) + +`tests/test_audit_completeness.py` was the unified AST-scan suite as of issue #5; issue #6 extends it with a fifth scan: `Call(func=Name(id="PruneEvent"))` is permitted only in `src/signalforge/prune/audit.py`. The sanity check that *at least one* construction exists in the blessed module guards against accidental rename-without-update. + +The pattern: whenever a new audit-event type is added — whether for prune, grade (#7), diff (#8), or beyond — extend `tests/test_audit_completeness.py` with a sixth/seventh/etc. AST scan that gates the type's construction to one module. Mirrors `safety.AuditEvent` and `draft.LLMResponseEvent` precedent. + +If a new module genuinely needs to construct a gated event-type (e.g., a deserialiser for resumption), update the scan's exclusion list AND document the audit-write seam. **Don't suppress the test** — the entire point of the AST scan is to catch unaudited audit-event constructions, and a one-line exclusion entry is cheaper than the hour you'll spend debugging a missing JSONL line. + +## ANSI-safe lazy-format JSON logger + grep gate (DEC-017) + +Same rule as `safety-layer.md` DEC-022 and `llm-drafter.md` DEC-011 — never f-string-interpolate user-controlled strings into a `_LOGGER` call: + +```python +_LOGGER.warning("budget exceeded: %s", json.dumps({"unstarted_count": n, "model": model_unique_id})) +``` + +**Never** `_LOGGER.warning(f"budget exceeded: {n} tests for {model_unique_id}")`. A model unique_id containing ANSI escapes (`\x1b[31m...`) would inject into log viewers. JSON encoding handles this; f-string interpolation does not. + +The grep gate at `tests/llm/test_logger_grep_gate.py` scans `src/signalforge/{llm,draft,prune}` and rejects any `_LOGGER\.\w+\(f"` hit. Extend the scan to a fourth directory when the grader (#7) ships, rather than copy-pasting a per-layer gate; the single test is the source of truth. + +## Custom `__repr__` on result-shaped models (DEC-022, post-QG fix) + +Pydantic v2's default `__repr__` emits every field. `PruneResult` carries the full decision tuple, each `PruneDecision` carries `compiled_sql` and `sample_failures` — an accidental `_LOGGER.warning("result: %s", result)` in a future caller would dump megabytes of SQL and (potentially) sample-row contents to the log. + +`PruneResult.__repr__` shows only `model_unique_id`, `kept_count`, `dropped_count`, `elapsed_ms`. `PruneDecision.__repr__` shows only `test.test_type`, `test.column` (or sentinel `""`), `decision`, `reason`. Compiled SQL and sample failures stay accessible via field access; they just don't slip out the casual debug-print path. + +Apply to any future result-shaped model whose fields include user-content payloads. The pattern is "minimal `__repr__`; rich access via fields" — don't override `__str__` (Pydantic uses it for serialisation). + +User-supplied strings in error messages render via `repr()` (`_format_value` helper) — same DEC-022 rule applies to `PruneError` subclasses. A model unique_id containing `\x1b[31m` quoted via `repr()` shows as `'\\x1b[31m...'` in error output; raw interpolation would inject. + +## Drift detectors are mandatory for read-back models (DEC-010) + +Every `extra="ignore"` production model — `PruneResult`, `PruneDecision`, `PruneEvent` — pairs with a `Strict(extra="forbid")` detector in `tests/prune/test_drift_detector.py`, validated against a committed JSON/JSONL fixture (`tests/fixtures/prune/prune_event_v1.jsonl` for the audit type). Adding a field to production without updating the strict mirror OR the fixture breaks the test loudly. + +The QG review for #6 caught one near-miss where a production field landed without updating the strict mirror — the drift detector flagged it before it merged. Bake the rule: production `extra="ignore"` change = strict-model change = fixture refresh, in the same commit. + +The `extra=` placement convention from `safety-layer.md` DEC-015 applies verbatim: + +- `PruneConfig`, `_PruneConfigFile` → `extra="forbid"` (config-shaped; typos like `scop:` must fail loud). +- `PruneResult`, `PruneDecision`, `PruneEvent` → `extra="ignore"` (read-back shapes; forward-compat matters). + +## API alignment with adjacent stages (post-QG fix) + +`load_prune_config(project_dir, path=None) -> PruneConfig` matches the signature of `load_safety_config` and `load_draft_config`. The default `audit_path` resolves relative to `project_dir` (`/.signalforge/prune.jsonl` per DEC-016). Same for the orchestrator entry: `prune_tests(model, adapter, candidates, manifest, *, config=None, audit_path=None)` — keyword-only optionals, model-and-adapter front-paired, mirroring `draft_schema(model, adapter, policy, manifest, *, config)` from `llm-drafter.md`. + +The CLI (#9) and any future orchestrator wants one calling convention; new stage configs and orchestrators must match the precedent. If you find yourself writing `load_grade_config(*, project_root)` or `grade_artifacts(adapter, model, ...)` (adapter-first), you've broken the alignment. Match the existing seam. + +## `signalforge.yml` top-level namespace: `prune:` (DEC-020) + +The prune-stage block is `{ prune: { audit_path, mode, total_budget_seconds, test_timeout_seconds, partition_filter, trusted_models, ... } }`. Sibling top-level keys (`safety:`, `llm:`, future `grade:`/`diff:`) are reserved for other stages and silently ignored by the prune loader. + +`PruneConfig` uses `extra="forbid"` (config-shaped; typos fail loud). The wrapping `_PruneConfigFile` uses `extra="ignore"` at the top level so unknown sibling stages don't break the loader. Mirrors `llm-drafter.md` DEC-027 / `safety-layer.md` DEC-025 verbatim. + +When introducing a new pipeline-stage config, claim its own top-level key. Don't pile under `prune:` — each stage's behaviour-knob block stays separate, and the v2-config migration cost grows linearly with the number of stages whose config got merged. + +## `trusted_models` is validated at orchestrator entry, not at config load (DEC-008) + +`PruneConfig.trusted_models: tuple[str, ...]` is a tuple of `unique_id` strings. It is **not** validated at `load_prune_config(...)` time — the manifest isn't loaded yet, so we can't check membership. Validation happens at `prune_tests(...)` entry: every entry must appear in `manifest.models`; otherwise raises `PruneTrustedModelNotFoundError(unique_id=...)` **before any warehouse call**. + +A typo like `trusted_models: ["model.proj.cusotmers"]` (vs. `customers`) MUST fail loud — silent no-op (the model isn't trusted because it doesn't match anything) is exactly the failure mode that leaks "failed-on-known-clean-data" tests through as `kept`. Apply the same "validate-against-manifest at orchestrator entry" pattern to any future config field that references manifest entities by `unique_id`. + +## Reference + +`plans/super/6-prune-engine.md` — DEC-001 … DEC-028. `src/signalforge/prune/` — current implementation. `docs/prune-ops.md` — operational reference. `tests/prune/test_drift_detector.py` — schema-drift gate. `tests/test_audit_completeness.py` — AST-scan suite (5 scans as of #6). `tests/llm/test_logger_grep_gate.py` — lazy-format logger gate (3 dirs as of #6). `tests/fixtures/prune/prune_event_v1.jsonl` — committed audit fixture for the drift detector. diff --git a/CLAUDE.md b/CLAUDE.md index 4489bf32..1a17ba10 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ Pre-alpha. Six issues shipped: - **#3 (BigQuery warehouse adapter)** — `signalforge.warehouse` subpackage: `WarehouseAdapter` ABC + `from_profile` factory, `BigQueryAdapter` (the only v0.1 concrete adapter), `load_profile` for dbt `profiles.yml`, deterministic hash-mod sampling with fail-loud sizing checks, identifier-validation at construction time, `QueryJobConfig` defaults that pin `use_query_cache=False`. See `docs/warehouse-adapter-ops.md` for the operational reference. - **#4 (PII safety layer)** — `signalforge.safety` subpackage: schema-only-default sampling-mode policy, fail-closed audit JSONL writer (`O_APPEND` + `fsync` + size cap), column-name redaction via stable blake2b-4 hashes (DEC-010), four opt-out signals (column meta, model meta, `tags:[pii]`, `meta.contains_pii`) with documented precedence, AST audit-completeness scan rejecting direct `LLMRequest` construction, drift-detector test for `AuditEvent`. `signalforge.yml` config namespace `{ safety: { ... } }` reserved for the safety layer. See `docs/safety-ops.md` for the operational reference and `.claude/rules/safety-layer.md` for the rules distilled from this ticket. - **#5 (LLM draft pipeline)** — `signalforge.draft` + `signalforge.llm` subpackages: centralized `call_anthropic` SDK seam (full retry taxonomy via clauditor pattern), `draft_schema(model, adapter, policy, manifest, *, config)` end-to-end drafter, typed `CandidateSchema` + discriminated-union `CandidateTest`, anchor-contract validator (every test references a real column; whole-draft fail-loud), fail-closed `LLMResponseEvent` JSONL response audit adjacent to safety audit, `` prompt-injection envelope, deterministic `prompt_version` hash. See `docs/draft-ops.md`. -- **#6 (test prune engine)** — `signalforge.prune` subpackage: `prune_tests(model, adapter, candidates, manifest, *, config=None, audit_path=None) -> PruneResult` end-to-end orchestrator. Compiles dbt-style tests (`not_null`, `unique`, `accepted_values`, `relationships`) to BigQuery failing-rows SQL via `Dialect.quote_char`-driven dispatch (so v0.2 Snowflake/Postgres slot in unchanged), routes outcomes through five `DropReason` literals (`always-passes`, `requires-future-data`, `failed-on-known-clean-data`, `kept`, `kept-without-evidence`), enforces a total wall-clock budget (DEC-011), emits a fail-closed `prune.jsonl` audit (mirrors safety / draft fail-closed semantics; AST-gated single construction seam). Trusted-models opt-in via `signalforge.yml prune.trusted_models: [...]`, validated against the manifest at orchestrator entry (DEC-008). See `docs/prune-ops.md` for the operational reference and `plans/super/6-prune-engine.md` for the design. +- **#6 (test prune engine)** — `signalforge.prune` subpackage: `prune_tests(model, adapter, candidates, manifest, *, config=None, audit_path=None) -> PruneResult` end-to-end orchestrator. Compiles dbt-style tests (`not_null`, `unique`, `accepted_values`, `relationships`) to BigQuery failing-rows SQL via `Dialect.quote_char`-driven dispatch (so v0.2 Snowflake/Postgres slot in unchanged), routes outcomes through five `DropReason` literals (`always-passes`, `requires-future-data`, `failed-on-known-clean-data`, `kept`, `kept-without-evidence`), enforces a total wall-clock budget (DEC-011), emits a fail-closed `prune.jsonl` audit (mirrors safety / draft fail-closed semantics; AST-gated single construction seam). Trusted-models opt-in via `signalforge.yml prune.trusted_models: [...]`, validated against the manifest at orchestrator entry (DEC-008). See `docs/prune-ops.md` for the operational reference, `plans/super/6-prune-engine.md` for the design, and `.claude/rules/prune-engine.md` for the rules distilled from this ticket. Design is happening in the open on the `dev` branch; remaining feature work (grader #7, diff renderer #8, CLI #9) lands next. From e5fb7cf52645027efc38a3be6c2a12fbb9c7acd1 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Fri, 1 May 2026 08:03:26 -0700 Subject: [PATCH 22/24] 6/QG: wire prune scope/sample_size/partition_filter into compiled SQL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #20 review (Copilot + CodeRabbit) flagged that PruneConfig.scope was advisory-only — the engine wrote decision.scope into audit metadata but never wrapped the failing-rows SQL with the deterministic-sample CTE. Sample-mode tests ran against the FULL table regardless of config, defeating the cost model US-003 was built on. The compiler is now the single seam for the wrapping. _compile_test accepts scope, sample_size, sample_bucket, and partition_filter; sample mode wraps the test in a `WITH sample AS (SELECT * FROM AS t WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), ) < 1 [AND ] LIMIT ) ` compound matching the warehouse adapter's sample_rows shape (DEC-006 of issue #3). Full mode + a partition_filter composes via a derived table so per-test WHERE-clause shapes don't have to be edited. The orchestrator computes sample_bucket once per run from Table.num_rows / sample_size via the SDK shim seam (acknowledged minor encapsulation crack, documented inline; v0.2 may add a public WarehouseAdapter.get_table_metadata seam). When num_rows is unavailable in sample mode the orchestrator raises PruneError rather than silently degrade to "every row" — that fail-loud signal lands in front of the operator. Sample-mode relationships samples the CHILD only (the parent stays at full so an orphan detected in the child sample is not a false positive caused by the parent's missing-from-sample row). Also fixes the relationships parent-resolution ambiguity case: when two or more manifest models share Model.name (cross-package collision), the compiler returned the FIRST match silently. It now returns _RequiresFutureData with a precise count, routing the test to "requires-future-data" rather than picking a wrong parent. Pinned snapshot fixtures for each of the four candidate-test variants in sample mode (not_null_sample.sql, unique_sample.sql, accepted_values_sample.sql, relationships_sample.sql). 15 new tests across compiler + engine; existing tests (which were implicit-default sample-mode but tested full-mode SQL shapes) updated to scope="full" so they continue to test the legacy unwrapped path. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/prune/compiler.py | 315 +++++++++++++++++- src/signalforge/prune/engine.py | 99 +++++- .../compiled_sql/accepted_values_sample.sql | 1 + .../prune/compiled_sql/not_null_sample.sql | 1 + .../compiled_sql/relationships_sample.sql | 1 + .../prune/compiled_sql/unique_sample.sql | 1 + tests/prune/test_compiler.py | 204 +++++++++++- tests/prune/test_engine.py | 290 +++++++++++++++- 8 files changed, 880 insertions(+), 32 deletions(-) create mode 100644 tests/fixtures/prune/compiled_sql/accepted_values_sample.sql create mode 100644 tests/fixtures/prune/compiled_sql/not_null_sample.sql create mode 100644 tests/fixtures/prune/compiled_sql/relationships_sample.sql create mode 100644 tests/fixtures/prune/compiled_sql/unique_sample.sql diff --git a/src/signalforge/prune/compiler.py b/src/signalforge/prune/compiler.py index 1c87c695..172245bf 100644 --- a/src/signalforge/prune/compiler.py +++ b/src/signalforge/prune/compiler.py @@ -51,6 +51,7 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import date, datetime from hashlib import blake2b from typing import TYPE_CHECKING @@ -63,10 +64,11 @@ ) from signalforge.warehouse._sql_safety import escape_bq_string_literal, validate_identifier from signalforge.warehouse.errors import InvalidIdentifierError -from signalforge.warehouse.models import TableRef +from signalforge.warehouse.models import PartitionFilter, TableRef if TYPE_CHECKING: from signalforge.manifest.models import Manifest + from signalforge.prune.models import Scope from signalforge.warehouse.models import Dialect @@ -106,6 +108,67 @@ class _InvalidIdentifier: reason: str +def _render_partition_filter(pf: PartitionFilter, quote_char: str) -> str: + """Render a :class:`PartitionFilter` to a SQL fragment for the WHERE clause. + + Mirrors :meth:`signalforge.warehouse.adapters.bigquery.BigQueryAdapter._render_partition_filter` + — the prune compiler reuses the same render rules so the partition + predicate the engine threads through the deterministic sample CTE + matches what the warehouse adapter would emit on its own + ``sample_rows`` path. + + ``datetime`` → ``TIMESTAMP('…')``; ``date`` → ``DATE('…')``; + ``str`` is escaped via :func:`escape_bq_string_literal` for safe + inclusion inside a single-quoted BigQuery string literal. The column + name is already DEC-013-validated by :class:`PartitionFilter`'s + ``__post_init__``. + """ + # ``datetime`` is a subclass of ``date``, so check it first. + if isinstance(pf.value, datetime): + rendered = f"TIMESTAMP('{pf.value.isoformat()}')" + elif isinstance(pf.value, date): + rendered = f"DATE('{pf.value.isoformat()}')" + else: + rendered = f"'{escape_bq_string_literal(str(pf.value))}'" + return f"{quote_char}{pf.column}{quote_char} {pf.op} {rendered}" + + +def _render_sample_cte( + table: str, + *, + sample_size: int, + sample_bucket: int, + partition_filter: PartitionFilter | None, + quote_char: str, +) -> str: + """Render a deterministic-sample CTE matching the adapter's + :meth:`signalforge.warehouse.adapters.bigquery.BigQueryAdapter.sample_rows` + SQL shape. + + Every sample-mode failing-rows query is wrapped as:: + + WITH sample AS ( + SELECT * FROM
AS t + WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), ) < 1 + [AND ] + LIMIT + ) + + + The hash-mod predicate is identical to the adapter's + ``sample_rows`` deterministic-sample shape (DEC-006 of issue #3) so + sampling decisions stay consistent between the adapter's internal + samples and the prune engine's wrapped tests. + """ + where_clauses = [ + f"MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), {sample_bucket}) < 1", + ] + if partition_filter is not None: + where_clauses.append(_render_partition_filter(partition_filter, quote_char)) + where_sql = " AND ".join(where_clauses) + return f"WITH sample AS (SELECT * FROM {table} AS t WHERE {where_sql} LIMIT {sample_size})" + + def _quote(identifier: str, quote_char: str) -> str: """Wrap ``identifier`` in ``quote_char`` for SQL embedding. @@ -134,10 +197,88 @@ def _qualified_table_name(table_ref: TableRef, quote_char: str) -> str: return f"{quote_char}{table_ref.project}.{table_ref.dataset}.{table_ref.name}{quote_char}" +def _wrap_with_sample_or_partition( + *, + test_sql: str, + table_sql: str, + table_alias_sql: str, + scope: Scope, + sample_size: int | None, + sample_bucket: int | None, + partition_filter: PartitionFilter | None, + quote_char: str, +) -> str: + """Apply scope/sample/partition_filter wrapping to a per-test SELECT. + + ``test_sql`` is the failing-rows SELECT rendered against + ``table_alias_sql`` (typically the same string as ``table_sql`` for + the unwrapped path; substituted to ``sample`` when wrapping). The + function returns: + + * ``scope == "sample"`` — a ``WITH sample AS (...) `` + compound where the test targets the CTE rather than the raw table. + The deterministic-sample predicate matches the adapter's + :meth:`sample_rows` shape (DEC-006 of issue #3). + * ``scope == "full"`` AND ``partition_filter is not None`` — the + partition predicate is appended to the test's existing WHERE clause + (or added as a new WHERE when the test has none). The caller is + responsible for emitting a test SQL that already targets the raw + table with a ``WHERE`` clause; partition-only injection assumes the + caller never emits ``WHERE`` for ``unique`` (it does emit one for + ``not_null`` / ``accepted_values`` / ``relationships``). To avoid + that fragility this helper composes the predicate via subquery: the + caller's ``test_sql`` runs against a derived ``(SELECT * FROM +
WHERE ) AS t`` rather than the raw table. + * ``scope == "full"`` AND ``partition_filter is None`` — returns + ``test_sql`` unchanged. + + The partition-via-subquery composition is uniform across all four + test shapes (no per-test ``WHERE``-clause surgery), which keeps the + helper a true wrapper. + """ + del table_alias_sql # currently unused — kept in signature for symmetry / future use + if scope == "sample": + if sample_size is None or sample_bucket is None: + # Defensive: the orchestrator must supply both when scope=sample. + # Falling back silently would defeat US-003's cost model. + raise ValueError( + "scope='sample' requires both sample_size and sample_bucket; " + "the orchestrator should have computed these before calling _compile_test." + ) + cte = _render_sample_cte( + table_sql, + sample_size=sample_size, + sample_bucket=sample_bucket, + partition_filter=partition_filter, + quote_char=quote_char, + ) + return f"{cte} {test_sql}" + # scope == "full" + if partition_filter is not None: + partition_sql = _render_partition_filter(partition_filter, quote_char) + # Compose via derived table so per-test WHERE-clause shapes don't + # have to be edited. Every per-test compiler emits exactly one + # ``FROM `` fragment for the primary (child) table. + # ``relationships`` emits a second ``LEFT JOIN `` + # which is intentionally NOT rewritten: the partition filter + # applies to the model under prune (the child), not its + # referenced parent. + needle = f"FROM {table_sql}" + replacement = f"FROM (SELECT * FROM {table_sql} WHERE {partition_sql})" + if needle in test_sql: + return test_sql.replace(needle, replacement, 1) + return test_sql + + def _compile_not_null( test: CandidateTestNotNull, table_ref: TableRef, quote_char: str, + *, + scope: Scope, + sample_size: int | None, + sample_bucket: int | None, + partition_filter: PartitionFilter | None, ) -> str | _InvalidIdentifier: """Compile ``not_null(col)`` to ``SELECT col FROM t WHERE col IS NULL``.""" try: @@ -150,13 +291,29 @@ def _compile_not_null( ) col = _quote(test.column, quote_char) table = _qualified_table_name(table_ref, quote_char) - return f"SELECT {col} FROM {table} WHERE {col} IS NULL" + target = "sample" if scope == "sample" else table + test_sql = f"SELECT {col} FROM {target} WHERE {col} IS NULL" + return _wrap_with_sample_or_partition( + test_sql=test_sql, + table_sql=table, + table_alias_sql=target, + scope=scope, + sample_size=sample_size, + sample_bucket=sample_bucket, + partition_filter=partition_filter, + quote_char=quote_char, + ) def _compile_unique( test: CandidateTestUnique, table_ref: TableRef, quote_char: str, + *, + scope: Scope, + sample_size: int | None, + sample_bucket: int | None, + partition_filter: PartitionFilter | None, ) -> str | _InvalidIdentifier: """Compile ``unique(col)`` to a GROUP BY ... HAVING COUNT(*) > 1. @@ -174,13 +331,31 @@ def _compile_unique( ) col = _quote(test.column, quote_char) table = _qualified_table_name(table_ref, quote_char) - return f"SELECT {col} FROM {table} WHERE {col} IS NOT NULL GROUP BY {col} HAVING COUNT(*) > 1" + target = "sample" if scope == "sample" else table + test_sql = ( + f"SELECT {col} FROM {target} WHERE {col} IS NOT NULL GROUP BY {col} HAVING COUNT(*) > 1" + ) + return _wrap_with_sample_or_partition( + test_sql=test_sql, + table_sql=table, + table_alias_sql=target, + scope=scope, + sample_size=sample_size, + sample_bucket=sample_bucket, + partition_filter=partition_filter, + quote_char=quote_char, + ) def _compile_accepted_values( test: CandidateTestAcceptedValues, table_ref: TableRef, quote_char: str, + *, + scope: Scope, + sample_size: int | None, + sample_bucket: int | None, + partition_filter: PartitionFilter | None, ) -> str | _InvalidIdentifier: """Compile ``accepted_values(col, values)`` to a ``NOT IN`` predicate. @@ -203,8 +378,21 @@ def _compile_accepted_values( ) col = _quote(test.column, quote_char) table = _qualified_table_name(table_ref, quote_char) + target = "sample" if scope == "sample" else table rendered_values = ", ".join(f"'{escape_bq_string_literal(v)}'" for v in test.values) - return f"SELECT {col} FROM {table} WHERE {col} IS NOT NULL AND {col} NOT IN ({rendered_values})" + test_sql = ( + f"SELECT {col} FROM {target} WHERE {col} IS NOT NULL AND {col} NOT IN ({rendered_values})" + ) + return _wrap_with_sample_or_partition( + test_sql=test_sql, + table_sql=table, + table_alias_sql=target, + scope=scope, + sample_size=sample_size, + sample_bucket=sample_bucket, + partition_filter=partition_filter, + quote_char=quote_char, + ) def _resolve_parent_table_ref( @@ -216,20 +404,36 @@ def _resolve_parent_table_ref( The drafter's :class:`CandidateTestRelationships.to` field carries only the parent model's :attr:`Model.name` (not a full ``unique_id``); :class:`Manifest` indexes by ``unique_id``, so the lookup scans - :attr:`Manifest.nodes` for a model whose :attr:`Model.name` matches. + :attr:`Manifest.nodes` for every model whose :attr:`Model.name` + matches. - Returns a :class:`_RequiresFutureData` sentinel when no match is - found (DEC-026). When a match is found, returns the parent's + Returns a :class:`_RequiresFutureData` sentinel when: + + * No match is found (parent is absent from the manifest — DEC-026). + * Two or more models in the manifest share ``parent_name`` (e.g. + multiple packages with a ``customers`` model). The compiler does + not have enough information to disambiguate; routing to + ``requires-future-data`` ships the test to the operator with a + precise diagnostic rather than silently picking a parent. + + When exactly one match is found, returns the parent's :class:`TableRef` via :meth:`TableRef.from_model`; that call may raise :class:`ManifestProjectNotFoundError` or :class:`ManifestSchemaNotFoundError` if the parent model lacks ``database`` / ``schema`` — those are manifest-shape problems and propagate, not prune problems to swallow. """ - for parent_model in manifest.nodes.values(): - if parent_model.name == parent_name: - return TableRef.from_model(parent_model) - return _RequiresFutureData(reason=f"relationships parent {parent_name!r} not in manifest") + matches = [m for m in manifest.nodes.values() if m.name == parent_name] + if not matches: + return _RequiresFutureData(reason=f"relationships parent {parent_name!r} not in manifest") + if len(matches) > 1: + return _RequiresFutureData( + reason=( + f"relationships parent {parent_name!r} ambiguous: " + f"matched {len(matches)} models in manifest" + ) + ) + return TableRef.from_model(matches[0]) def _compile_relationships( @@ -237,6 +441,11 @@ def _compile_relationships( table_ref: TableRef, quote_char: str, manifest: Manifest, + *, + scope: Scope, + sample_size: int | None, + sample_bucket: int | None, + partition_filter: PartitionFilter | None, ) -> str | _RequiresFutureData | _InvalidIdentifier: """Compile ``relationships(child_col, to=parent, field=parent_col)``. @@ -253,6 +462,12 @@ def _compile_relationships( routes that to ``kept-without-evidence``. ``to`` is NOT shape-checked here — it's a model name resolved via :func:`_resolve_parent_table_ref` and a missing parent yields the ``_RequiresFutureData`` branch. + + Sample-mode asymmetry: when ``scope == "sample"``, only the CHILD + table is sampled. The parent stays at full so an orphan detected in + the child sample is not a false positive caused by the parent's + missing-from-sample row. ``partition_filter`` likewise applies only + to the child (the model under prune). """ try: validate_identifier("CandidateTestRelationships.column", test.column) @@ -272,13 +487,24 @@ def _compile_relationships( parent_col = _quote(test.field, quote_char) child_table = _qualified_table_name(table_ref, quote_char) parent_table = _qualified_table_name(parent_table_ref, quote_char) - return ( + child_target = "sample" if scope == "sample" else child_table + test_sql = ( f"SELECT child.{child_col} " - f"FROM {child_table} AS child " + f"FROM {child_target} AS child " f"LEFT JOIN {parent_table} AS parent " f"ON child.{child_col} = parent.{parent_col} " f"WHERE child.{child_col} IS NOT NULL AND parent.{parent_col} IS NULL" ) + return _wrap_with_sample_or_partition( + test_sql=test_sql, + table_sql=child_table, + table_alias_sql=child_target, + scope=scope, + sample_size=sample_size, + sample_bucket=sample_bucket, + partition_filter=partition_filter, + quote_char=quote_char, + ) def _compile_test( @@ -286,6 +512,11 @@ def _compile_test( table_ref: TableRef, dialect: Dialect, manifest: Manifest, + *, + scope: Scope = "full", + sample_size: int | None = None, + sample_bucket: int | None = None, + partition_filter: PartitionFilter | None = None, ) -> str | _RequiresFutureData | _InvalidIdentifier: """Render a candidate test as a failing-rows SELECT. @@ -304,16 +535,66 @@ def _compile_test( a test whose ``column`` / ``field`` fails the SQL-identifier shape check returns :class:`_InvalidIdentifier`; the orchestrator distinguishes the three return shapes via :func:`isinstance`. + + Sampling and partition-filter wiring (post-PR-#20 review fix): + + * ``scope="sample"`` — wraps the test in a deterministic-sample CTE + (``WITH sample AS (SELECT * FROM
AS t WHERE + MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), ) < 1 + [AND ] LIMIT ) ``). The orchestrator + derives ``sample_bucket`` from ``num_rows / sample_size`` and + passes both kwargs in. + * ``scope="full"`` — emits the test SQL against the raw table; when + ``partition_filter`` is supplied the predicate is composed via + derived table (``FROM (SELECT * FROM
WHERE + )``). + * ``relationships`` in sample mode samples the CHILD table only; + the parent stays at full so an orphan detected in the child + sample is not a false positive of the parent's missing-from-sample + row. ``partition_filter`` likewise applies to the child only. """ quote_char = dialect.quote_char if isinstance(test, CandidateTestNotNull): - return _compile_not_null(test, table_ref, quote_char) + return _compile_not_null( + test, + table_ref, + quote_char, + scope=scope, + sample_size=sample_size, + sample_bucket=sample_bucket, + partition_filter=partition_filter, + ) if isinstance(test, CandidateTestUnique): - return _compile_unique(test, table_ref, quote_char) + return _compile_unique( + test, + table_ref, + quote_char, + scope=scope, + sample_size=sample_size, + sample_bucket=sample_bucket, + partition_filter=partition_filter, + ) if isinstance(test, CandidateTestAcceptedValues): - return _compile_accepted_values(test, table_ref, quote_char) + return _compile_accepted_values( + test, + table_ref, + quote_char, + scope=scope, + sample_size=sample_size, + sample_bucket=sample_bucket, + partition_filter=partition_filter, + ) # Only CandidateTestRelationships remains in the discriminated union. - return _compile_relationships(test, table_ref, quote_char, manifest) + return _compile_relationships( + test, + table_ref, + quote_char, + manifest, + scope=scope, + sample_size=sample_size, + sample_bucket=sample_bucket, + partition_filter=partition_filter, + ) def _compute_compiled_sql_hash(sql: str) -> str: diff --git a/src/signalforge/prune/engine.py b/src/signalforge/prune/engine.py index b96b558d..41ee720e 100644 --- a/src/signalforge/prune/engine.py +++ b/src/signalforge/prune/engine.py @@ -55,7 +55,7 @@ v0.1 relies on the adapter's default :class:`QueryJobConfig` (no per-test timeout). Per-test budget enforcement requires an additional ``run_test_sql(..., timeout_ms=...)`` parameter, deferred to v0.2 — the -existing ``make_query_job_config`` plumbing from US-002 is in place but +existing ``_make_query_job_config`` plumbing from US-002 is in place but not exposed at :meth:`run_test_sql`. The ``kept-without-evidence`` *outcome* is still emitted via two paths: @@ -98,6 +98,7 @@ from signalforge.prune.errors import ( PruneAuditRecordTooLargeError, PruneAuditWriteError, + PruneError, PruneTrustedModelNotFoundError, ) from signalforge.prune.models import PruneDecision, PruneResult, Scope @@ -462,6 +463,79 @@ def _decide_kept_without_evidence_budget( ) +def _resolve_sample_bucket( + *, + adapter: WarehouseAdapter, + table_ref: TableRef, + scope: Scope, + sample_size: int, +) -> int | None: + """Return the deterministic-sample bucket for ``scope=sample``, + or ``None`` when ``scope=full``. + + The bucket is computed as ``max(num_rows // sample_size, 1)`` — + matches the warehouse adapter's :meth:`sample_rows` derivation + (DEC-006 of issue #3) so samples drawn through this seam align with + samples the adapter would draw on its own. + + Reaches through the warehouse adapter's + :func:`signalforge.warehouse.adapters._client._get_client` path to + fetch ``Table.num_rows``. The adapter's own :meth:`sample_rows` + encapsulates this lookup but does not surface row count separately; + the prune layer needs the row count BEFORE issuing tests, so we + accept the minor encapsulation crack here. v0.2 may add an explicit + :meth:`WarehouseAdapter.get_table_metadata` seam. + + Raises :class:`PruneError` when ``num_rows`` is unknown — sampling + against an unsized table would silently degrade to "every row" and + defeat US-003's cost model. The fail-loud signal lands in front of + the operator. + """ + if scope != "sample": + return None + # Reach through the SDK shim to fetch row count. ``_get_client`` is + # the documented internal seam on the BigQuery adapter (DEC-019 of + # issue #3 — every SDK-noise comment lives in + # ``adapters/_client.py``); the abstract base does NOT declare it, + # so we route through ``getattr`` to keep pyright clean. v0.2 may + # add an explicit ``WarehouseAdapter.get_table_metadata`` seam to + # avoid the encapsulation crack; for now the prune layer needs + # ``num_rows`` BEFORE issuing tests and the adapter's public + # ``sample_rows`` does not surface it. + get_client = getattr(adapter, "_get_client", None) + if get_client is None: # pragma: no cover - defensive; v0.1 only ships BigQuery + raise PruneError( + "sample-mode prune requires the adapter to expose `_get_client`; " + f"{type(adapter).__name__} does not.", + remediation=( + "Set `prune.scope: full` in signalforge.yml until the v0.2 " + "WarehouseAdapter.get_table_metadata seam lands." + ), + ) + try: + client = get_client() + meta = client.get_table(table_ref) + except WarehouseError: + # Adapter mapped the SDK exception. Re-raise as PruneError so + # the caller's catch surface stays homogeneous: any failure to + # establish the deterministic-sample bucket is a prune-layer + # configuration / setup error. + raise + num_rows = getattr(meta, "num_rows", None) + if num_rows is None or num_rows == 0: + raise PruneError( + f"sample-mode prune requires Table.num_rows for " + f"{table_ref.qualified_name!r} but the warehouse returned None.", + remediation=( + "Verify the table is materialised and accessible. " + "If it is genuinely empty (or num_rows is unavailable on the " + "warehouse type), set `prune.scope: full` in signalforge.yml " + "to bypass the deterministic-sample CTE." + ), + ) + return max(num_rows // sample_size, 1) + + def prune_tests( model: Model, adapter: WarehouseAdapter, @@ -629,6 +703,18 @@ def prune_tests( resolved_config.model_dump_json(), ) + # Resolve the deterministic-sample bucket once per run when + # scope=sample. The bucket is derived from ``num_rows / sample_size`` + # so the wrapped CTE samples ~sample_size rows; deriving once and + # threading through every per-test compile keeps every test in the + # run targeting the same sampled row set. + sample_bucket = _resolve_sample_bucket( + adapter=adapter, + table_ref=table_ref, + scope=scope, + sample_size=resolved_config.sample_size, + ) + pairs = _iter_candidate_tests(candidates) decisions: list[PruneDecision] = [] start_ms = _now_monotonic_ms() @@ -663,7 +749,16 @@ def prune_tests( # Compile the candidate test to failing-rows SQL. Returns either # a string (the SELECT), a _RequiresFutureData sentinel, or an # _InvalidIdentifier sentinel. - compile_result = _compile_test(test, table_ref, dialect, manifest) + compile_result = _compile_test( + test, + table_ref, + dialect, + manifest, + scope=scope, + sample_size=resolved_config.sample_size if scope == "sample" else None, + sample_bucket=sample_bucket, + partition_filter=resolved_config.partition_filter, + ) if isinstance(compile_result, _RequiresFutureData): decision = _decide_requires_future_data( test=test, diff --git a/tests/fixtures/prune/compiled_sql/accepted_values_sample.sql b/tests/fixtures/prune/compiled_sql/accepted_values_sample.sql new file mode 100644 index 00000000..54001d8e --- /dev/null +++ b/tests/fixtures/prune/compiled_sql/accepted_values_sample.sql @@ -0,0 +1 @@ +WITH sample AS (SELECT * FROM `fake_project.dataset.orders` AS t WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), 10) < 1 LIMIT 100000) SELECT `status` FROM sample WHERE `status` IS NOT NULL AND `status` NOT IN ('placed', 'shipped', 'cancelled') \ No newline at end of file diff --git a/tests/fixtures/prune/compiled_sql/not_null_sample.sql b/tests/fixtures/prune/compiled_sql/not_null_sample.sql new file mode 100644 index 00000000..e07dd896 --- /dev/null +++ b/tests/fixtures/prune/compiled_sql/not_null_sample.sql @@ -0,0 +1 @@ +WITH sample AS (SELECT * FROM `fake_project.dataset.orders` AS t WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), 10) < 1 LIMIT 100000) SELECT `customer_id` FROM sample WHERE `customer_id` IS NULL \ No newline at end of file diff --git a/tests/fixtures/prune/compiled_sql/relationships_sample.sql b/tests/fixtures/prune/compiled_sql/relationships_sample.sql new file mode 100644 index 00000000..9d2a9e19 --- /dev/null +++ b/tests/fixtures/prune/compiled_sql/relationships_sample.sql @@ -0,0 +1 @@ +WITH sample AS (SELECT * FROM `fake_project.dataset.orders` AS t WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), 10) < 1 LIMIT 100000) SELECT child.`customer_id` FROM sample AS child LEFT JOIN `fake_project.dataset.customers` AS parent ON child.`customer_id` = parent.`id` WHERE child.`customer_id` IS NOT NULL AND parent.`id` IS NULL \ No newline at end of file diff --git a/tests/fixtures/prune/compiled_sql/unique_sample.sql b/tests/fixtures/prune/compiled_sql/unique_sample.sql new file mode 100644 index 00000000..7636f252 --- /dev/null +++ b/tests/fixtures/prune/compiled_sql/unique_sample.sql @@ -0,0 +1 @@ +WITH sample AS (SELECT * FROM `fake_project.dataset.orders` AS t WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), 10) < 1 LIMIT 100000) SELECT `customer_id` FROM sample WHERE `customer_id` IS NOT NULL GROUP BY `customer_id` HAVING COUNT(*) > 1 \ No newline at end of file diff --git a/tests/prune/test_compiler.py b/tests/prune/test_compiler.py index 960d4d3c..118de4fc 100644 --- a/tests/prune/test_compiler.py +++ b/tests/prune/test_compiler.py @@ -37,7 +37,7 @@ _RequiresFutureData, ) from signalforge.warehouse._sql_safety import validate_test_sql -from signalforge.warehouse.models import BIGQUERY_DIALECT, Dialect, TableRef +from signalforge.warehouse.models import BIGQUERY_DIALECT, Dialect, PartitionFilter, TableRef _FIXTURES_DIR = Path(__file__).parent.parent / "fixtures" / "prune" / "compiled_sql" @@ -121,6 +121,160 @@ def test_compile_relationships_matches_snapshot() -> None: assert actual == expected +# --------------------------------------------------------------------------- +# Sample-mode snapshot tests — pinned byte-exact output for each variant +# wrapped in a deterministic-sample CTE. Confirms the post-PR-#20 review +# wiring threads scope/sample_size/sample_bucket/partition_filter through +# the compiler so ``prune.scope: sample`` actually samples in the SQL. +# --------------------------------------------------------------------------- + + +def test_compile_not_null_sample_mode_matches_snapshot() -> None: + expected = _read_fixture("not_null_sample.sql") + test = CandidateTestNotNull(column="customer_id") + actual = _compile_test( + test, + _make_orders_table_ref(), + BIGQUERY_DIALECT, + _make_manifest(), + scope="sample", + sample_size=100_000, + sample_bucket=10, + ) + assert actual == expected + + +def test_compile_unique_sample_mode_matches_snapshot() -> None: + expected = _read_fixture("unique_sample.sql") + test = CandidateTestUnique(column="customer_id") + actual = _compile_test( + test, + _make_orders_table_ref(), + BIGQUERY_DIALECT, + _make_manifest(), + scope="sample", + sample_size=100_000, + sample_bucket=10, + ) + assert actual == expected + + +def test_compile_accepted_values_sample_mode_matches_snapshot() -> None: + expected = _read_fixture("accepted_values_sample.sql") + test = CandidateTestAcceptedValues(column="status", values=("placed", "shipped", "cancelled")) + actual = _compile_test( + test, + _make_orders_table_ref(), + BIGQUERY_DIALECT, + _make_manifest(), + scope="sample", + sample_size=100_000, + sample_bucket=10, + ) + assert actual == expected + + +def test_compile_sample_mode_with_partition_filter_threads_predicate() -> None: + """``partition_filter`` is rendered into the deterministic-sample CTE + alongside the hash-mod predicate. + + The orchestrator threads ``PruneConfig.partition_filter`` through + every per-test compile so the wrapped CTE matches the warehouse + adapter's :meth:`sample_rows` shape. Required by the warehouse + adapter for tables ≥ 100M rows. + """ + test = CandidateTestNotNull(column="customer_id") + pf = PartitionFilter(column="event_dt", op=">=", value="2026-01-01") + actual = _compile_test( + test, + _make_orders_table_ref(), + BIGQUERY_DIALECT, + _make_manifest(), + scope="sample", + sample_size=100_000, + sample_bucket=10, + partition_filter=pf, + ) + assert isinstance(actual, str) + # The CTE carries both the hash-mod predicate AND the partition AND. + expected_predicate = ( + "MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), 10) < 1 AND `event_dt` >= '2026-01-01'" + ) + assert expected_predicate in actual + # Test still runs against the sample alias. + assert "FROM sample WHERE `customer_id` IS NULL" in actual + + +def test_compile_full_mode_with_partition_filter_wraps_table_in_subquery() -> None: + """``scope="full"`` + ``partition_filter`` composes via derived table. + + The test runs against a partition-filtered subquery rather than the + raw table. Composing via subquery (rather than editing the per-test + WHERE clause) is uniform across all four test shapes so the wrapper + stays a true wrapper. + """ + test = CandidateTestNotNull(column="customer_id") + pf = PartitionFilter(column="event_dt", op=">=", value="2026-01-01") + actual = _compile_test( + test, + _make_orders_table_ref(), + BIGQUERY_DIALECT, + _make_manifest(), + scope="full", + partition_filter=pf, + ) + assert isinstance(actual, str) + # The original table was wrapped in a derived-table partition filter. + assert ( + "FROM (SELECT * FROM `fake_project.dataset.orders` WHERE `event_dt` >= '2026-01-01')" + in actual + ) + # No CTE — full mode does not wrap with WITH sample. + assert "WITH sample" not in actual + + +def test_compile_full_mode_no_partition_filter_emits_unwrapped_sql() -> None: + """``scope="full"`` + no partition_filter is byte-identical to the + legacy unwrapped output. Snapshot fixtures pin the unwrapped shape. + """ + test = CandidateTestNotNull(column="customer_id") + actual = _compile_test( + test, + _make_orders_table_ref(), + BIGQUERY_DIALECT, + _make_manifest(), + scope="full", + partition_filter=None, + ) + assert actual == _read_fixture("not_null.sql") + + +def test_compile_relationships_sample_mode_matches_snapshot() -> None: + """Sample-mode relationships samples the CHILD table only. + + The parent stays at full so an orphan detected in the child sample + is not a false positive caused by the parent's missing-from-sample + row. The pinned fixture asserts the WITH sample CTE wraps the child + only; the LEFT JOIN target stays at the qualified parent table. + """ + expected = _read_fixture("relationships_sample.sql") + test = CandidateTestRelationships(column="customer_id", to="customers", field="id") + actual = _compile_test( + test, + _make_orders_table_ref(), + BIGQUERY_DIALECT, + _make_manifest(), + scope="sample", + sample_size=100_000, + sample_bucket=10, + ) + assert actual == expected + # Belt-and-braces: the parent table is NOT sampled — verify the + # full-qualified parent identifier survives the wrap. + assert isinstance(actual, str) + assert "LEFT JOIN `fake_project.dataset.customers` AS parent" in actual + + # --------------------------------------------------------------------------- # DEC-023: NULL-exclusion conventions match dbt-core. # --------------------------------------------------------------------------- @@ -254,6 +408,54 @@ def test_relationships_returns_requires_future_data_when_parent_missing() -> Non assert "nonexistent_model" in result.reason +def test_relationships_returns_requires_future_data_when_parent_ambiguous() -> None: + """Multiple manifest models sharing ``Model.name`` (e.g. cross-package + name collision) cannot be silently disambiguated by the compiler. + + Picking the first match would let the prune layer issue a wrong-table + join — exactly the silent-failure mode the ``requires-future-data`` + branch exists to prevent. The reason text quantifies the ambiguity + so the reviewer sees how many parents matched. + """ + customers_a = Model( + unique_id="model.shop_a.customers", + name="customers", + resource_type="model", + package_name="shop_a", + original_file_path="models/customers.sql", + path="customers.sql", + database="fake_project", + schema="dataset_a", # type: ignore[call-arg] + columns={"id": Column(name="id")}, + raw_code="select 1", + ) + customers_b = Model( + unique_id="model.shop_b.customers", + name="customers", + resource_type="model", + package_name="shop_b", + original_file_path="models/customers.sql", + path="customers.sql", + database="fake_project", + schema="dataset_b", # type: ignore[call-arg] + columns={"id": Column(name="id")}, + raw_code="select 1", + ) + manifest = Manifest( + metadata={"dbt_schema_version": "v12"}, + nodes={ + "model.shop.orders": _make_orders_model(), + "model.shop_a.customers": customers_a, + "model.shop_b.customers": customers_b, + }, + ) + test = CandidateTestRelationships(column="customer_id", to="customers", field="id") + result = _compile_test(test, _make_orders_table_ref(), BIGQUERY_DIALECT, manifest) + assert isinstance(result, _RequiresFutureData) + assert "ambiguous" in result.reason + assert "2 models" in result.reason + + def test_relationships_resolves_parent_via_manifest_name_lookup() -> None: """Manifest indexes by ``unique_id``; ``to`` is a model ``name``. diff --git a/tests/prune/test_engine.py b/tests/prune/test_engine.py index d468925e..121514af 100644 --- a/tests/prune/test_engine.py +++ b/tests/prune/test_engine.py @@ -48,7 +48,8 @@ ) from signalforge.warehouse.adapters.bigquery import BigQueryAdapter from signalforge.warehouse.errors import TableNotFoundError -from tests.warehouse._fake import FakeBigQueryClient +from signalforge.warehouse.models import PartitionFilter, TableRef +from tests.warehouse._fake import FakeBigQueryClient, FakeTable # --------------------------------------------------------------------------- # Helpers @@ -147,7 +148,7 @@ def test_prune_tests_always_passes_drops_test(tmp_path: Path) -> None: model = _make_orders_model() manifest = _make_manifest(model) candidates = _candidates_with_one_test("id") - config = PruneConfig(capture_failure_rows=0) + config = PruneConfig(scope="full", capture_failure_rows=0) result = prune_tests( model, @@ -187,7 +188,7 @@ def test_prune_tests_kept_for_real_failure_untrusted_model(tmp_path: Path) -> No model = _make_orders_model() manifest = _make_manifest(model) candidates = _candidates_with_one_test("id") - config = PruneConfig(capture_failure_rows=0) # untrusted by default + config = PruneConfig(scope="full", capture_failure_rows=0) # untrusted by default result = prune_tests( model, @@ -220,6 +221,7 @@ def test_prune_tests_failed_on_known_clean_data_for_trusted_model(tmp_path: Path manifest = _make_manifest(model) candidates = _candidates_with_one_test("id") config = PruneConfig( + scope="full", trusted_models=(model.unique_id,), capture_failure_rows=0, ) @@ -272,7 +274,7 @@ def test_prune_tests_requires_future_data_for_unknown_relationships_parent( ), ), ) - config = PruneConfig(capture_failure_rows=0) + config = PruneConfig(scope="full", capture_failure_rows=0) result = prune_tests( model, @@ -311,7 +313,7 @@ def test_prune_tests_warehouse_error_routes_to_kept_without_evidence( model = _make_orders_model() manifest = _make_manifest(model) candidates = _candidates_with_one_test("id") - config = PruneConfig(capture_failure_rows=0) + config = PruneConfig(scope="full", capture_failure_rows=0) result = prune_tests( model, @@ -347,7 +349,7 @@ def test_prune_tests_total_budget_exceeded_marks_remaining_kept_without_evidence model = _make_orders_model() manifest = _make_manifest(model) candidates = _candidates_with_n_tests(5) - config = PruneConfig(total_budget_seconds=1, capture_failure_rows=0) + config = PruneConfig(scope="full", total_budget_seconds=1, capture_failure_rows=0) # Stub `_now_monotonic_ms` so the second iteration sees the budget # exhausted (advances past 1000 ms after the first call). The clock @@ -403,7 +405,7 @@ def test_prune_tests_trusted_models_validation_at_entry(tmp_path: Path) -> None: model = _make_orders_model() manifest = _make_manifest(model) candidates = _candidates_with_one_test("id") - config = PruneConfig(trusted_models=("model.shop.nonexistent",)) + config = PruneConfig(scope="full", trusted_models=("model.shop.nonexistent",)) with pytest.raises(PruneTrustedModelNotFoundError) as excinfo: prune_tests( @@ -444,7 +446,7 @@ def test_prune_tests_audit_write_oserror_aborts_run( model = _make_orders_model() manifest = _make_manifest(model) candidates = _candidates_with_n_tests(3) - config = PruneConfig(capture_failure_rows=0) + config = PruneConfig(scope="full", capture_failure_rows=0) call_count = {"n": 0} boom = OSError("disk full") @@ -489,7 +491,7 @@ def test_prune_tests_audit_record_too_large_propagates( model = _make_orders_model() manifest = _make_manifest(model) candidates = _candidates_with_one_test("id") - config = PruneConfig(capture_failure_rows=0) + config = PruneConfig(scope="full", capture_failure_rows=0) expected = PruneAuditRecordTooLargeError(size=5000, limit=4000) @@ -557,7 +559,7 @@ def test_prune_tests_compute_config_hash_is_deterministic(tmp_path: Path) -> Non model = _make_orders_model() manifest = _make_manifest(model) candidates = _candidates_with_one_test("id") - config = PruneConfig(capture_failure_rows=0) + config = PruneConfig(scope="full", capture_failure_rows=0) prune_tests( model, @@ -610,7 +612,7 @@ def test_prune_tests_default_audit_path_resolves_relative_to_project_dir( model = _make_orders_model() manifest = _make_manifest(model) candidates = _candidates_with_one_test("id") - config = PruneConfig(capture_failure_rows=0) + config = PruneConfig(scope="full", capture_failure_rows=0) prune_tests( model, @@ -662,7 +664,7 @@ def test_prune_tests_audit_path_symlink_outside_project_raises( model = _make_orders_model() manifest = _make_manifest(model) candidates = _candidates_with_one_test("id") - config = PruneConfig(capture_failure_rows=0) + config = PruneConfig(scope="full", capture_failure_rows=0) with pytest.raises(PruneAuditWriteError): prune_tests( @@ -679,3 +681,267 @@ def test_prune_tests_audit_path_symlink_outside_project_raises( # before the writer opens any file. assert not outside_target.exists() fake.assert_all_expectations_met() + + +# --------------------------------------------------------------------------- +# PR #20 review fix: sampling / partition_filter wiring. +# +# Pre-fix behavior: ``decision.scope = config.scope`` was advisory-only — +# every test ran against the FULL table regardless of ``prune.scope``. +# These tests pin the post-fix wiring: scope, sample_size, and +# partition_filter all reach the compiled SQL. +# --------------------------------------------------------------------------- + + +def test_prune_tests_sample_mode_wraps_sql_with_deterministic_sample_cte( + tmp_path: Path, +) -> None: + """``config.scope="sample"`` wraps the failing-rows test in a + deterministic-sample CTE matching the warehouse adapter's + :meth:`sample_rows` shape. + + The bucket is derived from ``num_rows / sample_size``. With + ``num_rows=1_000_000`` and ``sample_size=100_000`` the bucket is 10. + The compiled SQL the orchestrator dispatches must carry the CTE + AND the hash-mod predicate. + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + # Engine fetches num_rows once before the test loop. + fake.expect_get_table( + ref=TableRef(project="fake_project", dataset="dataset", name="orders"), + returns=FakeTable(num_rows=1_000_000), + ) + fake.expect_query( + matching=( + r"WITH sample AS \(SELECT \* FROM `fake_project\.dataset\.orders` " + r"AS t WHERE MOD\(ABS\(FARM_FINGERPRINT\(TO_JSON_STRING\(t\)\)\), " + r"10\) < 1 LIMIT 100000\)" + ), + returns=[{"failures": 0}], + ) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig( + scope="sample", + sample_size=100_000, + capture_failure_rows=0, + ) + + result = prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + project_dir=tmp_path, + ) + + assert result.total_tests == 1 + assert result.decisions[0].decision == "dropped" + assert result.decisions[0].reason == "always-passes" + fake.assert_all_expectations_met() + + +def test_prune_tests_full_mode_does_not_wrap_with_cte(tmp_path: Path) -> None: + """``config.scope="full"`` (the default for this test) emits the + unwrapped failing-rows SELECT — no ``WITH sample`` CTE, no + deterministic-sample predicate, no ``get_table`` lookup. + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + # Note: NO expect_get_table — full-mode skips num_rows lookup. + # The query expectation is anchored on a regex that REJECTS any + # CTE wrapping by requiring the SELECT to start without ``WITH``. + fake.expect_query( + matching=( + r"^SELECT COUNT\(\*\) AS failures FROM " + r"\(SELECT `id` FROM `fake_project\.dataset\.orders` " + r"WHERE `id` IS NULL\)" + ), + returns=[{"failures": 0}], + ) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig(scope="full", capture_failure_rows=0) + + prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + project_dir=tmp_path, + ) + fake.assert_all_expectations_met() + + +def test_prune_tests_partition_filter_threads_through(tmp_path: Path) -> None: + """``config.partition_filter`` reaches the compiled SQL — verifies + the orchestrator threads the typed :class:`PartitionFilter` through + :func:`_compile_test`. Sample-mode + partition_filter renders both + predicates inside the deterministic-sample CTE. + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + fake.expect_get_table( + ref=TableRef(project="fake_project", dataset="dataset", name="orders"), + returns=FakeTable(num_rows=1_000_000), + ) + fake.expect_query( + matching=( + r"MOD\(ABS\(FARM_FINGERPRINT\(TO_JSON_STRING\(t\)\)\), 10\) < 1 " + r"AND `dt` >= '2026-01-01'" + ), + returns=[{"failures": 0}], + ) + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig( + scope="sample", + sample_size=100_000, + capture_failure_rows=0, + partition_filter=PartitionFilter(column="dt", op=">=", value="2026-01-01"), + ) + + prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + project_dir=tmp_path, + ) + fake.assert_all_expectations_met() + + +def test_prune_tests_sample_mode_relationships_samples_child_only( + tmp_path: Path, +) -> None: + """Sample-mode + relationships samples the CHILD only; the parent + table stays at full. + + Documented asymmetry — an orphan detected in the child sample is not + a false positive caused by the parent's missing-from-sample row. + """ + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + fake.expect_get_table( + ref=TableRef(project="fake_project", dataset="dataset", name="orders"), + returns=FakeTable(num_rows=1_000_000), + ) + # The query carries a sample CTE wrapping the child AND a LEFT JOIN + # against the full-qualified parent table (NOT another sample alias). + fake.expect_query( + matching=( + r"WITH sample AS .* SELECT child\.`customer_id` " + r"FROM sample AS child " + r"LEFT JOIN `fake_project\.dataset\.customers` AS parent" + ), + returns=[{"failures": 0}], + ) + adapter = _make_adapter(fake) + + # Build a manifest with both the orders model AND a customers parent. + orders = _make_orders_model() + customers = Model( + unique_id="model.shop.customers", + name="customers", + resource_type="model", + package_name="shop", + original_file_path="models/customers.sql", + path="customers.sql", + database="fake_project", + schema="dataset", # type: ignore[call-arg] + columns={"id": Column(name="id")}, + raw_code="select 1", + ) + manifest = Manifest( + metadata={"dbt_schema_version": "v12"}, + nodes={ + orders.unique_id: orders, + customers.unique_id: customers, + }, + ) + candidates = CandidateSchema( + name="orders", + description="Order events.", + columns=( + CandidateColumn( + name="customer_id", + description="FK to customers.", + tests=( + CandidateTestRelationships( + column="customer_id", + to="customers", + field="id", + ), + ), + ), + ), + ) + config = PruneConfig( + scope="sample", + sample_size=100_000, + capture_failure_rows=0, + ) + + prune_tests( + orders, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + project_dir=tmp_path, + ) + fake.assert_all_expectations_met() + + +def test_prune_tests_sample_mode_unknown_num_rows_raises_prune_error( + tmp_path: Path, +) -> None: + """Sample-mode requires ``Table.num_rows`` to size the deterministic + bucket. When the warehouse returns ``None`` the orchestrator raises + a typed :class:`PruneError` — silent degradation to "every row" + would defeat US-003's cost model. + """ + from signalforge.prune.errors import PruneError + + audit_path = tmp_path / "prune.jsonl" + fake = FakeBigQueryClient(project="fake_project") + fake.expect_get_table( + ref=TableRef(project="fake_project", dataset="dataset", name="orders"), + returns=FakeTable(num_rows=None), + ) + # No expect_query — the failure happens BEFORE any test dispatches. + adapter = _make_adapter(fake) + + model = _make_orders_model() + manifest = _make_manifest(model) + candidates = _candidates_with_one_test("id") + config = PruneConfig(scope="sample", sample_size=100_000, capture_failure_rows=0) + + with pytest.raises(PruneError): + prune_tests( + model, + adapter, + candidates, + manifest, + config=config, + audit_path=audit_path, + project_dir=tmp_path, + ) + fake.assert_all_expectations_met() From 1ae462f5949adceadcb78ff75719bf563f907e3a Mon Sep 17 00:00:00 2001 From: wjduenow Date: Fri, 1 May 2026 08:03:45 -0700 Subject: [PATCH 23/24] 6/QG: prune audit short-write loop + SF_RUN_BQ truthy guard + BQ probe race fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three independent PR-#20 review items grouped because they all affect the I/O / opt-in seams: (1) signalforge.prune.audit._write_prune_event now loops on os.write until the full payload lands. POSIX write(2) may return fewer bytes than requested (EINTR on signal-interrupted calls; short writes on some filesystems / kernels); the previous single-call assumption could silently truncate the JSONL line. A persistent zero-byte return raises OSError so we don't spin forever. (2) The SF_RUN_BQ env-var check in three integration test files now restricts truthy values to {"1", "true", "yes", "on"} (case-insensitive) via a _bq_runs_enabled() helper. The previous `not os.environ.get(...)` treated SF_RUN_BQ=0 / SF_RUN_BQ=false / SF_RUN_BQ=no as truthy because they're non-empty strings — surprising for a user trying to disable the runs. (3) tests/warehouse/test_sample_cost_probe.py now reads total_bytes_billed directly off the QueryJob instance returned by client.query(...) rather than via INFORMATION_SCHEMA.JOBS_BY_USER. The previous lookup filtered by signalforge_stage='warehouse_sample' label and a creation-time lower-bound, which could attribute bytes to the wrong job (a leftover prune-stage run, or a concurrent process emitting the same label). Reading total_bytes_billed straight off the just-issued QueryJob eliminates the race entirely. The probe deliberately bypasses the adapter's public sample_rows path to access job stats; documented inline as a v0.2 deferral (DEC-027 of issue #6). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/signalforge/prune/audit.py | 14 +- tests/prune/test_audit.py | 57 ++++++ tests/prune/test_integration_bigquery.py | 14 +- tests/warehouse/test_bigquery_integration.py | 24 ++- tests/warehouse/test_sample_cost_probe.py | 185 +++++++++---------- 5 files changed, 189 insertions(+), 105 deletions(-) diff --git a/src/signalforge/prune/audit.py b/src/signalforge/prune/audit.py index 930a3230..ed8faf76 100644 --- a/src/signalforge/prune/audit.py +++ b/src/signalforge/prune/audit.py @@ -219,7 +219,19 @@ def _write_prune_event(event: PruneEvent, path: Path) -> None: # descriptor is released; it does NOT silence the syscall failures. fd = os.open(str(path), os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) try: - os.write(fd, encoded) + # ``os.write`` may return fewer bytes than requested (EINTR on a + # signal-interrupted call, or short writes on certain filesystems + # / kernels). Loop until the full payload lands; raise on a + # zero-byte return (disk full / unrecoverable I/O failure). + # POSIX atomicity for ``O_APPEND`` writes still holds at the + # ``write(2)`` boundary up to ``PIPE_BUF``; the loop is the + # documented short-write recovery, not a contract violation. + written = 0 + while written < len(encoded): + n = os.write(fd, encoded[written:]) + if n == 0: + raise OSError("os.write returned 0 — disk full or other I/O failure") + written += n os.fsync(fd) finally: # Best-effort close; the write/fsync above already succeeded diff --git a/tests/prune/test_audit.py b/tests/prune/test_audit.py index 650fa725..ada1dd7c 100644 --- a/tests/prune/test_audit.py +++ b/tests/prune/test_audit.py @@ -242,6 +242,63 @@ def writer(thread_idx: int) -> None: json.loads(line) +def test_write_prune_event_loops_on_short_writes(tmp_path: Path) -> None: + """``os.write`` may return fewer bytes than requested (EINTR or + short-write semantics on some filesystems). The writer loops until + the full payload lands; the on-disk record must contain every byte. + + Pre-fix behavior: a single ``os.write(fd, encoded)`` call assumed + the full payload was written. A short return would silently truncate + the JSONL line, leaving a torn audit record. Post-fix: loop while + ``written < len(encoded)``; raise on a zero-byte return (disk full + or unrecoverable I/O failure). + """ + audit_path = tmp_path / "prune.jsonl" + real_write = os.write + call_count = {"n": 0} + + def short_write(fd: int, data: bytes) -> int: + call_count["n"] += 1 + # First call: write only half the bytes. Second call: write the + # remainder. Mirrors real-world EINTR / partial-write recovery. + if call_count["n"] == 1: + half = len(data) // 2 + return real_write(fd, data[:half]) + return real_write(fd, data) + + with patch("signalforge.prune.audit.os.write", side_effect=short_write): + _write_prune_event(_make_event(), audit_path) + + # The on-disk file carries the FULL JSONL line — nothing torn. + contents = audit_path.read_text(encoding="utf-8") + assert contents.endswith("\n") + assert len(contents.splitlines()) == 1 + payload = json.loads(contents.splitlines()[0]) + assert payload["audit_schema_version"] == 1 + # The loop ran (at least two ``os.write`` calls — one short, one to + # complete). + assert call_count["n"] >= 2 + + +def test_write_prune_event_short_write_zero_bytes_raises(tmp_path: Path) -> None: + """A persistent zero-byte return from ``os.write`` raises ``OSError`` + rather than spinning forever — the writer guards against an + infinite loop on a wedged file descriptor. + """ + audit_path = tmp_path / "prune.jsonl" + + def stuck_write(fd: int, data: bytes) -> int: + return 0 + + with ( + patch("signalforge.prune.audit.os.write", side_effect=stuck_write), + pytest.raises(OSError) as excinfo, + ): + _write_prune_event(_make_event(), audit_path) + + assert "returned 0" in str(excinfo.value) or "disk full" in str(excinfo.value) + + def test_compute_config_hash_is_deterministic_and_16_hex() -> None: """:func:`_compute_config_hash` returns 16 hex characters and is deterministic across calls — matches safety's ``policy_hash`` (DEC-005). diff --git a/tests/prune/test_integration_bigquery.py b/tests/prune/test_integration_bigquery.py index 8142ee56..a6f06b3d 100644 --- a/tests/prune/test_integration_bigquery.py +++ b/tests/prune/test_integration_bigquery.py @@ -51,8 +51,20 @@ _SF_RUN_BQ_REASON = "requires SF_RUN_BQ=1 and ADC" +def _bq_runs_enabled() -> bool: + """Return ``True`` when the user opted into BigQuery integration runs. + + Restricts the set of "truthy" ``SF_RUN_BQ`` values to the explicit + affirmatives. The naive ``not os.environ.get("SF_RUN_BQ")`` would + treat ``SF_RUN_BQ=0``, ``SF_RUN_BQ=false``, ``SF_RUN_BQ=no`` as + truthy (any non-empty string) — surprising behaviour for a user + trying to turn the runs OFF. + """ + return os.environ.get("SF_RUN_BQ", "").lower() in {"1", "true", "yes", "on"} + + @pytest.mark.bigquery -@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), reason=_SF_RUN_BQ_REASON) +@pytest.mark.skipif(not _bq_runs_enabled(), reason=_SF_RUN_BQ_REASON) def test_prune_iowa_liquor_sales(tmp_path: Path) -> None: """End-to-end: two candidate tests against the public sales table. diff --git a/tests/warehouse/test_bigquery_integration.py b/tests/warehouse/test_bigquery_integration.py index a7397453..d3287afc 100644 --- a/tests/warehouse/test_bigquery_integration.py +++ b/tests/warehouse/test_bigquery_integration.py @@ -45,6 +45,18 @@ _SF_RUN_BQ_REASON = "requires SF_RUN_BQ=1 and ADC" +def _bq_runs_enabled() -> bool: + """Return ``True`` when the user opted into BigQuery integration runs. + + Restricts the set of "truthy" ``SF_RUN_BQ`` values to the explicit + affirmatives. The naive ``not os.environ.get("SF_RUN_BQ")`` would + treat ``SF_RUN_BQ=0``, ``SF_RUN_BQ=false``, ``SF_RUN_BQ=no`` as + truthy (any non-empty string) — surprising behaviour for a user + trying to turn the runs OFF. + """ + return os.environ.get("SF_RUN_BQ", "").lower() in {"1", "true", "yes", "on"} + + def _shakespeare_ref() -> TableRef: """Build the Shakespeare ``TableRef`` for the integration tests. @@ -56,7 +68,7 @@ def _shakespeare_ref() -> TableRef: @pytest.mark.bigquery -@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), reason=_SF_RUN_BQ_REASON) +@pytest.mark.skipif(not _bq_runs_enabled(), reason=_SF_RUN_BQ_REASON) def test_int_sample_rows_returns_n_rows_from_shakespeare() -> None: """Sampling Shakespeare with n=10 must return at most 10 rows whose schema matches the public dataset (``word``, ``word_count``, ``corpus``, @@ -73,7 +85,7 @@ def test_int_sample_rows_returns_n_rows_from_shakespeare() -> None: @pytest.mark.bigquery -@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), reason=_SF_RUN_BQ_REASON) +@pytest.mark.skipif(not _bq_runs_enabled(), reason=_SF_RUN_BQ_REASON) def test_int_column_stats_returns_correct_count_for_corpus() -> None: """``column_stats`` on Shakespeare's ``word`` column must report a positive ``count`` and ``distinct``, and a ``data_type`` that BigQuery @@ -91,7 +103,7 @@ def test_int_column_stats_returns_correct_count_for_corpus() -> None: @pytest.mark.bigquery -@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), reason=_SF_RUN_BQ_REASON) +@pytest.mark.skipif(not _bq_runs_enabled(), reason=_SF_RUN_BQ_REASON) def test_int_run_test_sql_passes_for_known_clean_query() -> None: """``WHERE FALSE`` always returns zero rows — the adapter must report ``passed=True`` and ``failure_count=0``.""" @@ -105,7 +117,7 @@ def test_int_run_test_sql_passes_for_known_clean_query() -> None: @pytest.mark.bigquery -@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), reason=_SF_RUN_BQ_REASON) +@pytest.mark.skipif(not _bq_runs_enabled(), reason=_SF_RUN_BQ_REASON) def test_int_run_test_sql_fails_for_known_dirty_query() -> None: """A test-SQL that intentionally returns rows must report ``passed=False``, a positive ``failure_count``, and a non-empty @@ -124,7 +136,7 @@ def test_int_run_test_sql_fails_for_known_dirty_query() -> None: @pytest.mark.bigquery -@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), reason=_SF_RUN_BQ_REASON) +@pytest.mark.skipif(not _bq_runs_enabled(), reason=_SF_RUN_BQ_REASON) def test_int_max_bytes_billed_blocks_oversize_query() -> None: """A 1-byte ``max_bytes_billed`` cap must trip BigQuery's pre-flight estimator and surface as :class:`BytesBilledExceededError` (DEC-015, @@ -137,7 +149,7 @@ def test_int_max_bytes_billed_blocks_oversize_query() -> None: @pytest.mark.bigquery -@pytest.mark.skipif(not os.environ.get("SF_RUN_BQ"), reason=_SF_RUN_BQ_REASON) +@pytest.mark.skipif(not _bq_runs_enabled(), reason=_SF_RUN_BQ_REASON) def test_int_adc_unconfigured_raises_typed_error( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/warehouse/test_sample_cost_probe.py b/tests/warehouse/test_sample_cost_probe.py index 7fbe9a67..e4f35ce2 100644 --- a/tests/warehouse/test_sample_cost_probe.py +++ b/tests/warehouse/test_sample_cost_probe.py @@ -34,12 +34,12 @@ a successful run. US-014 (``docs/prune-ops.md``) folds the recorded figure into the cost-model section. -If the post-call ``INFORMATION_SCHEMA.JOBS_BY_USER`` lookup fails (IAM, -region availability, eventual-consistency lag, etc.), the test marks -itself ``xfail`` with the documented reason rather than failing the -suite — exposing ``total_bytes_billed`` cleanly via the adapter is a -v0.2 concern (DEC-027 of issue #6, ``plans/super/6-prune-engine.md``) -and out of scope for this story. +The probe captures ``total_bytes_billed`` directly off the +``QueryJob`` instance returned by ``client.query(...)`` — no +``INFORMATION_SCHEMA.JOBS_BY_USER`` round-trip, no race against +concurrent prune-stage jobs labelled with the same +``signalforge_stage`` tag. The deliberate departure from the adapter's +public ``sample_rows`` path is documented inline. """ from __future__ import annotations @@ -88,94 +88,81 @@ _LOGGER = logging.getLogger("signalforge.warehouse.test.cost_probe") -# --------------------------------------------------------------------------- -# Helpers. -# --------------------------------------------------------------------------- +def _bq_runs_enabled() -> bool: + """Return ``True`` when the user opted into BigQuery integration runs. + Restricts the set of "truthy" ``SF_RUN_BQ`` values to the explicit + affirmatives. The naive ``not os.environ.get("SF_RUN_BQ")`` would + treat ``SF_RUN_BQ=0``, ``SF_RUN_BQ=false``, ``SF_RUN_BQ=no`` as + truthy (any non-empty string) — surprising behaviour for a user + trying to turn the runs OFF. + """ + return os.environ.get("SF_RUN_BQ", "").lower() in {"1", "true", "yes", "on"} -def _adapter_billing_project(adapter: BigQueryAdapter) -> str: - """Return the billing project the adapter resolves to. - ``BigQueryAdapter`` does not expose its underlying client publicly; - we reach through the private ``_get_client`` to read ``project`` - because the probe needs a region-qualified - ``INFORMATION_SCHEMA.JOBS_BY_USER`` lookup. This is test-only access - — production code routes through the adapter API. - """ - # Accessing a private member is intentional: the probe is a - # diagnostic that lives outside the adapter's public contract. - client = adapter._get_client() # noqa: SLF001 - return str(client.project) +# --------------------------------------------------------------------------- +# Helpers. +# --------------------------------------------------------------------------- -def _lookup_total_bytes_billed( +def _issue_sample_capture_bytes_billed( adapter: BigQueryAdapter, *, - started_after_epoch_s: float, -) -> int | None: - """Query ``INFORMATION_SCHEMA.JOBS_BY_USER`` for the most recent - ``warehouse_sample`` job and return its ``total_bytes_billed``. - - Returns ``None`` if no matching job is found (eventual-consistency - lag) or if the lookup itself fails for any reason — the caller then - ``xfail``s with the documented reason. We deliberately do NOT - propagate the exception: a probe that hard-fails on environmental - quirks (regional dataset availability, missing - ``bigquery.jobs.list`` permission, etc.) is worse than no probe. - - The query filters on the ``signalforge_stage`` label that - ``BigQueryAdapter._default_job_config`` stamps on every job - (DEC-015 of issue #3). Filtering by label scopes the result to the - sample we just issued without needing the job-id round-trip. + target: TableRef, + sample_size: int, +) -> tuple[list[dict[str, Any]], int]: + """Issue the deterministic sample directly through the SDK shim and + return ``(rows, total_bytes_billed)``. + + The probe deliberately bypasses the adapter's public + :meth:`signalforge.warehouse.adapters.bigquery.BigQueryAdapter.sample_rows` + so it can read ``QueryJob.total_bytes_billed`` after the job + completes. The adapter's public surface drops the ``QueryJob`` + after iterating its rows; widening the public API to expose job + stats is a v0.2 concern (DEC-027 of issue #6). + + The probe replicates the adapter's deterministic-sample SQL shape + (DEC-006 of issue #3) byte-for-byte so the figure recorded matches + what production prune-stage runs will spend. Reading + ``total_bytes_billed`` straight off the ``QueryJob`` instance + eliminates the previous race where the + ``INFORMATION_SCHEMA.JOBS_BY_USER`` lookup could have attributed + bytes to the wrong job (a leftover prune-stage run, or a concurrent + process emitting the same ``signalforge_stage`` label). """ - project = _adapter_billing_project(adapter) - # ``INFORMATION_SCHEMA.JOBS_BY_USER`` is region-qualified. We - # default to US (the multi-region the public ``bigquery-public-data`` - # tables live in) when the adapter has no explicit location. - region = adapter._location or "US" # noqa: SLF001 - region_qualifier = f"region-{region.lower()}" - - # ``creation_time`` is a TIMESTAMP; the probe records monotonic - # epoch seconds and we compare against TIMESTAMP_SECONDS at lookup. + client = adapter._get_client() # noqa: SLF001 # documented seam + table = client.get_table(target) + num_rows = getattr(table, "num_rows", None) + if not num_rows: + # Public dataset; this is a defensive guard rather than a + # production-relevant path. If ever it fires the probe + # short-circuits via xfail in the caller. + raise RuntimeError( + f"sample probe: public table {target.qualified_name!r} returned " + f"unknown num_rows; cannot derive bucket" + ) + bucket = max(num_rows // sample_size, 1) + sql = ( - f"SELECT total_bytes_billed " - f"FROM `{project}.{region_qualifier}.INFORMATION_SCHEMA.JOBS_BY_USER` " - f"WHERE creation_time >= TIMESTAMP_SECONDS(@since) " - f"AND EXISTS (SELECT 1 FROM UNNEST(labels) AS l " - f"WHERE l.key = 'signalforge_stage' AND l.value = 'warehouse_sample') " - f"ORDER BY creation_time DESC LIMIT 1" + f"SELECT * FROM `{target.qualified_name}` AS t " + f"WHERE MOD(ABS(FARM_FINGERPRINT(TO_JSON_STRING(t))), {bucket}) < 1 " + f"LIMIT {sample_size}" ) - try: - # Build an ad-hoc query-parameterised job. We bypass the - # adapter's ``run_test_sql`` because the probe SQL is not a - # failing-rows test — it's a metadata read. - from google.cloud import bigquery # type: ignore[import-not-found] - - client = adapter._get_client() # noqa: SLF001 - job_config = bigquery.QueryJobConfig( - query_parameters=[ - bigquery.ScalarQueryParameter("since", "INT64", int(started_after_epoch_s)), - ], - use_query_cache=False, - ) - rows = list(client.query(sql, job_config=job_config).result()) - except Exception as exc: # pragma: no cover - environment-dependent - _LOGGER.info( - "INFORMATION_SCHEMA lookup failed; the probe will xfail. error_class=%s", - type(exc).__name__, - ) - return None - - if not rows: - return None - bytes_billed = getattr(rows[0], "total_bytes_billed", None) - if bytes_billed is None: - # The row exists but the column is null — for a non-dry-run - # query this should not happen, but treat as "unknown" rather - # than zero so the xfail path engages cleanly. - return None - return int(bytes_billed) + job = client.query( + sql, + job_config=adapter._default_job_config(stage="warehouse_sample"), # noqa: SLF001 + ) + rows = list(job.result()) # consume to ensure completion + total_bytes_billed = getattr(job, "total_bytes_billed", None) + if total_bytes_billed is None: # pragma: no cover - environment-dependent + # Non-dry-run jobs always populate this; if absent treat as + # unknown via -1 so the caller can xfail cleanly. + total_bytes_billed = -1 + rows_dict: list[dict[str, Any]] = [ + dict(r.items()) if hasattr(r, "items") else dict(r) for r in rows + ] + return rows_dict, int(total_bytes_billed) # --------------------------------------------------------------------------- @@ -185,7 +172,7 @@ def _lookup_total_bytes_billed( @pytest.mark.bigquery @pytest.mark.skipif( - not os.environ.get("SF_RUN_BQ"), + not _bq_runs_enabled(), reason="set SF_RUN_BQ=1 to run BigQuery integration probes", ) def test_sample_rows_bytes_billed_within_budget( @@ -200,18 +187,23 @@ def test_sample_rows_bytes_billed_within_budget( * If ``bytes_billed >= 500 MB``, a WARNING fires carrying the figure so the run log records it for ``docs/prune-ops.md``. - If the post-call ``INFORMATION_SCHEMA`` lookup yields no row - (eventual-consistency lag) or fails for any other reason, the test - marks itself ``xfail`` with the documented reason. The probe is - documentation-grade: a missing figure is acceptable; a wrong figure - is not. + Uses the SDK seam directly (rather than the adapter's public + ``sample_rows``) so the ``QueryJob.total_bytes_billed`` figure can + be read off the just-issued job — no ``INFORMATION_SCHEMA`` + round-trip, no race against concurrent prune-stage jobs labelled + with the same ``signalforge_stage`` tag. Adapter-side bytes_billed + exposure on the public surface remains a v0.2 concern (DEC-027 of + issue #6). """ adapter = BigQueryAdapter() - started_at_epoch_s = time.time() started_at_monotonic = time.monotonic() with adapter: - rows: list[dict[str, Any]] = adapter.sample_rows(_TARGET, _SAMPLE_SIZE) + rows, bytes_billed = _issue_sample_capture_bytes_billed( + adapter, + target=_TARGET, + sample_size=_SAMPLE_SIZE, + ) elapsed_s = time.monotonic() - started_at_monotonic # Belt-and-suspenders: the sampler must actually return rows. If @@ -219,13 +211,12 @@ def test_sample_rows_bytes_billed_within_budget( assert rows, "expected at least one sampled row from a 30M-row table" assert len(rows) <= _SAMPLE_SIZE - bytes_billed = _lookup_total_bytes_billed(adapter, started_after_epoch_s=started_at_epoch_s) - if bytes_billed is None: + if bytes_billed < 0: # pragma: no cover - environment-dependent pytest.xfail( - "bytes-billed retrieval via INFORMATION_SCHEMA.JOBS_BY_USER did " - "not return a matching row (IAM, regional availability, or " - "eventual-consistency lag). Adapter-side bytes_billed exposure " - "is a v0.2 concern (DEC-027 of issue #6)." + "QueryJob.total_bytes_billed was unavailable on the just-issued " + "sample job (rare; non-dry-run jobs populate this field). " + "Adapter-side bytes_billed exposure is a v0.2 concern (DEC-027 of " + "issue #6)." ) _LOGGER.info( From 27e2beede156f4744a607102e105930314b423e2 Mon Sep 17 00:00:00 2001 From: wjduenow Date: Fri, 1 May 2026 08:04:06 -0700 Subject: [PATCH 24/24] 6/QG: docs/rules corrections + rename make_query_job_config to _make_query_job_config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #20 review (Copilot + CodeRabbit) flagged six accuracy items in the documentation, rules, and adapter docstrings: (.claude/rules/prune-engine.md) - requires-future-data was described as "Kept" — it's actually "Dropped". - engine.py::_classify_decision was a fictional reference; renamed to the actual helper (_decide_from_test_result). - The signalforge.yml prune-block field list referenced phantom keys (audit_path, mode); replaced with the real PruneConfig fields. (src/signalforge/prune/config.py) - The trusted_models docstring contradicted the implementation. Trusted models route to "failed-on-known-clean-data" (drop, presumed buggy test); the docstring incorrectly claimed they "surface as real failures rather than the drop". Rewritten to match. (docs/prune-ops.md) - prune_tests signature was missing the project_dir parameter. - The test_timeout_seconds entry was unclear about what's wired vs deferred to v0.2; clarified that v0.1 has no per-test timeout enforcement (run_test_sql does not yet accept timeout_ms; the knob is reserved for v0.2 alongside the existing _make_query_job_config plumbing). (docs/warehouse-adapter-ops.md) - The _default_job_config example was missing the required stage= kwarg. - Removed the false claim that "the prune layer (#6) uses this for per-test budget enforcement" — issue #6 ships with total_budget_seconds enforcement only. (src/signalforge/warehouse/adapters/bigquery.py) - _default_job_config docstring claimed prune layer was the only caller supplying a non-None timeout_ms; rewritten to match reality (currently used only by tests; reserved for v0.2 prune integration). (rename make_query_job_config → _make_query_job_config) - The function lives inside the SDK seam (_client.py) and is internal to the adapter layer. The privacy-prefix convention dictates the underscore. Mechanical rename across _client.py, bigquery.py, prune/engine.py docstring, and the two test files that import it. Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/rules/prune-engine.md | 6 +++--- docs/prune-ops.md | 4 ++-- docs/warehouse-adapter-ops.md | 10 +++++----- src/signalforge/prune/config.py | 16 +++++++++++++--- src/signalforge/warehouse/adapters/_client.py | 4 ++-- src/signalforge/warehouse/adapters/bigquery.py | 17 +++++++++-------- tests/warehouse/test_bigquery_unit.py | 4 ++-- .../test_default_job_config_timeout.py | 8 ++++---- 8 files changed, 40 insertions(+), 29 deletions(-) diff --git a/.claude/rules/prune-engine.md b/.claude/rules/prune-engine.md index 8154a693..b47a79ef 100644 --- a/.claude/rules/prune-engine.md +++ b/.claude/rules/prune-engine.md @@ -10,11 +10,11 @@ The prune layer sits between the LLM-drafting pipeline (#5) and the quality grad - `"always-passes"` — test ran, returned zero failing rows on a representative sample, scope was sufficient. Drop. - `"failed-on-known-clean-data"` — test ran against a `trusted_models` opt-in target and returned non-zero failing rows. The model is known clean; the test is wrong. Drop. -- `"requires-future-data"` — test compiled to a sentinel (e.g. `relationships(to: )` where the target model is absent from the manifest). Cannot evaluate; ship it for the operator to revisit when the dependency lands. Kept. +- `"requires-future-data"` — test compiled to a sentinel (e.g. `relationships(to: )` where the target model is absent from the manifest). Cannot evaluate; the operator revisits when the dependency lands. Dropped (no warehouse call issued; the structured drop reason carries the diagnostic that surfaces in the diff). - `"kept"` — test ran and returned non-zero failing rows on an untrusted model. Real signal. Kept. - `"kept-without-evidence"` — total budget elapsed before this test ran, the test SQL was rejected by `validate_identifier`, the warehouse call raised, or any other "cannot positively evaluate" outcome. Kept. -The decision matrix lives in `engine.py::_classify_decision`. The load-bearing invariant: **kept-without-evidence routes to `decision="kept"`, not `decision="dropped"`.** SignalForge ships tests it cannot evaluate; the LLM proposed them, and absent contradicting warehouse evidence the operator gets to make the call. Dropping a test silently because we couldn't reach the warehouse is exactly the failure mode this commitment exists to prevent. +The decision matrix lives in `engine.py::_decide_from_test_result` (the routing helper the orchestrator dispatches to once a `TestResult` lands). The load-bearing invariant: **kept-without-evidence routes to `decision="kept"`, not `decision="dropped"`.** SignalForge ships tests it cannot evaluate; the LLM proposed them, and absent contradicting warehouse evidence the operator gets to make the call. Dropping a test silently because we couldn't reach the warehouse is exactly the failure mode this commitment exists to prevent. If you add a sixth `DropReason` literal in v0.2 (e.g. `"sample-too-small"`), update production `DropReason` AND `StrictPruneDecision` (the drift detector) AND the fixture at `tests/fixtures/prune/prune_event_v1.jsonl` AND the decision-matrix table in `docs/prune-ops.md` in the same change. @@ -115,7 +115,7 @@ The CLI (#9) and any future orchestrator wants one calling convention; new stage ## `signalforge.yml` top-level namespace: `prune:` (DEC-020) -The prune-stage block is `{ prune: { audit_path, mode, total_budget_seconds, test_timeout_seconds, partition_filter, trusted_models, ... } }`. Sibling top-level keys (`safety:`, `llm:`, future `grade:`/`diff:`) are reserved for other stages and silently ignored by the prune loader. +The prune-stage block is `{ prune: { scope, sample_size, test_timeout_seconds, total_budget_seconds, capture_failure_rows, trusted_models, partition_filter } }`. Sibling top-level keys (`safety:`, `llm:`, future `grade:`/`diff:`) are reserved for other stages and silently ignored by the prune loader. `PruneConfig` uses `extra="forbid"` (config-shaped; typos fail loud). The wrapping `_PruneConfigFile` uses `extra="ignore"` at the top level so unknown sibling stages don't break the loader. Mirrors `llm-drafter.md` DEC-027 / `safety-layer.md` DEC-025 verbatim. diff --git a/docs/prune-ops.md b/docs/prune-ops.md index 535b5869..6c355167 100644 --- a/docs/prune-ops.md +++ b/docs/prune-ops.md @@ -41,7 +41,7 @@ Import from `signalforge.prune`. The 14 names exported by `__all__`: ### Orchestrator -- **`prune_tests(model, adapter, candidates, manifest, *, config=None, audit_path=None) -> PruneResult`** — End-to-end orchestrator. Compiles every `CandidateTest`, runs each through the warehouse adapter, writes one JSONL audit record per decision, returns the aggregate `PruneResult`. Mirrors `signalforge.draft.draft_schema` so the CLI / wrapper layers see one consistent end-to-end shape across pipeline stages. `audit_path` defaults to `/.signalforge/prune.jsonl`. +- **`prune_tests(model, adapter, candidates, manifest, *, config=None, project_dir=None, audit_path=None) -> PruneResult`** — End-to-end orchestrator. Compiles every `CandidateTest`, runs each through the warehouse adapter, writes one JSONL audit record per decision, returns the aggregate `PruneResult`. Mirrors `signalforge.draft.draft_schema` so the CLI / wrapper layers see one consistent end-to-end shape across pipeline stages. `project_dir` defaults to `Path.cwd()`. `audit_path` defaults to `/.signalforge/prune.jsonl`. ### Result shapes @@ -111,7 +111,7 @@ Field-by-field: - **`scope`** — `"sample"` | `"full"`. Default `"sample"`. Whether candidate tests run against a deterministic warehouse sample or a full table scan. Switch to `"full"` only when the model is small enough that `sample_size` would scan most of it anyway. - **`sample_size`** — Integer row count for sample scope. Default `100_000`. Passed to `WarehouseAdapter.sample_rows`. Increase when the always-pass false-positive rate on small samples hides real signal; decrease to cap query bytes on very wide tables (column-pruning does NOT apply through `FARM_FINGERPRINT(TO_JSON_STRING(t))` — see [Cost model](#cost-model-us-003-verification)). -- **`test_timeout_seconds`** — Per-test wall-clock budget. Default `30`. v0.1 sets the timeout on the adapter's `QueryJobConfig` but does NOT yet thread it through `WarehouseAdapter.run_test_sql` per call (deferred to v0.2; see [v0.2 deferrals](#v02-deferrals)). +- **`test_timeout_seconds`** — Per-test wall-clock budget. Default `30`. **Reserved for v0.2** — the adapter's `_default_job_config(timeout_ms=...)` plumbing exists (US-002 of issue #3) but `WarehouseAdapter.run_test_sql` does not yet accept a per-call timeout kwarg, so v0.1 does not enforce this knob. Per-test wall-clock control in v0.1 comes implicitly from `total_budget_seconds` plus the `WarehouseError` catch path: a test that exceeds the warehouse's own budget surfaces as a typed error → `kept-without-evidence`. See [v0.2 deferrals](#v02-deferrals). - **`total_budget_seconds`** — Whole-run wall-clock budget. Default `600`. Once exceeded, every remaining test drains to `kept-without-evidence` with `why="Total prune budget (Ns) exceeded before evaluation."` (DEC-011). Conservative bias — no test is silently dropped because the run ran long. - **`capture_failure_rows`** — Number of failing rows recorded on the `PruneDecision.sample_failures` field per failed test. Default `3`. Set to `0` to omit row-level evidence entirely (the audit record stays compact for very wide tables). - **`trusted_models`** — List of manifest `unique_id`s whose data is treated as known-clean. A failure on a trusted model surfaces as `failed-on-known-clean-data` (drop, presumed buggy test) rather than `kept`. Opt-in only (Q1=B). Validated against the manifest at `prune_tests` entry — typos raise `PruneTrustedModelNotFoundError` BEFORE any warehouse call (DEC-008). diff --git a/docs/warehouse-adapter-ops.md b/docs/warehouse-adapter-ops.md index 8a4becf1..54253519 100644 --- a/docs/warehouse-adapter-ops.md +++ b/docs/warehouse-adapter-ops.md @@ -113,13 +113,13 @@ The BigQuery adapter is opinionated about cost on every query. v0.2 may re-enable caching behind an explicit opt-in; in v0.1 it is unconditionally off. - **Per-call `timeout_ms`** (DEC-013 of issue #6): pass an integer to - `_default_job_config(timeout_ms=...)` to set + `_default_job_config(stage="...", timeout_ms=...)` to set `QueryJobConfig.job_timeout_ms`; BigQuery cancels the job server-side at expiry. Bytes-scanned through the cancellation point - still bill — set conservatively. Default `None` (no timeout). The - prune layer (#6) uses this for per-test budget enforcement; the - adapter's own `sample_rows` / `column_stats` / `run_test_sql` paths - leave it unset. + still bill — set conservatively. Default `None` (no timeout). + Reserved for v0.2 prune layer integration (issue #6 ships with + `total_budget_seconds` enforcement only; v0.1 has no public + `WarehouseAdapter.run_test_sql` kwarg for per-test timeouts). - **BigQuery job labels are auto-set** on every query: - `signalforge_stage` — the pipeline stage that issued the query. Values are `warehouse_sample` (from `sample_rows`), `warehouse_stats` diff --git a/src/signalforge/prune/config.py b/src/signalforge/prune/config.py index 4f159858..6607bac1 100644 --- a/src/signalforge/prune/config.py +++ b/src/signalforge/prune/config.py @@ -96,10 +96,20 @@ class PruneConfig(BaseModel): trusted_models: tuple[str, ...] = () """Manifest ``unique_id``s whose data is treated as known-clean - (Q1=B, opt-in only). A failure on a trusted model surfaces as a real - failure rather than the "drop on clean-data failure" branch. + (Q1=B, opt-in only). + + Routing semantics — a non-zero failure count on a candidate test: + + * **trusted model** (``model.unique_id`` in this tuple): the data + is presumed clean, so the test is presumed buggy → drop with + ``reason="failed-on-known-clean-data"``. + * **untrusted model** (default): the data may be wrong, ship the + test for review → keep with ``reason="kept"``. + Validated against the manifest at :func:`prune_tests` entry per - DEC-008 — NOT here.""" + DEC-008 — NOT here. A typo'd ``unique_id`` raises + :class:`signalforge.prune.errors.PruneTrustedModelNotFoundError` + BEFORE any warehouse call is issued.""" partition_filter: PartitionFilter | None = None """Optional :class:`PartitionFilter` scoping every sample query diff --git a/src/signalforge/warehouse/adapters/_client.py b/src/signalforge/warehouse/adapters/_client.py index 583db73b..2644051d 100644 --- a/src/signalforge/warehouse/adapters/_client.py +++ b/src/signalforge/warehouse/adapters/_client.py @@ -73,7 +73,7 @@ def make_real_client( raise WarehouseAuthError(message=str(exc)) from exc -def make_query_job_config( +def _make_query_job_config( *, max_bytes_billed: int, stage: str, @@ -229,7 +229,7 @@ def row_to_dict(row: Any) -> dict[str, Any]: __all__ = [ "_BQClientProtocol", - "make_query_job_config", + "_make_query_job_config", "make_real_client", "map_bq_exception", "row_to_dict", diff --git a/src/signalforge/warehouse/adapters/bigquery.py b/src/signalforge/warehouse/adapters/bigquery.py index 1875d82d..58f2bab7 100644 --- a/src/signalforge/warehouse/adapters/bigquery.py +++ b/src/signalforge/warehouse/adapters/bigquery.py @@ -56,7 +56,7 @@ from signalforge.warehouse._sql_safety import validate_identifier, validate_test_sql from signalforge.warehouse.adapters._client import ( _BQClientProtocol, - make_query_job_config, + _make_query_job_config, make_real_client, map_bq_exception, row_to_dict, @@ -229,16 +229,17 @@ def _default_job_config(self, *, stage: str, timeout_ms: int | None = None) -> A for v0.2 cost attribution. ``timeout_ms`` (DEC-013 of issue #6, AR-B2): when not ``None``, - threaded through to :func:`make_query_job_config` so the - underlying ``QueryJobConfig.job_timeout_ms`` is set. The prune - layer (#6 US-009) is the only caller that supplies a non-None - value; the warehouse adapter's own ``sample_rows`` / - ``column_stats`` / ``run_test_sql`` paths leave it ``None`` so - existing behaviour is unchanged. + threaded through to :func:`_make_query_job_config` so the + underlying ``QueryJobConfig.job_timeout_ms`` is set. Currently + used only by tests; the prune layer (issue #6) reserves this + for v0.2 budget enforcement and does NOT thread a non-None + value through ``run_test_sql`` in v0.1. The warehouse adapter's + own ``sample_rows`` / ``column_stats`` / ``run_test_sql`` paths + leave it ``None``. """ from signalforge import __version__ - return make_query_job_config( + return _make_query_job_config( max_bytes_billed=self._max_bytes_billed, stage=stage, version=__version__, diff --git a/tests/warehouse/test_bigquery_unit.py b/tests/warehouse/test_bigquery_unit.py index df353179..a1548399 100644 --- a/tests/warehouse/test_bigquery_unit.py +++ b/tests/warehouse/test_bigquery_unit.py @@ -113,9 +113,9 @@ def test_make_query_job_config_sets_use_query_cache_false() -> None: from google.cloud import bigquery from signalforge import __version__ - from signalforge.warehouse.adapters._client import make_query_job_config + from signalforge.warehouse.adapters._client import _make_query_job_config - cfg = make_query_job_config(stage="warehouse_sample", max_bytes_billed=12345) + cfg = _make_query_job_config(stage="warehouse_sample", max_bytes_billed=12345) assert isinstance(cfg, bigquery.QueryJobConfig) assert cfg.use_query_cache is False diff --git a/tests/warehouse/test_default_job_config_timeout.py b/tests/warehouse/test_default_job_config_timeout.py index c8e898ed..2ea6ddee 100644 --- a/tests/warehouse/test_default_job_config_timeout.py +++ b/tests/warehouse/test_default_job_config_timeout.py @@ -1,4 +1,4 @@ -"""``timeout_ms`` plumbing on ``make_query_job_config`` / ``_default_job_config``. +"""``timeout_ms`` plumbing on ``_make_query_job_config`` / ``_default_job_config``. Issue #6 US-002 (DEC-013, AR-B2) extends the single-seam ``QueryJobConfig`` factory with a ``timeout_ms`` kwarg threaded into @@ -16,7 +16,7 @@ from __future__ import annotations from signalforge.warehouse import BigQueryAdapter -from signalforge.warehouse.adapters._client import make_query_job_config +from signalforge.warehouse.adapters._client import _make_query_job_config from tests.warehouse._fake import FakeBigQueryClient @@ -27,7 +27,7 @@ def test_make_query_job_config_no_timeout_leaves_unset() -> None: ``run_test_sql``) supply no ``timeout_ms``; their behaviour must not change. The BigQuery SDK exposes an unset timeout as ``None``. """ - cfg = make_query_job_config(max_bytes_billed=100_000_000, stage="warehouse_test") + cfg = _make_query_job_config(max_bytes_billed=100_000_000, stage="warehouse_test") assert cfg.job_timeout_ms is None @@ -39,7 +39,7 @@ def test_make_query_job_config_explicit_timeout_propagates() -> None: both shapes so a future SDK upgrade that keeps the value as ``int`` doesn't silently flip the contract. """ - cfg = make_query_job_config( + cfg = _make_query_job_config( max_bytes_billed=100_000_000, stage="warehouse_test", timeout_ms=30_000,