From 88879fdac7a199fe9654412910f669d035ed60bb Mon Sep 17 00:00:00 2001 From: bioedca Date: Mon, 3 Aug 2026 16:14:43 -0500 Subject: [PATCH 1/3] =?UTF-8?q?feat(calib):=20P3-07=20recalibration=20stac?= =?UTF-8?q?k=20=E2=80=94=20calib-only=20temperature=20scaling=20+=20Saeren?= =?UTF-8?q?s/Elkan=20prior-shift?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inputs: calib/temperature.py::{fit_temperature, apply_temperature, STACK_ORDER} (P2-13); the P3-02 `calib` column contract; stage2/model.py "tbox_logit" (B,) raw 1-logit head. Outputs: src/tbox_finder/calib/recalibrate.py::{temperature_scale, prior_shift, calibrated_posterior, assign_rung, rung_labels, prior_from_odds_ratio, deployment_prior_in_prd_band}; tests/unit/test_recalibrate.py; dev-log stanza. ADR-0005 D11 pins the stack order (train -> temperature-scale -> prior-shift) and, more consequentially, which posterior is graded: GATE-2's in-distribution ECE is measured on the NAMED posterior, temperature-scaled and PRE prior-shift. calibrated_posterior therefore returns both, with gated_posterior_key naming the former inside the payload. One temperature fit, not two. Stage 2's head is a single logit, and stacking it as [0, z] makes P2-13's multi-class fit identically the binary one (softmax([0, bz])[1] = sigmoid(bz)) — the same reduction stage2/losses.py already uses for the binary term. The adapter changes inputs, never arithmetic, so P2-13's degenerate-limit refusals carry through. Delegation makes "A agrees with B" a tautology, so the primary test is a hand-derived closed form: z=[+a,+a,-a], y=[1,0,0] => 3*sigmoid(b*a)=2 => b* = ln2/a exactly, plus a second root at a second scale, a ternary search in the test (minimises the value, not the derivative's root), and a manufactured-miscalibration recovery of a known T. "T fit only on calib, never test" is structural: temperature_scale takes the whole table plus a per-row rung and selects the calib rows itself. Around it — an emptiness raise (an all-False calib column filters clean and fits on nothing, which reads exactly like a working filter), an unknown-rung-token raise (a vanished row shrinks the set while the counts look fine), a single-class raise, and assign_rung re-deriving the P3-02 invariant with masking.is_missing (bool(nan) is True, and pandas 3 delivers nulls as NaN). Fixtures are asymmetric 7/4/3/5 and assert identity both ways round. No deployment prior is pinned and none is invented: ~10^3-10^4:1 is PRD prose only, so both priors are required kwargs with no defaults, a half-specified shift raises, and the PRD band is reporting-only (shown to discriminate — D7's own 100:1 falls outside it). No T is fitted here: nothing in the repo reads a Stage-2 checkpoint back, so real calib-split logits do not exist yet. The score producer belongs to P3-08/P3-10. Path drift recorded in imp.md: calib/recalibrate.py, not calibration/recalibrate.py — the calib package already exists and the block's sibling calibration/ paths drifted at P0-31 too. Validation: 108/108 unit (test_recalibrate), test_temperature 81 unchanged; ruff 0.15.15 + black 25.11.0 clean; 15 source sabotages, each RED against its NAMED test, each restored byte-identically. No ADR amendment, no §7 sign-off: D11 is implemented verbatim and the one value that would need a pin is the one the module refuses to supply. --- analyses/phase3_log.qmd | 84 ++++ src/tbox_finder/calib/__init__.py | 11 +- src/tbox_finder/calib/recalibrate.py | 523 ++++++++++++++++++++ tests/unit/test_recalibrate.py | 685 +++++++++++++++++++++++++++ 4 files changed, 1302 insertions(+), 1 deletion(-) create mode 100644 src/tbox_finder/calib/recalibrate.py create mode 100644 tests/unit/test_recalibrate.py diff --git a/analyses/phase3_log.qmd b/analyses/phase3_log.qmd index 0b6a5c3e..583a40d3 100644 --- a/analyses/phase3_log.qmd +++ b/analyses/phase3_log.qmd @@ -1631,3 +1631,87 @@ result, and a reasonable place to stop reviewing rather than a claim that nothin **Validation.** Trainer **121 pass**, sizing 20, overrides 22, backfill 4; ruff + black clean. **Commit.** Feature branch `p3-06-stage2-lora`; squash SHA derived at the phase-exit gate. + +## Step P3-07 — 2026-08-03: the recalibration stack, and two numbers it refuses to invent + +**Inputs.** `calib/temperature.py::{fit_temperature, apply_temperature, STACK_ORDER}` (P2-13, +the exact convex β = 1/T bisection); the P3-02 `calib` column contract (859 positives + 230 +decoys, every one on the `fold_random == "train"` rung, asserted disjoint from val/test at +build time); the Stage-2 binary-head contract `Stage2Model.forward -> "tbox_logit"`, shape +`(B,)`, one raw logit. **Process.** `src/tbox_finder/calib/recalibrate.py` ships the ADR-0005 +D11 stack for the Stage-2 head: `temperature_scale` fits T, `prior_shift` applies the +Saerens/Elkan log-odds correction, and `calibrated_posterior` returns **both** the named +posterior (temperature-scaled, pre-shift — the GATE-2-gated object) and the prior-shifted one, +with `gated_posterior_key` naming the former inside the payload. **Outputs.** One module, one +test module (108 tests), no data artifact and no fitted T — see the last paragraph. + +**There is one temperature fit in this repo, and this is not a second one.** Stage 2's head is +a single logit; stacking it as `[0, z]` makes P2-13's multi-class fit *identically* the binary +one, because `softmax([0, βz])[1] = σ(βz)`, so its NLL is exactly the binary cross-entropy at +`βz` — the same reduction `stage2/losses.py` already uses to route the binary term through the +audited focal-CE kernel. The adapter therefore touches the *inputs* and never the arithmetic, +and P2-13's two degenerate-limit refusals (perfect separation ⇒ β → ∞; worse-than-uniform ⇒ +β → 0) carry through unchanged. The cost of delegating is that "A agrees with B" becomes a +tautology, so the primary test is neither: for `z = [+a, +a, −a]`, `y = [1, 0, 0]` the +stationarity condition `Σ z_i(σ(βz_i) − y_i) = 0` collapses to `3σ(βa) = 2`, giving +**`β* = ln2/a` exactly**. A second case at a second scale (`4σ = 3` ⇒ `β* = ln3/a`), a ternary +search in the *test* that minimises the value rather than rooting the derivative, and a +manufactured-miscalibration draw that recovers `T ≈ 2.5` from 20,000 rows are the independent +confirmations; the delegation check is present and labelled secondary. + +**"T fit only on `calib`, never test" is a property of the code, not a claim about the caller.** +`temperature_scale` takes the whole table plus a per-row rung label and selects the `calib` +rows itself, so there is no argument through which a graded row can reach the fit. Three +refusals surround that: an **empty** selection raises with the full census (the all-`False` +calib column filters clean and fits on nothing, which reads exactly like a working filter); an +**unrecognised rung token** raises rather than being skipped (a row that silently vanishes +shrinks the calibration set while every count still looks fine); and a **single-class** calib +rung raises. `assign_rung` re-derives the P3-02 invariant — a calib row lives in the `train` +rung — at the point of use, and tests missingness with `masking.is_missing`, because +`bool(float("nan")) is True` and under the training env's pandas 3 a null cell arrives as NaN, +so the naive guard would promote a row with **no** calibration flag straight into the fit. + +**The fixtures are asymmetric on purpose.** The rung arms are 7 calib / 4 train / 3 val / 5 +test, and both the calib and test arms are independently fittable, so the inversion test can +run both ways round: swapping the two senses is asserted to change `n_fitted` *and* to produce +exactly the T of the test arm fitted alone. Balanced arms with count-only assertions would +refuse the right number of the wrong rows and pass. + +**Two numbers this step declines to produce, and why that is the deliverable rather than a +gap.** (1) **No deployment prior is pinned.** PRD.md:188 and ADR-0005:80 give the genome-scale +prevalence only as prose, `~10³–10⁴:1` (negatives per positive); nothing in the repo encodes a +value. Pinning a scalar would be a new blinded-frozen default needing ADR sign-off, and +defaulting one silently would put a fabricated quantity into every downstream posterior. So +both priors are required keyword arguments with no defaults, a half-specified shift raises, and +`DEPLOYMENT_PRIOR_RANGE` exists only as the band a supplied prior is *reported* against — a +predicate shown to discriminate, since ADR-0005 D7's own `100:1` benchmark prevalence must fall +outside it. (2) **No T is fitted here.** Nothing in the repo reads a Stage-2 checkpoint back — +`stage2_heads.pt` and `lora_adapter/` have write sites and no readers, and `train.py::evaluate` +discards per-row logits — so real `calib`-split logits do not yet exist. This step ships and +certifies the machinery; producing the scores, and with them the first real T, belongs to the +step that first needs a calibrated posterior (P3-08/P3-10). A T invented to fill the gap would +look exactly like a measured one. + +**Path drift, recorded not silently taken.** imp.md names +`src/tbox_finder/calibration/recalibrate.py`; the module lands at +`src/tbox_finder/calib/recalibrate.py` because a `calib/` package already exists and two +near-homonymous calibration namespaces is a permanent confusion. imp.md's `calibration/` paths +have drifted once already on their own — `calibration/ece.py::binned_ece` shipped as +`metrics.py::binned_ece` at P0-31 — and P3-02 carries an identically-shaped recorded +correction. The P3-07 block in imp.md now records this one. + +**Validation.** `test_recalibrate` **108 pass**, `test_temperature` 81 pass (unchanged), full +unit tier 3,125 pass with the same 12 failures the untouched `main` shows in this runner (a +py3.11/numpy-2.4.2 local env, not the pinned CI stack); ruff 0.15.15 + black 25.11.0 clean. +**15 source sabotages, each RED against its NAMED test, each restored byte-identically** (fold +sense inverted, emptiness guard removed, unknown-token refusal removed, `is_missing` weakened +to `is None`, P3-02 invariant dropped, shift direction inverted, `+` flipped to `−`, gated key +repointed at the shifted posterior, a deployment prior given a default, the half-specified +guard removed, the PRD band widened to swallow D7's `100:1`, the branch-wise sigmoid replaced +by the overflowing one, single-class guard removed, priors of exactly 0/1 admitted, +`prior_from_odds_ratio` inverted). + +**Decisions.** No new ADR and no §7 item-2 sign-off: D11 pins the stack order and the named +posterior, both of which this implements verbatim, and the one value that would have needed a +pin is the one the module refuses to supply. **Commit.** Feature branch +`p3-07-calibration-stack`; squash SHA derived at the phase-exit gate. diff --git a/src/tbox_finder/calib/__init__.py b/src/tbox_finder/calib/__init__.py index cc4413d3..3d216a7d 100644 --- a/src/tbox_finder/calib/__init__.py +++ b/src/tbox_finder/calib/__init__.py @@ -14,8 +14,17 @@ (per-class one-vs-rest binned ECE through the frozen ``metrics.binned_ece``, with cluster-blocked CIs). GATE-2's gated ECE is P3-exit business on the P3-02 ``calib`` carve — nothing here is a calibration claim, and no ``T`` fitted here is shipped. +``recalibrate`` + The P3-07 stack for the **Stage-2 binary head**: ``T`` fitted on the P3-02 ``calib`` + rung (and structurally on nothing else), then the Saerens/Elkan log-odds prior-shift, + then a producer exposing **both** the D11 named posterior (temperature-scaled, + pre-prior-shift — the GATE-2-gated object) and the prior-shifted one (reported, + non-gated). It **reuses** ``temperature.fit_temperature`` rather than carrying a second + temperature fit: Stage 2's one logit stacked as ``[0, z]`` makes that multi-class fit + identically the binary one. No deployment prior is pinned here — the PRD gives it only + as prose, so both priors are required arguments with no defaults. """ from __future__ import annotations -__all__ = ["temperature"] +__all__ = ["recalibrate", "temperature"] diff --git a/src/tbox_finder/calib/recalibrate.py b/src/tbox_finder/calib/recalibrate.py new file mode 100644 index 00000000..ed7e3596 --- /dev/null +++ b/src/tbox_finder/calib/recalibrate.py @@ -0,0 +1,523 @@ +"""The P3-07 recalibration stack — temperature-scale on the disjoint ``calib`` split, then +the Saerens/Elkan prior-shift (PRD §11/§12; ADR-0005 D11). + +D11 pins the **order** — ``train -> temperature_scale -> prior_shift`` — and, more +consequentially, pins *which posterior is graded*: GATE-2's in-distribution ECE ≤ 0.05 is +measured on the **named posterior**, the temperature-scaled posterior **before** the +deployment prior-shift, at the in-distribution split's own prevalence. The prior-shifted +posterior is miscalibrated at benchmark prevalence *by construction*, so it is reported +non-gated. This module therefore never returns "the" posterior: :func:`calibrated_posterior` +returns **both**, under keys that say which is which, plus ``gated_posterior_key`` naming the +one GATE-2 grades. A caller cannot pick the wrong object by accident, and a report built from +this dict carries the distinction into the artifact. + +**The temperature fit is not reimplemented here.** ``calib.temperature.fit_temperature`` +(P2-13, ADR-0005 A11 Pin 2/3) already solves the exact convex 1-D problem by bisection on +``β = 1/T``, is seedless and optimiser-free, and refuses the two degenerate limits by name +instead of returning a bracket endpoint. Stage 2's binary head is **one** logit +(``stage2/model.py::Stage2Model.forward -> "tbox_logit"``, shape ``(B,)``, raw), and stacking +it as ``[0, z]`` makes that multi-class fit *identically* the binary one: +``softmax([0, βz])[1] = σ(βz)``, so ``Σ_i [logsumexp(0, βz_i) − βz_i·y_i]`` is exactly the +binary cross-entropy at logit ``βz``. That is the same reduction ``stage2/losses.py`` already +uses to route the binary term through the audited focal-CE kernel. A second temperature fit +would mean fixing one copy and shipping the bug in the other, so there is one fit and this +module adapts its *inputs* — the non-tautological guard on the shared kernel is the +hand-derived closed-form root in ``tests/unit/test_recalibrate.py``, not agreement with the +thing being delegated to. + +**"T is fitted on ``calib``, never on test" is enforced structurally, not asserted after the +fact.** :func:`temperature_scale` takes a per-row rung label and fits on the ``calib`` rows it +selects itself; it cannot be handed a graded row to fit on, because it never fits on a row it +did not select by name. It refuses an empty selection (a mask whose legitimate value is never +zero is the one that must raise — a vacuously-clean filter reads exactly like a working one), +refuses any rung token outside the pinned vocabulary (so a stringified null or a typo cannot +silently evaporate a row), and records ``n_by_rung`` over *every* rung so what was excluded is +auditable rather than assumed. :func:`assign_rung` re-derives the P3-02 invariant — a ``calib`` +row lives in the ``fold_random == "train"`` rung — at the point of use rather than trusting it +upstream. + +**No deployment prior is pinned here, and none is invented.** PRD.md:188 and ADR-0005:80 give +the genome-scale prevalence only as prose, ``~10³–10⁴:1`` (negatives per positive); no number +for it exists anywhere in the repo. Pinning a scalar π_deploy would be a new blinded-frozen +default requiring ADR sign-off (CLAUDE.md §7 item 2), and defaulting one silently would put a +fabricated quantity into every downstream posterior (§10.3). So :func:`prior_shift` takes +**both** priors as required keyword arguments with no defaults, :func:`calibrated_posterior` +refuses a half-specified shift, and :data:`DEPLOYMENT_PRIOR_RANGE` is exposed only as the +PRD-quoted band a supplied prior can be *reported* against +(:func:`deployment_prior_in_prd_band`) — never as a value the code supplies on its own. The +prior the scan actually deploys against is P5 business. + +Structure: pinned vocabulary and the PRD band; pure numeric helpers; the two stack stages; the +producer that exposes both posteriors. numpy-only (no scipy, no sklearn, no torch), so the +unit tier runs in CI as-is. +""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np + +from tbox_finder import masking +from tbox_finder.calib.temperature import ( + BETA_BRACKET, + FIT_MAX_ITER, + FIT_TOL, + STACK_ORDER, + TemperatureFit, + TemperatureFitError, + apply_temperature, + fit_temperature, +) + +__all__ = [ + "CALIB_RUNG", + "DEPLOYMENT_PRIOR_ODDS_RANGE", + "DEPLOYMENT_PRIOR_RANGE", + "GATED_POSTERIOR_KEY", + "GRADED_RUNGS", + "NAMED_POSTERIOR_KEY", + "PRIOR_SHIFTED_POSTERIOR_KEY", + "RUNG_VOCABULARY", + "STACK_ORDER", + "TemperatureFitError", + "TemperatureScaleResult", + "apply_temperature", + "assign_rung", + "calibrated_posterior", + "deployment_prior_in_prd_band", + "log_odds_shift", + "odds_ratio_from_prior", + "posterior_from_logits", + "prior_from_odds_ratio", + "prior_shift", + "rung_labels", + "temperature_scale", +] +# ``TemperatureFitError`` and ``apply_temperature`` are re-exported so running one stack does +# not mean importing two calibration modules. They are the same objects, not wrappers — there +# is one temperature implementation and this module adapts its inputs, never its arithmetic. + +STEP = "P3-07" +GENERATED_BY = "src/tbox_finder/calib/recalibrate.py" +ADR = "ADR-0005 D11 (stack order + named posterior); PRD §11/§12" + +# --------------------------------------------------------------------------- # +# Pinned vocabulary (PRD §9.2 scheme A + the P3-02 carve) and the PRD prior band +# --------------------------------------------------------------------------- # + +#: The rung ``T`` is fitted on — the P3-02 disjoint calibration carve (``calib`` column of +#: ``data/processed/splits/split_assignments.parquet``, a boolean overlay on the ``train`` +#: rung, whole-cluster-assigned and asserted disjoint from val/test at build time). +CALIB_RUNG = "calib" + +#: The rungs GATE-2 and GATE-4 grade on. Named so the refusal message can say *why* a row is +#: not fittable, rather than reporting an anonymous count. +GRADED_RUNGS: tuple[str, ...] = ("val", "test") + +#: The closed vocabulary :func:`temperature_scale` accepts. Closed on purpose: an unknown +#: token is an error, not a row to skip. A rung column that arrives carrying ``"None"`` or +#: ``"Train"`` would otherwise be filtered out silently and the fit would run clean on a +#: quietly smaller set ([[stringified-null-survives-missing-checks]]). +RUNG_VOCABULARY: tuple[str, ...] = ("calib", "train", "val", "test") + +#: PRD.md:188 / ADR-0005:80 state the genome-scale deployment prevalence as ``~10³–10⁴:1`` +#: **negatives per positive** — the same units as ADR-0005 D7's pinned ``100:1`` benchmark +#: decoy prevalence and §9.1's ``~10:1`` training seed ratio. It is a **prose band**, not a +#: pinned scalar: nothing in the repo encodes a deployment prior, and this module does not +#: create one (see the module docstring). +DEPLOYMENT_PRIOR_ODDS_RANGE: tuple[float, float] = (1e3, 1e4) + +#: The same band as a positive-class prior, ``π = 1/(1 + negatives_per_positive)``, ascending. +#: Reporting-only: :func:`deployment_prior_in_prd_band` reads it, no function defaults to it. +DEPLOYMENT_PRIOR_RANGE: tuple[float, float] = ( + 1.0 / (1.0 + DEPLOYMENT_PRIOR_ODDS_RANGE[1]), + 1.0 / (1.0 + DEPLOYMENT_PRIOR_ODDS_RANGE[0]), +) + +#: Keys of the :func:`calibrated_posterior` payload. ``GATED_POSTERIOR_KEY`` is the D11 named +#: posterior — the temperature-scaled, **pre**-prior-shift object GATE-2 grades. +NAMED_POSTERIOR_KEY = "named_posterior" +PRIOR_SHIFTED_POSTERIOR_KEY = "prior_shifted_posterior" +GATED_POSTERIOR_KEY = NAMED_POSTERIOR_KEY + + +@dataclass(frozen=True) +class TemperatureScaleResult: + """A temperature certified to have been fitted on the ``calib`` rung and nothing else. + + ``n_by_rung`` is the census of the *whole* input, as sorted ``(rung, count)`` pairs — the + excluded rows are counted, not discarded silently, so a reader can tell "fitted on the + 859 calib rows of a 30,542-row table" from "fitted on the 859 rows someone handed me". + ``fit`` is the underlying :class:`~tbox_finder.calib.temperature.TemperatureFit`, carrying + the stationarity residual so convergence can be re-derived rather than trusted. + """ + + temperature: float + beta: float + fitted_on: str + n_fitted: int + n_total: int + n_by_rung: tuple[tuple[str, int], ...] + fit: TemperatureFit + + def counts(self) -> dict[str, int]: + """``n_by_rung`` as a mapping (the dataclass stores pairs so it stays hashable).""" + return dict(self.n_by_rung) + + +# --------------------------------------------------------------------------- # +# Pure numeric helpers +# --------------------------------------------------------------------------- # +def _require_prior(value: Any, name: str) -> float: + """A class prior must be a real number strictly inside ``(0, 1)``. + + Strictly: ``π = 0`` or ``π = 1`` sends the log-odds to ``∓inf``, and a shift of ``inf`` + is not a correction, it is a constant posterior wearing one. ``bool`` is rejected before + the range test because ``bool`` is an ``int`` subclass and ``True`` would otherwise read + as the prior ``1.0``. + """ + if isinstance(value, bool) or not isinstance(value, (int, float, np.floating, np.integer)): + raise ValueError(f"{name} must be a real number in (0, 1), got {value!r}") + prior = float(value) + if not math.isfinite(prior) or not 0.0 < prior < 1.0: + raise ValueError(f"{name} must be finite and strictly inside (0, 1), got {prior!r}") + return prior + + +def prior_from_odds_ratio(negatives_per_positive: float) -> float: + """Positive-class prior ``π = 1/(1 + r)`` from an ``r : 1`` negative:positive ratio. + + The repo states every prevalence as a negative:positive ratio (ADR-0005 D7's ``100:1`` + benchmark, §9.1's ``~10:1`` training seed, PRD.md:188's ``~10³–10⁴:1`` genome scale), and + every formula here wants a prior. One conversion, so the two never drift. + """ + if isinstance(negatives_per_positive, bool) or not isinstance( + negatives_per_positive, (int, float, np.floating, np.integer) + ): + raise ValueError( + f"negatives_per_positive must be a real number, got {negatives_per_positive!r}" + ) + ratio = float(negatives_per_positive) + if not math.isfinite(ratio) or ratio <= 0.0: + raise ValueError(f"negatives_per_positive must be finite and > 0, got {ratio!r}") + return 1.0 / (1.0 + ratio) + + +def odds_ratio_from_prior(prior: float) -> float: + """Inverse of :func:`prior_from_odds_ratio`: ``r = (1 − π)/π``.""" + p = _require_prior(prior, "prior") + return (1.0 - p) / p + + +def deployment_prior_in_prd_band(prior: float) -> bool: + """Does ``prior`` sit inside the PRD/ADR-quoted ``~10³–10⁴:1`` genome-scale band? + + **Reporting only.** A ``False`` is not a refusal — the deployment prior a corpus actually + carries is a P5 quantity, and PRD §7.2 explicitly contemplates near-zero-prior corpora + (Archaea/DPANN) with their *own* prior outside this Bacteria-derived band. This predicate + exists so an artifact can say which side of the quoted band its prior fell on instead of + leaving a reader to eyeball it. + """ + p = _require_prior(prior, "prior") + lo, hi = DEPLOYMENT_PRIOR_RANGE + return lo <= p <= hi + + +def _as_logits(logits: Any, *, name: str = "logits") -> np.ndarray: + """1-D float64 view of a raw binary logit vector, non-finite refused.""" + z = np.asarray(logits, dtype=np.float64) + if z.ndim != 1: + raise ValueError(f"{name} must be 1-D (one raw logit per row), got shape {z.shape}") + if z.size and not np.isfinite(z).all(): + raise ValueError(f"{name} carries a non-finite value") + return z + + +def posterior_from_logits(logits: Any) -> np.ndarray: + """``σ(z)`` computed branch-wise so neither tail overflows. + + ``exp(-z)`` overflows for very negative ``z`` and ``exp(z)`` for very positive ``z``; each + branch below uses only the exponential that underflows harmlessly toward 0. + """ + z = _as_logits(logits) + out = np.empty_like(z) + pos = z >= 0.0 + out[pos] = 1.0 / (1.0 + np.exp(-z[pos])) + ez = np.exp(z[~pos]) + out[~pos] = ez / (1.0 + ez) + return out + + +def log_odds_shift(*, source_prior: float, target_prior: float) -> float: + """The Saerens/Elkan additive log-odds correction ``logit(π_target) − logit(π_source)``. + + Direction is source → target: the classifier's posterior is calibrated at ``source_prior`` + (for the D11 stack, the prevalence of the ``calib`` split ``T`` was fitted on), and the + correction moves it to ``target_prior``. Both are required and neither is defaulted, so + the direction cannot be inverted by omission. + """ + p_s = _require_prior(source_prior, "source_prior") + p_t = _require_prior(target_prior, "target_prior") + return math.log(p_t / (1.0 - p_t)) - math.log(p_s / (1.0 - p_s)) + + +# --------------------------------------------------------------------------- # +# Stack stage 1 — temperature scaling, fitted on `calib` and nowhere else +# --------------------------------------------------------------------------- # +def assign_rung(*, calib: Any, fold_random: Any) -> str: + """The rung of one row, from its ``calib`` flag and its scheme-A ``fold_random`` value. + + The single derivation of the rung label, so the ``calib``-overlay-on-``train`` shape lives + in one place. Fail-closed in three directions, each of which is a real failure mode rather + than a hypothetical: + + * a **missing** ``calib`` or ``fold_random`` raises — tested with + :func:`tbox_finder.masking.is_missing`, because under pandas 3 a null string cell + arrives as ``NaN``, which is *truthy*, so ``bool(value)`` would read a missing calib + flag as ``True`` and a missing fold as the string ``"nan"``; + * a ``calib`` row outside the ``train`` rung raises — that is the P3-02 invariant + (``splits._assert_calib_disjoint``), re-derived here at the point where violating it + would corrupt a gate rather than trusted from upstream; + * an unrecognised ``fold_random`` raises, instead of being bucketed as "other". + """ + if masking.is_missing(calib): + raise ValueError("calib is missing — a row with no calibration flag has no rung") + fold = masking.row_text(fold_random).strip() + if not fold: + raise ValueError("fold_random is missing — a row with no scheme-A fold has no rung") + in_calib = bool(calib) + if in_calib: + if fold != "train": + raise ValueError( + f"calib row carries fold_random={fold!r}, but the P3-02 carve is taken from " + "the training folds only (splits._assert_calib_disjoint); a calib row in a " + "graded fold is leakage, not a rung" + ) + return CALIB_RUNG + if fold not in RUNG_VOCABULARY: + raise ValueError(f"unrecognised fold_random {fold!r}; known: {sorted(RUNG_VOCABULARY)}") + if fold == CALIB_RUNG: + raise ValueError( + "fold_random == 'calib' with calib False — calib is a boolean overlay on the " + "'train' rung, never a fourth fold_random value" + ) + return fold + + +def rung_labels(*, calib: Sequence[Any], fold_random: Sequence[Any]) -> list[str]: + """:func:`assign_rung` over two equal-length columns, erroring with the offending index.""" + if len(calib) != len(fold_random): + raise ValueError( + f"calib carries {len(calib)} rows but fold_random carries {len(fold_random)}" + ) + out: list[str] = [] + for i, (c, f) in enumerate(zip(calib, fold_random, strict=True)): + try: + out.append(assign_rung(calib=c, fold_random=f)) + except ValueError as exc: + raise ValueError(f"row {i}: {exc}") from exc + return out + + +def _rung_census(rungs: Sequence[str]) -> tuple[tuple[str, int], ...]: + counts = {rung: 0 for rung in RUNG_VOCABULARY} + for rung in rungs: + counts[rung] += 1 + return tuple(sorted(counts.items())) + + +def temperature_scale( + logits: Any, + labels: Any, + *, + rung: Sequence[Any], + bracket: tuple[float, float] = BETA_BRACKET, + tol: float = FIT_TOL, + max_iter: int = FIT_MAX_ITER, + grad_tol: float | None = None, +) -> TemperatureScaleResult: + """Fit one shared scalar ``T`` on the ``calib`` rows of ``(logits, labels)`` (D11 stage 2). + + ``logits`` is the Stage-2 binary head's raw output, one logit per row + (``stage2/model.py`` ``"tbox_logit"``, shape ``(B,)``); ``labels`` the matching 0/1 + targets; ``rung`` the per-row label from :func:`rung_labels`. The whole table is passed in + and this function does the selecting, which is what makes "fitted on ``calib``, never on + test" a property of the code rather than a claim about the caller. + + The fit itself is :func:`tbox_finder.calib.temperature.fit_temperature` on the ``[0, z]`` + stack — exact, since ``softmax([0, βz])[1] = σ(βz)`` makes its multi-class NLL identically + the binary cross-entropy at ``βz``. Its degenerate-limit refusals therefore carry over + unchanged: a perfectly-separated calibration set (``β → ∞``, ``T → 0``) and a + worse-than-uniform one (``β → 0``, ``T → ∞``) both raise + :class:`~tbox_finder.calib.temperature.TemperatureFitError` by name instead of returning a + bracket endpoint dressed as a temperature. + + Raises ``ValueError`` when the selection is empty or single-class — the two ways this can + run "successfully" on nothing. An all-``False`` calib column is exactly as clean-looking as + a working filter, and a single-class calibration set has no calibration to measure. + """ + z = _as_logits(logits) + y = np.asarray(labels) + if y.ndim != 1: + raise ValueError(f"labels must be 1-D, got shape {y.shape}") + if z.shape[0] != y.shape[0]: + raise ValueError(f"logits carries {z.shape[0]} rows but labels carries {y.shape[0]}") + if len(rung) != z.shape[0]: + raise ValueError(f"logits carries {z.shape[0]} rows but rung carries {len(rung)}") + y = y.astype(np.int64, copy=False) + if y.size and (int(y.min()) < 0 or int(y.max()) > 1): + raise ValueError( + f"labels must be binary 0/1, got range [{int(y.min())}, {int(y.max())}] — the " + "Stage-2 binary head is one logit, not a multi-class score" + ) + + rungs = [masking.row_text(r).strip() for r in rung] + unknown = sorted({r for r in rungs if r not in RUNG_VOCABULARY}) + if unknown: + raise ValueError( + f"unrecognised rung label(s) {unknown}; known: {sorted(RUNG_VOCABULARY)}. An " + "unknown token is refused rather than filtered: a row that silently vanishes " + "shrinks the calibration set while every count still looks clean" + ) + + keep = np.array([r == CALIB_RUNG for r in rungs], dtype=bool) + n_fitted = int(keep.sum()) + if n_fitted == 0: + census = dict(_rung_census(rungs)) + n_graded = sum(census[r] for r in GRADED_RUNGS) + raise ValueError( + f"no rows on the {CALIB_RUNG!r} rung out of {len(rungs)} — an empty calibration " + f"split is not a fit (census: {census}). GATE-2's T is fitted on the P3-02 carve; " + "if it is empty the carve or the join is wrong, and falling back to the remainder " + f"would fit T on the {n_graded} rows of the graded rungs {list(GRADED_RUNGS)}" + ) + y_fit = y[keep] + n_pos = int(y_fit.sum()) + if n_pos == 0 or n_pos == n_fitted: + raise ValueError( + f"the {CALIB_RUNG!r} rung is single-class ({n_pos} positive of {n_fitted}) — " + "there is no calibration to fit on a set with only one outcome" + ) + + z_fit = z[keep] + stacked = np.column_stack((np.zeros_like(z_fit), z_fit)) + fit = fit_temperature( + stacked, + y_fit, + bracket=bracket, + tol=tol, + max_iter=max_iter, + grad_tol=grad_tol, + ) + return TemperatureScaleResult( + temperature=fit.temperature, + beta=fit.beta, + fitted_on=CALIB_RUNG, + n_fitted=n_fitted, + n_total=len(rungs), + n_by_rung=_rung_census(rungs), + fit=fit, + ) + + +# --------------------------------------------------------------------------- # +# Stack stage 3 — the Saerens/Elkan prior-shift +# --------------------------------------------------------------------------- # +def prior_shift(logits: Any, *, source_prior: float, target_prior: float) -> np.ndarray: + """Shift calibrated **logits** from ``source_prior`` to ``target_prior`` (D11 stage 3). + + The Saerens/Elkan correction in the log-odds form ADR-0005 D11 names [Saerens, Latinne & + Decaestecker, *Neural Computation* 14(1):21–41, DOI:10.1162/089976602753284446 (accessed + 2026-08-03); Elkan, *The foundations of cost-sensitive learning*, IJCAI 2001, Theorem 1]: + under a pure prior shift the class-conditional densities are unchanged, so the likelihood + ratio is unchanged and the whole correction is the additive constant + ``logit(π_t) − logit(π_s)``:: + + z' = z + log(π_t/(1−π_t)) − log(π_s/(1−π_s)) + + **Scope, stated because the two halves of that paper are not interchangeable.** This is + the **known-target-prior closed form**. Saerens et al.'s headline contribution is the *EM + procedure* for the case where π_target is *unknown* and must be estimated from unlabelled + deployment data; that estimator is not implemented here and is not what D11 asks for — + D11 shifts "to the deployment prior", i.e. to a prior supplied from outside. If P5 ever + needs π_deploy estimated rather than stated, that is the EM half of the same paper and a + separate piece of machinery, not a parameter of this one. + + Equivalently on the posterior, ``p' = p·r / (p·r + (1−p))`` with + ``r = [π_t/(1−π_t)]·[(1−π_s)/π_s]``. Returning **logits** rather than posteriors is + deliberate: the correction is exactly additive there, so it composes with a second shift + and inverts exactly, while the posterior form round-trips only up to float error. + + Being an additive constant, it is strictly monotone in ``z`` — it moves every posterior + but reorders none, which is why the operating point is set on the ranking and the prior + shift changes only what the score *means*. It is **not** the gated object: a prior-shifted + posterior is miscalibrated at benchmark prevalence by construction, so GATE-2 grades the + pre-shift named posterior and this one is reported non-gated (D11). + """ + z = _as_logits(logits) + return z + log_odds_shift(source_prior=source_prior, target_prior=target_prior) + + +# --------------------------------------------------------------------------- # +# The producer — both posteriors, and which one is gated +# --------------------------------------------------------------------------- # +def calibrated_posterior( + logits: Any, + *, + temperature: float, + source_prior: float | None = None, + target_prior: float | None = None, +) -> dict[str, Any]: + """The D11 stack applied to raw Stage-2 binary logits, exposing **both** posteriors. + + Returns ``named_posterior`` — ``σ(z/T)``, the temperature-scaled, pre-prior-shift object + GATE-2's in-distribution ECE ≤ 0.05 is graded on — and ``prior_shifted_posterior``, the + Saerens/Elkan-corrected one, which is ``None`` unless both priors are given. + ``gated_posterior_key`` names the former inside the payload so a downstream report cannot + quietly grade the wrong one. + + Both priors or neither: a half-specified shift raises rather than being completed with a + default, because the only defaultable value would be a deployment prior the repo does not + have (the PRD gives ``~10³–10⁴:1`` as prose, and inventing a scalar from it is exactly the + fabrication CLAUDE.md §10.3 forbids). + """ + if (source_prior is None) != (target_prior is None): + missing = "target_prior" if target_prior is None else "source_prior" + raise ValueError( + f"prior_shift needs both source_prior and target_prior; {missing} is missing. " + "Neither is defaulted: the deployment prior is PRD prose (~10^3-10^4:1), not a " + "value this module may invent" + ) + tempered = apply_temperature(_as_logits(logits), temperature) + named = posterior_from_logits(tempered) + + payload: dict[str, Any] = { + "stack_order": list(STACK_ORDER), + "temperature": float(temperature), + NAMED_POSTERIOR_KEY: named, + PRIOR_SHIFTED_POSTERIOR_KEY: None, + "gated_posterior_key": GATED_POSTERIOR_KEY, + "prior_shift_applied": False, + "log_odds_shift": None, + "source_prior": None, + "target_prior": None, + "target_prior_in_prd_band": None, + "n_rows": int(named.shape[0]), + } + if source_prior is None: + payload["stack_applied"] = ["train", "temperature_scale"] + return payload + + shift = log_odds_shift(source_prior=source_prior, target_prior=target_prior) + payload[PRIOR_SHIFTED_POSTERIOR_KEY] = posterior_from_logits(tempered + shift) + payload["prior_shift_applied"] = True + payload["log_odds_shift"] = shift + payload["source_prior"] = _require_prior(source_prior, "source_prior") + payload["target_prior"] = _require_prior(target_prior, "target_prior") + payload["target_prior_in_prd_band"] = deployment_prior_in_prd_band(target_prior) + payload["stack_applied"] = list(STACK_ORDER) + return payload diff --git a/tests/unit/test_recalibrate.py b/tests/unit/test_recalibrate.py new file mode 100644 index 00000000..e49002db --- /dev/null +++ b/tests/unit/test_recalibrate.py @@ -0,0 +1,685 @@ +"""P3-07 — the Stage-2 recalibration stack (PRD §11/§12; ADR-0005 D11). + +**Torch-free and scipy-free by construction.** CI's test env installs numpy/pandas/ +scikit-learn and no torch; the whole stack is closed-form or an exact convex 1-D solve, so +every tier here runs where the suite actually runs rather than ``importorskip``-ing green. + +What each tier locks, and why that tier is the one that could bite: + +* **the fit against hand-derived roots** — ``temperature_scale`` delegates to + ``calib.temperature.fit_temperature``, so "the two agree" would be a tautology + ([[promote-dont-duplicate-is-a-correctness-rule]]). The primary check is therefore an + **independently derived closed form**: for ``z = [+a, +a, −a]``, ``y = [1, 0, 0]`` the + binary NLL's stationarity condition ``Σ z_i(σ(βz_i) − y_i) = 0`` reduces to + ``3σ(βa) = 2``, so ``β* = ln2/a`` exactly. A second case with different ``a`` and a + different root (``4σ = 3`` ⇒ ``β* = ln3/a``) pins the scale dependence, a ternary search + in the *test* re-finds the minimum by a different algorithm (minimising the value, not + rooting the derivative), and a manufactured-miscalibration draw recovers a known ``T``; +* **the calib-only guarantee** — the imp.md gate is "``T`` fit only on ``calib``, never + test". A count-only check is blind to a fold-sense inversion, so the fixture uses + **asymmetric** rung sizes and asserts **identity** (the ``T`` from the mixed table equals + the ``T`` from the intended rows fitted alone, and *differs* from the swapped-sense one) + ([[symmetric-count-fixture-blind-to-inversion]]). Every refusal is paired with the + identical clean input succeeding, so a guard that refused everything would fail here + ([[raises-test-needs-a-positive-control]]); +* **the emptiness direction** — an all-``False`` calib column filters clean and fits on + nothing, which reads exactly like a working filter ([[clauses-must-guard-emptiness]], + [[namespace-mismatch-invisible-noop]]). It must raise, and it must say what it saw; +* **the pandas-3 direction** — a missing ``calib`` cell arrives as ``NaN`` in the training + env, and ``NaN`` is **truthy**, so ``bool(cell)`` would silently promote a row with no + flag into the calibration set ([[pandas-3-nan-truthy-in-training-env]]). NaN is asserted + as a literal, in both columns; +* **the prior-shift closed form** — hand-computed log-odds on a known prevalence swing, + the posterior-form equivalence, exact invertibility, and the band predicate shown to + *discriminate* (ADR-0005 D7's own ``100:1`` benchmark prevalence must fall outside the + PRD's ``10³–10⁴:1`` deployment band, or the predicate certifies nothing); +* **the no-fabricated-prior contract** — the module must be unable to supply a deployment + prior on its own, so a half-specified shift raises and neither prior has a default + ([[pinned-constant-that-nothing-reads]] in reverse: the band is read, never defaulted to). +""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest + +from tbox_finder.calib import recalibrate as R +from tbox_finder.calib import temperature as T + + +# --------------------------------------------------------------------------- # +# Helpers — independent arithmetic, never the module's own +# --------------------------------------------------------------------------- # +def _sigmoid(x: float) -> float: + """Plain-math sigmoid, so the expectations are not the module's implementation.""" + return 1.0 / (1.0 + math.exp(-x)) + + +def _binary_nll(z: np.ndarray, y: np.ndarray, beta: float) -> float: + """``Σ_i [softplus(βz_i) − βz_i·y_i]`` — the objective, written out longhand.""" + total = 0.0 + for zi, yi in zip(z.tolist(), y.tolist(), strict=True): + s = beta * zi + total += math.log1p(math.exp(-abs(s))) + max(s, 0.0) - s * yi + return total + + +def _ternary_min(z: np.ndarray, y: np.ndarray, lo: float, hi: float) -> float: + """Minimise the convex NLL by ternary search — a different algorithm from the module's. + + The module bisects the *derivative* to find its root; this narrows a bracket on the + *value*. Agreement between the two is therefore evidence about the answer, not about a + shared implementation. + """ + for _ in range(400): + m1 = lo + (hi - lo) / 3.0 + m2 = hi - (hi - lo) / 3.0 + if _binary_nll(z, y, m1) < _binary_nll(z, y, m2): + hi = m2 + else: + lo = m1 + return 0.5 * (lo + hi) + + +def _table(spec: list[tuple[str, float, int]]) -> tuple[np.ndarray, np.ndarray, list[str]]: + """``[(rung, logit, label), ...]`` → the three parallel columns the API takes.""" + rungs = [row[0] for row in spec] + z = np.array([row[1] for row in spec], dtype=np.float64) + y = np.array([row[2] for row in spec], dtype=np.int64) + return z, y, rungs + + +#: A **non-separable** general-purpose fixture. Separability is not a detail here: the fit +#: legitimately *refuses* a perfectly-separated set (``β → ∞``, ``T → 0``), so a fixture whose +#: labels agree with the sign of every logit tests the refusal path and nothing else. Each arm +#: below therefore carries at least one pair of equal logits with opposite labels. +_NONSEP: list[tuple[float, int]] = [ + (-2.0, 0), + (-1.0, 1), + (-0.5, 0), + (0.5, 1), + (1.0, 0), + (2.0, 1), + (0.5, 0), + (-0.5, 1), +] + +#: Deliberately **asymmetric** rung sizes (7 calib / 4 train / 3 val / 5 test): with equal +#: arms a swapped fold sense would refuse the right *count* of the *wrong* rows and every +#: count-based assertion would still pass. The calib and test arms are each independently +#: fittable, so the inversion test can actually run both ways round. +_MIXED: list[tuple[str, float, int]] = [ + ("calib", 1.0, 1), + ("calib", 1.0, 0), + ("calib", -1.0, 0), + ("calib", 2.0, 1), + ("calib", -0.5, 0), + ("calib", 0.25, 1), + ("calib", -2.0, 0), + ("train", 3.0, 1), + ("train", -3.0, 0), + ("train", 0.1, 1), + ("train", 0.2, 0), + ("val", 4.0, 1), + ("val", -4.0, 0), + ("val", 0.3, 1), + ("test", 5.0, 1), + ("test", -5.0, 0), + ("test", 0.4, 1), + ("test", 0.4, 0), + ("test", 1.5, 1), +] + +#: The smallest non-separable calib table — the hand-derived ``β* = ln2`` fixture reused +#: wherever a test needs *a* successful fit rather than a specific one. +_TINY: list[tuple[str, float, int]] = [("calib", 1.0, 1), ("calib", 1.0, 0), ("calib", -1.0, 0)] + + +# --------------------------------------------------------------------------- # +# The fit — hand-derived closed forms first, delegation second +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + "a,ratio,expected_numerator", + [ + (1.0, "3s=2", math.log(2.0)), # z=[+a,+a,-a], y=[1,0,0] -> 3σ(βa)=2 -> βa=ln2 + (2.0, "3s=2", math.log(2.0)), # same root, different scale: β* = ln2 / a + ], +) +def test_fit_recovers_the_hand_derived_root_three_point(a, ratio, expected_numerator): + """``dNLL/dβ = a[3σ(βa) − 2]`` ⇒ ``β* = ln2/a``, ``T* = a/ln2`` — derived, not delegated.""" + z, y, rungs = _table([("calib", a, 1), ("calib", a, 0), ("calib", -a, 0)]) + result = R.temperature_scale(z, y, rung=rungs, grad_tol=1e-8) + assert result.beta == pytest.approx(expected_numerator / a, abs=1e-6) + assert result.temperature == pytest.approx(a / expected_numerator, abs=1e-6) + assert result.fitted_on == R.CALIB_RUNG + assert result.n_fitted == 3 + + +def test_fit_recovers_a_second_hand_derived_root_four_point(): + """``z=[+2,+2,+2,−2]``, ``y=[1,1,0,0]`` ⇒ ``4σ(2β)=3`` ⇒ ``β* = ln3/2``. + + A second root at a second scale: a fitter that happened to return ``ln2/a`` for any + input would pass the case above and fail here. + """ + z, y, rungs = _table( + [("calib", 2.0, 1), ("calib", 2.0, 1), ("calib", 2.0, 0), ("calib", -2.0, 0)] + ) + result = R.temperature_scale(z, y, rung=rungs, grad_tol=1e-8) + assert result.beta == pytest.approx(math.log(3.0) / 2.0, abs=1e-6) + assert result.temperature == pytest.approx(2.0 / math.log(3.0), abs=1e-6) + + +def test_fit_agrees_with_an_independent_minimiser_of_the_same_objective(): + """Ternary search on the NLL value re-finds the root the module bisects the derivative for.""" + z, y, rungs = _table([("calib", v, lab) for v, lab in _NONSEP]) + result = R.temperature_scale(z, y, rung=rungs, grad_tol=1e-9) + assert result.beta == pytest.approx(_ternary_min(z, y, 1e-3, 1e3), abs=1e-5) + + +def test_fit_recovers_a_manufactured_miscalibration(): + """Draw ``y ~ Bernoulli(σ(base))``, hand the fitter ``base·sharpen``, expect ``T ≈ sharpen``. + + The only honest end-to-end check of a calibration fitter: manufacture the miscalibration, + then see whether it is undone. Statistical, so the tolerance is loose — but a fitter + returning ``T ≡ 1`` (a no-op) or ``T ≡ 1/sharpen`` (an inverted parameterisation) misses + by far more than the tolerance. + """ + rng = np.random.default_rng(11) + n = 20_000 + sharpen = 2.5 + base = rng.normal(0.0, 1.8, size=n) + y = (rng.random(n) < 1.0 / (1.0 + np.exp(-base))).astype(np.int64) + result = R.temperature_scale(base * sharpen, y, rung=["calib"] * n) + assert result.temperature == pytest.approx(sharpen, rel=0.05) + + +def test_fit_lowers_the_objective_it_minimises(): + z, y, rungs = _table([("calib", v, lab) for v, lab in _NONSEP]) + result = R.temperature_scale(z, y, rung=rungs) + assert _binary_nll(z, y, result.beta) < _binary_nll(z, y, 1.0) + assert result.fit.nll_per_position_final <= result.fit.nll_per_position_initial + + +def test_fit_is_bit_deterministic(): + z, y, rungs = _table([("calib", v, lab) for v, lab in _NONSEP]) + first = R.temperature_scale(z, y, rung=rungs) + second = R.temperature_scale(z, y, rung=rungs) + assert first.temperature == second.temperature + assert first.beta == second.beta + + +def test_the_one_logit_stack_is_the_delegation_and_it_is_exact(): + """Consistency check, explicitly **secondary** to the closed forms above. + + ``softmax([0, βz])[1] = σ(βz)``, so the multi-class fit on the ``[0, z]`` stack *is* the + binary fit. This asserts the adapter builds that stack correctly; it says nothing about + whether the shared fit is right, which is what the hand-derived roots are for. + """ + z, y, rungs = _table([("calib", v, lab) for v, lab in _NONSEP]) + mine = R.temperature_scale(z, y, rung=rungs) + theirs = T.fit_temperature(np.column_stack((np.zeros_like(z), z)), y) + assert mine.temperature == theirs.temperature + assert mine.beta == theirs.beta + + +@pytest.mark.parametrize( + "spec,reason", + [ + ([("calib", 3.0, 1), ("calib", -3.0, 0)], "perfectly separated -> beta -> inf"), + ([("calib", 5.0, 1), ("calib", 6.0, 1), ("calib", -5.0, 0)], "separated, T -> 0"), + ], +) +def test_degenerate_calibration_sets_raise_rather_than_return_a_bracket_endpoint(spec, reason): + """The A11 refusals carry through the adapter unchanged — no T is invented for them.""" + z, y, rungs = _table(spec) + with pytest.raises(T.TemperatureFitError): + R.temperature_scale(z, y, rung=rungs) + + +# --------------------------------------------------------------------------- # +# The calib-only guarantee — identity, not counts; asymmetric, not balanced +# --------------------------------------------------------------------------- # +def test_t_is_fitted_on_the_calib_rows_and_on_no_others(): + """Identity: the mixed-table ``T`` equals the ``T`` from those same rows fitted alone.""" + z, y, rungs = _table(_MIXED) + mixed = R.temperature_scale(z, y, rung=rungs) + + calib_spec = [row for row in _MIXED if row[0] == R.CALIB_RUNG] + z_only, y_only, rungs_only = _table(calib_spec) + alone = R.temperature_scale(z_only, y_only, rung=rungs_only) + + assert mixed.temperature == alone.temperature # exact: same arithmetic, same rows + assert mixed.n_fitted == alone.n_fitted == len(calib_spec) == 7 + assert mixed.n_total == len(_MIXED) == 19 + + +def test_the_filter_is_not_vacuous_the_other_rows_would_move_t(): + """Fitting the whole table gives a *different* ``T`` — so the selection did something.""" + z, y, rungs = _table(_MIXED) + selected = R.temperature_scale(z, y, rung=rungs) + everything = R.temperature_scale(z, y, rung=[R.CALIB_RUNG] * len(rungs)) + assert selected.temperature != everything.temperature + + +def test_swapping_the_calib_and_test_senses_changes_the_fit(): + """The inversion a count-only assertion cannot see. + + Relabel calib↔test and the fit moves — proving the selector reads the rung *sense*, not + merely some 7 rows. With balanced arms this test would pass under the inversion. + """ + z, y, rungs = _table(_MIXED) + straight = R.temperature_scale(z, y, rung=rungs) + swapped_rungs = [ + "test" if r == R.CALIB_RUNG else (R.CALIB_RUNG if r == "test" else r) for r in rungs + ] + swapped = R.temperature_scale(z, y, rung=swapped_rungs) + assert swapped.n_fitted == 5 # the test arm's size, not the calib arm's + assert swapped.temperature != straight.temperature + # ...and it is the *test* rows it fitted, asserted by identity rather than by count. + test_only = [(R.CALIB_RUNG, v, lab) for r, v, lab in _MIXED if r == "test"] + z_t, y_t, rungs_t = _table(test_only) + assert swapped.temperature == R.temperature_scale(z_t, y_t, rung=rungs_t).temperature + + +def test_the_census_counts_every_rung_including_the_empty_ones(): + z, y, rungs = _table(_MIXED) + counts = R.temperature_scale(z, y, rung=rungs).counts() + assert counts == {"calib": 7, "train": 4, "test": 5, "val": 3} + assert set(counts) == set(R.RUNG_VOCABULARY) # a 0-row rung is reported, not absent + + calib_only = [row for row in _MIXED if row[0] == R.CALIB_RUNG] + z2, y2, rungs2 = _table(calib_only) + assert R.temperature_scale(z2, y2, rung=rungs2).counts() == { + "calib": 7, + "train": 0, + "val": 0, + "test": 0, + } + + +def test_an_empty_calib_rung_raises_and_names_what_it_saw(): + """The all-``False`` calib column: filters clean, fits on nothing, must not look fine.""" + spec = [row for row in _MIXED if row[0] != R.CALIB_RUNG] + z, y, rungs = _table(spec) + with pytest.raises(ValueError, match="no rows on the 'calib' rung"): + R.temperature_scale(z, y, rung=rungs) + + # Positive control: the identical rows, with the (non-separable) train arm relabelled + # calib, fit fine — so the refusal above is about emptiness, not about these rows. + rescued = ["calib" if i < 4 else r for i, r in enumerate(rungs)] + assert R.temperature_scale(z, y, rung=rescued).n_fitted == 4 + + +def test_a_single_class_calib_rung_raises(): + z, y, rungs = _table([("calib", 1.0, 1), ("calib", 2.0, 1), ("test", -1.0, 0)]) + with pytest.raises(ValueError, match="single-class"): + R.temperature_scale(z, y, rung=rungs) + + # Positive control: opposite labels on the same rung and it is a fit again. + z2, y2, rungs2 = _table([*_TINY, ("test", -1.0, 0)]) + assert R.temperature_scale(z2, y2, rung=rungs2).n_fitted == 3 + + +@pytest.mark.parametrize("token", ["None", "nan", "calibration", "Calib", "CALIB", "", " "]) +def test_an_unrecognised_rung_token_is_refused_not_filtered(token): + """A token nobody recognises must not evaporate a row and leave the counts looking clean.""" + spec = list(_MIXED) + spec[0] = (token, 1.0, 1) + z, y, rungs = _table(spec) + with pytest.raises(ValueError, match="unrecognised rung label"): + R.temperature_scale(z, y, rung=rungs) + + # Positive control: the same table with that row labelled properly fits. + spec[0] = (R.CALIB_RUNG, 1.0, 1) + z, y, rungs = _table(spec) + assert R.temperature_scale(z, y, rung=rungs).n_fitted == 7 + + +@pytest.mark.parametrize( + "bad", + [ + {"labels": np.array([0, 1, 2])}, + {"logits": np.zeros((3, 2))}, + ], +) +def test_shape_and_domain_violations_raise(bad): + z, y, rungs = _table(_TINY) + kwargs = {"logits": z, "labels": y} + kwargs.update(bad) + with pytest.raises(ValueError): + R.temperature_scale(kwargs["logits"], kwargs["labels"], rung=rungs) + + # Positive control: the untouched arguments succeed. + assert R.temperature_scale(z, y, rung=rungs).n_fitted == 3 + + +def test_length_disagreements_raise(): + z, y, rungs = _table(_TINY) + with pytest.raises(ValueError, match="rung carries"): + R.temperature_scale(z, y, rung=rungs[:2]) + with pytest.raises(ValueError, match="labels carries"): + R.temperature_scale(z, y[:2], rung=rungs) + + +def test_non_finite_logits_are_refused(): + z, y, rungs = _table(_TINY) + z_bad = z.copy() + z_bad[1] = np.inf + with pytest.raises(ValueError, match="non-finite"): + R.temperature_scale(z_bad, y, rung=rungs) + assert R.temperature_scale(z, y, rung=rungs).n_fitted == 3 + + +# --------------------------------------------------------------------------- # +# assign_rung / rung_labels — the P3-02 invariant, re-derived where it matters +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + "calib,fold,expected", + [ + (True, "train", "calib"), + (np.True_, "train", "calib"), + (False, "train", "train"), + (False, "val", "val"), + (False, "test", "test"), + (np.False_, "test", "test"), + ], +) +def test_assign_rung_maps_the_calib_overlay_onto_the_scheme_a_folds(calib, fold, expected): + assert R.assign_rung(calib=calib, fold_random=fold) == expected + + +@pytest.mark.parametrize("fold", ["val", "test"]) +def test_a_calib_row_outside_the_training_fold_raises(fold): + """The P3-02 invariant (``splits._assert_calib_disjoint``), re-derived at the fit site.""" + with pytest.raises(ValueError, match="P3-02 carve"): + R.assign_rung(calib=True, fold_random=fold) + assert R.assign_rung(calib=True, fold_random="train") == "calib" # positive control + + +@pytest.mark.parametrize("missing", [None, float("nan"), np.nan]) +def test_a_missing_calib_flag_raises_rather_than_reading_nan_as_true(missing): + """``NaN`` is **truthy**: ``bool(float('nan')) is True``, asserted here as a literal. + + Under the training env's pandas 3 a null cell arrives as ``NaN``, so a guard written as + ``bool(cell)`` would promote a row with no calibration flag straight into the set ``T`` is + fitted on ([[pandas-3-nan-truthy-in-training-env]]). + """ + assert bool(float("nan")) is True # the bite, stated outright + with pytest.raises(ValueError, match="calib is missing"): + R.assign_rung(calib=missing, fold_random="train") + assert R.assign_rung(calib=True, fold_random="train") == "calib" # positive control + + +@pytest.mark.parametrize("missing", [None, float("nan"), "", " "]) +def test_a_missing_fold_raises(missing): + with pytest.raises(ValueError, match="fold_random is missing"): + R.assign_rung(calib=False, fold_random=missing) + + +@pytest.mark.parametrize("fold", ["Train", "selection_val", "None", "nan", "0"]) +def test_an_unrecognised_fold_raises(fold): + with pytest.raises(ValueError, match="unrecognised fold_random"): + R.assign_rung(calib=False, fold_random=fold) + + +def test_calib_is_never_a_fourth_fold_random_value(): + with pytest.raises(ValueError, match="boolean overlay"): + R.assign_rung(calib=False, fold_random="calib") + + +def test_rung_labels_reports_the_offending_row_index(): + with pytest.raises(ValueError, match=r"row 2: .*unrecognised fold_random"): + R.rung_labels(calib=[False, False, False], fold_random=["train", "val", "Train"]) + assert R.rung_labels(calib=[True, False], fold_random=["train", "test"]) == ["calib", "test"] + + +def test_rung_labels_refuses_ragged_columns(): + with pytest.raises(ValueError, match="fold_random carries"): + R.rung_labels(calib=[True, False], fold_random=["train"]) + + +def test_rung_labels_feeds_temperature_scale_end_to_end(): + """The intended call shape: derive rungs from the two real columns, then fit.""" + calib = [True, True, True, False, False, False] + fold = ["train", "train", "train", "train", "val", "test"] + z = np.array([1.0, 1.0, -1.0, 9.0, -9.0, 9.0]) + y = np.array([1, 0, 0, 1, 0, 1]) + result = R.temperature_scale(z, y, rung=R.rung_labels(calib=calib, fold_random=fold)) + assert result.n_fitted == 3 + assert result.counts() == {"calib": 3, "train": 1, "val": 1, "test": 1} + + +# --------------------------------------------------------------------------- # +# Monotonicity — temperature scaling and the prior shift both preserve the ranking +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("temperature", [0.25, 0.5, 1.0, 1.4426950408889634, 3.7]) +def test_temperature_scaling_is_strictly_monotone_in_the_raw_logit(temperature): + z = np.array([-6.0, -2.3, -0.4, 0.0, 0.15, 1.9, 4.4, 11.0]) + payload = R.calibrated_posterior(z, temperature=temperature) + p = payload[R.NAMED_POSTERIOR_KEY] + assert np.all(np.diff(p) > 0.0) + assert list(np.argsort(p)) == list(np.argsort(z)) + + +def test_the_prior_shift_moves_every_posterior_but_reorders_none(): + z = np.array([-6.0, -2.3, -0.4, 0.0, 0.15, 1.9, 4.4]) + payload = R.calibrated_posterior( + z, temperature=1.3, source_prior=0.8, target_prior=R.prior_from_odds_ratio(1e3) + ) + named = payload[R.NAMED_POSTERIOR_KEY] + shifted = payload[R.PRIOR_SHIFTED_POSTERIOR_KEY] + assert np.all(shifted < named) # the target prior is far lower, so every posterior drops + assert list(np.argsort(shifted)) == list(np.argsort(named)) == list(np.argsort(z)) + + +@pytest.mark.parametrize( + "bad", [0.0, -0.0, -0.1, float("nan"), float("inf"), float("-inf"), True, "1.0", None] +) +def test_a_degenerate_temperature_is_refused(bad): + """``T`` must be finite and strictly positive — ``T = 0`` divides, ``T < 0`` inverts the + ranking, and ``bool`` would read ``True`` as ``T = 1``.""" + z = np.array([-1.0, 0.0, 1.0]) + with pytest.raises((T.TemperatureFitError, ValueError)): + R.calibrated_posterior(z, temperature=bad) + assert R.calibrated_posterior(z, temperature=1.0)["n_rows"] == 3 # positive control + + +def test_the_posterior_does_not_overflow_in_either_tail(): + z = np.array([-800.0, -40.0, 0.0, 40.0, 800.0]) + with np.errstate(over="raise", invalid="raise"): + p = R.posterior_from_logits(z) + assert np.isfinite(p).all() + assert p[0] == 0.0 + assert p[-1] == 1.0 + assert p[2] == pytest.approx(0.5, abs=1e-15) + + +# --------------------------------------------------------------------------- # +# The prior shift — closed form, equivalence, invertibility +# --------------------------------------------------------------------------- # +def test_log_odds_shift_matches_the_hand_computed_value(): + """``π_s=0.5``, ``π_t=0.001`` ⇒ ``Δ = ln(0.001/0.999) − 0 = −ln(999)``.""" + shift = R.log_odds_shift(source_prior=0.5, target_prior=0.001) + assert shift == pytest.approx(-math.log(999.0), rel=0, abs=1e-12) + assert shift == pytest.approx(-6.906754778648554, rel=0, abs=1e-12) + + +@pytest.mark.parametrize("r_source,r_target", [(1.0, 1e3), (10.0, 1e4), (100.0, 1e3), (1e4, 10.0)]) +def test_log_odds_shift_of_two_odds_ratios_is_exactly_log_of_their_quotient(r_source, r_target): + """``prior_from_odds_ratio(r)`` has odds exactly ``1/r``, so ``Δ = ln(r_s) − ln(r_t)``.""" + shift = R.log_odds_shift( + source_prior=R.prior_from_odds_ratio(r_source), + target_prior=R.prior_from_odds_ratio(r_target), + ) + assert shift == pytest.approx(math.log(r_source / r_target), rel=0, abs=1e-12) + + +def test_prior_shift_equals_the_posterior_form_odds_ratio_correction(): + """``p' = p·r/(p·r + 1 − p)`` with ``r = odds_t/odds_s`` — the same rule, written the + other way, computed here with plain floats rather than the module's arithmetic.""" + z = np.array([-3.0, -0.7, 0.0, 0.9, 2.5]) + p_s, p_t = 0.75, 0.002 + r = (p_t / (1.0 - p_t)) / (p_s / (1.0 - p_s)) + expected = [(_sigmoid(v) * r) / (_sigmoid(v) * r + 1.0 - _sigmoid(v)) for v in z.tolist()] + got = R.posterior_from_logits(R.prior_shift(z, source_prior=p_s, target_prior=p_t)) + np.testing.assert_allclose(got, expected, rtol=1e-12, atol=0.0) + + +def test_prior_shift_is_exactly_invertible_and_the_identity_shift_is_a_no_op(): + z = np.array([-4.0, -0.25, 0.0, 1.1, 7.3]) + there = R.prior_shift(z, source_prior=0.8, target_prior=1e-4) + back = R.prior_shift(there, source_prior=1e-4, target_prior=0.8) + np.testing.assert_allclose(back, z, rtol=0.0, atol=1e-12) + np.testing.assert_allclose( + R.prior_shift(z, source_prior=0.3, target_prior=0.3), z, rtol=0.0, atol=0.0 + ) + + +@pytest.mark.parametrize( + "kwargs", + [ + {"source_prior": 0.0, "target_prior": 0.5}, + {"source_prior": 1.0, "target_prior": 0.5}, + {"source_prior": 0.5, "target_prior": 0.0}, + {"source_prior": 0.5, "target_prior": 1.0}, + {"source_prior": -0.1, "target_prior": 0.5}, + {"source_prior": 0.5, "target_prior": 1.5}, + {"source_prior": float("nan"), "target_prior": 0.5}, + {"source_prior": 0.5, "target_prior": float("inf")}, + {"source_prior": True, "target_prior": 0.5}, + {"source_prior": "0.5", "target_prior": 0.5}, + {"source_prior": None, "target_prior": 0.5}, + ], +) +def test_a_degenerate_prior_is_refused(kwargs): + z = np.array([-1.0, 0.0, 1.0]) + with pytest.raises(ValueError): + R.prior_shift(z, **kwargs) + # Positive control: identical call with both priors legal. + assert R.prior_shift(z, source_prior=0.5, target_prior=0.25).shape == (3,) + + +# --------------------------------------------------------------------------- # +# Priors, odds ratios and the PRD band +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("ratio", [1e-3, 1.0, 10.0, 100.0, 1e3, 1e4, 1e6]) +def test_prior_and_odds_ratio_round_trip(ratio): + assert R.odds_ratio_from_prior(R.prior_from_odds_ratio(ratio)) == pytest.approx( + ratio, rel=1e-12 + ) + + +def test_prior_from_odds_ratio_is_the_stated_formula(): + assert R.prior_from_odds_ratio(1e3) == pytest.approx(1.0 / 1001.0, rel=0, abs=1e-15) + assert R.prior_from_odds_ratio(1e4) == pytest.approx(1.0 / 10001.0, rel=0, abs=1e-15) + + +@pytest.mark.parametrize("bad", [0.0, -1.0, float("nan"), float("inf"), True, "10", None]) +def test_a_degenerate_odds_ratio_is_refused(bad): + with pytest.raises(ValueError): + R.prior_from_odds_ratio(bad) + assert R.prior_from_odds_ratio(10.0) == pytest.approx(1.0 / 11.0) + + +def test_the_prd_band_is_the_quoted_range_and_it_discriminates(): + """Both endpoints of PRD.md:188's ``~10³–10⁴:1`` are inside; the neighbours are not. + + ADR-0005 D7's pinned ``100:1`` benchmark decoy prevalence and §9.1's ``~10:1`` training + seed ratio must both fall **outside** — a band that admitted them would certify nothing, + since it is precisely the distinction ADR-0005:80 draws. + """ + assert R.DEPLOYMENT_PRIOR_ODDS_RANGE == (1e3, 1e4) + assert R.DEPLOYMENT_PRIOR_RANGE == (1.0 / 10001.0, 1.0 / 1001.0) + assert R.deployment_prior_in_prd_band(R.prior_from_odds_ratio(1e3)) is True + assert R.deployment_prior_in_prd_band(R.prior_from_odds_ratio(1e4)) is True + assert R.deployment_prior_in_prd_band(R.prior_from_odds_ratio(3e3)) is True + assert R.deployment_prior_in_prd_band(R.prior_from_odds_ratio(100.0)) is False # D7 + assert R.deployment_prior_in_prd_band(R.prior_from_odds_ratio(10.0)) is False # §9.1 + assert R.deployment_prior_in_prd_band(R.prior_from_odds_ratio(1e5)) is False + + +# --------------------------------------------------------------------------- # +# The producer — both posteriors, the gated one named, no invented prior +# --------------------------------------------------------------------------- # +def test_the_named_posterior_is_the_pre_prior_shift_object_and_is_the_gated_one(): + """D11: GATE-2 grades the temperature-scaled posterior **before** the prior shift.""" + z = np.array([-2.0, 0.0, 1.5]) + temperature = 2.0 + payload = R.calibrated_posterior( + z, temperature=temperature, source_prior=0.5, target_prior=1e-3 + ) + assert payload["gated_posterior_key"] == R.NAMED_POSTERIOR_KEY + hand = [_sigmoid(v / temperature) for v in z.tolist()] + np.testing.assert_allclose(payload[R.NAMED_POSTERIOR_KEY], hand, rtol=1e-12, atol=0.0) + # ...and the gated object is *not* the shifted one, which is the confusion D11 exists for. + assert not np.allclose( + payload[payload["gated_posterior_key"]], payload[R.PRIOR_SHIFTED_POSTERIOR_KEY] + ) + + +def test_without_priors_the_stack_stops_at_the_named_posterior(): + z = np.array([-2.0, 0.0, 1.5]) + payload = R.calibrated_posterior(z, temperature=1.7) + assert payload[R.PRIOR_SHIFTED_POSTERIOR_KEY] is None + assert payload["prior_shift_applied"] is False + assert payload["log_odds_shift"] is None + assert payload["target_prior_in_prd_band"] is None + assert payload["stack_applied"] == ["train", "temperature_scale"] + assert payload["stack_order"] == list(T.STACK_ORDER) + + +def test_with_both_priors_the_full_pinned_stack_is_recorded_as_applied(): + z = np.array([-2.0, 0.0, 1.5]) + payload = R.calibrated_posterior( + z, temperature=1.1, source_prior=0.75, target_prior=R.prior_from_odds_ratio(1e4) + ) + assert payload["prior_shift_applied"] is True + assert payload["stack_applied"] == list(T.STACK_ORDER) + assert payload["log_odds_shift"] == pytest.approx( + R.log_odds_shift(source_prior=0.75, target_prior=R.prior_from_odds_ratio(1e4)) + ) + assert payload["target_prior_in_prd_band"] is True + + +@pytest.mark.parametrize("kwargs", [{"source_prior": 0.5}, {"target_prior": 0.001}]) +def test_a_half_specified_prior_shift_raises_rather_than_defaulting_one(kwargs): + """The module must be unable to invent a deployment prior (§10.3). + + The PRD gives the genome-scale prevalence only as prose (``~10³–10⁴:1``); a default here + would put a fabricated quantity into every downstream posterior while looking like a + convenience. + """ + z = np.array([-1.0, 0.0, 1.0]) + with pytest.raises(ValueError, match="needs both source_prior and target_prior"): + R.calibrated_posterior(z, temperature=1.0, **kwargs) + # Positive control: supplying both is fine, supplying neither is fine. + assert R.calibrated_posterior(z, temperature=1.0)["prior_shift_applied"] is False + assert ( + R.calibrated_posterior(z, temperature=1.0, source_prior=0.5, target_prior=0.001)[ + "prior_shift_applied" + ] + is True + ) + + +def test_neither_prior_has_a_default(): + """Structural, not behavioural: ``prior_shift`` cannot be called without both priors. + + If someone later gives either one a default, this fails — which is the point: the + deployment prior is the value that must never arrive by omission. + """ + z = np.array([-1.0, 0.0, 1.0]) + with pytest.raises(TypeError): + R.prior_shift(z) + with pytest.raises(TypeError): + R.prior_shift(z, source_prior=0.5) + with pytest.raises(TypeError): + R.log_odds_shift(source_prior=0.5) + + +def test_the_stack_order_is_the_pinned_one_and_is_not_a_second_copy(): + """One ``STACK_ORDER``, imported — a second literal would drift from ADR-0005 D11.""" + assert R.STACK_ORDER is T.STACK_ORDER + assert R.STACK_ORDER == ("train", "temperature_scale", "prior_shift") From a721d063002f2c97e7c136b262a03cf8d2d1d755 Mon Sep 17 00:00:00 2001 From: bioedca Date: Mon, 3 Aug 2026 16:42:59 -0500 Subject: [PATCH 2/3] =?UTF-8?q?fix(calib):=20P3-07=20review=20r1=20?= =?UTF-8?q?=E2=80=94=20two=20fail-open=20guards,=20and=20promote=20the=20b?= =?UTF-8?q?oolean=20parse=20instead=20of=20forking=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit r1 on 88879fd (1 actionable + 1 nitpick) plus a 44-agent adversarial pass (20 findings, 10 surviving two independent refuters), deduplicated to six fixes. Both sources independently found the same defect at the int64 cast. 1. The "labels must be binary 0/1" guard was VACUOUS: it ran after astype(np.int64), which truncates toward zero. Demonstrated — temperature_scale(z=[1,1,-1], y=[1.0,0.4,0.6]) SUCCEEDS and returns T bit-identical to the fit for [1,0,0]; a target written as 0.6 was silently scored a negative. A NaN label cast to the int64 sentinel and surfaced as "range [-9223372036854775808, 1]". The domain check now happens in the original dtype: bool/int pass, floats must be finite and integral, any other dtype is refused by name. 2. bool(calib) closed the NaN hole and left the string hole open. The guard exists BECAUSE pandas 3 delivers a null as NaN and bool(nan) is True; one step out, bool("False") and bool("0") are also True, so a string-typed calib column would promote its explicit negatives into the set T is fitted on with every count agreeing. Reproduced: rung_labels(calib=["True","False",...]) returned ["calib"]*6. The fix was already in the repo. stage2/train.py::_bool_or_none is exactly this parse and is documented as also reading `calib`, but cannot be imported here (that module is torch-adjacent; this one is deliberately numpy-only). So it is PROMOTED DOWN to masking.py, beside the is_missing/row_text it completes, with NULL_TOKENS moving with it. stage2.heads.NULL_TOKENS is now a re-export and stage2.train._bool_or_none a delegation: one vocabulary, one parse, three call sites. A fourth copy would have been the cheaper diff and the wrong one. 3. The single-class guard is a disjunct (n_pos == 0 or n_pos == n_fitted) and only the all-positive half was tested; with the other half deleted, an all-negative calib carve returns a cleanly converged, fully certified T. Both halves now parametrized. 4. test_shape_and_domain_violations_raise used a bare pytest.raises(ValueError) — both of this module's guards could be deleted and it stayed green, because the delegated fitter raises its own error further in. Now matches this module's own text. 5. The empty-calib refusal asserted only its first clause, leaving the census and the GRADED_RUNGS count (the only place that constant is read) unchecked. 6. The payload's source_prior/target_prior were write-only: swapping the two assignments left the suite green while recording the deployment prior as the calib prevalence. Also carries step/generated_by/adr now (CodeRabbit nitpick: three constants nothing read). Citation: "Elkan 2001, Theorem 1" -> Elkan 2001 with no theorem number. A reviewer put the base-rate result at Theorem 2; the numbering could not be verified from here, and a pointer to the wrong theorem is worse than one to the paper. Validation: test_recalibrate 138 pass (was 108); test_temperature 81; masking + heads + stage2_train_smoke 411 pass together (the promotion is behaviour-preserving on both sides). Full unit+golden+ml: the worktree's 14 failures are a strict SUBSET of the 31 untouched `main` produces in this local runner (py3.11/numpy 2.4.2) — zero regressions. ruff 0.15.15 + black 25.11.0 clean. 24 source sabotages (up from 15), each RED against its NAMED test, each restored byte-identically. --- analyses/phase3_log.qmd | 73 +++++++++++ src/tbox_finder/calib/recalibrate.py | 51 ++++++-- src/tbox_finder/masking.py | 47 +++++++ src/tbox_finder/stage2/heads.py | 6 +- src/tbox_finder/stage2/train.py | 28 ++--- tests/unit/test_recalibrate.py | 176 +++++++++++++++++++++++++-- 6 files changed, 341 insertions(+), 40 deletions(-) diff --git a/analyses/phase3_log.qmd b/analyses/phase3_log.qmd index 583a40d3..d0886ea9 100644 --- a/analyses/phase3_log.qmd +++ b/analyses/phase3_log.qmd @@ -1715,3 +1715,76 @@ by the overflowing one, single-class guard removed, priors of exactly 0/1 admitt posterior, both of which this implements verbatim, and the one value that would have needed a pin is the one the module refuses to supply. **Commit.** Feature branch `p3-07-calibration-stack`; squash SHA derived at the phase-exit gate. + +## Step P3-07 (review r1) — 2026-08-03: two fail-open guards, and a boolean parse that was already written + +**Two review sources, deduplicated to six fixes.** CodeRabbit r1 on `88879fd` returned 1 +actionable + 1 nitpick; a 44-agent adversarial pass over the same diff returned 20 findings, 10 +surviving two independent refuters. The overlap is the interesting part: **both** independently +found the same defect at the `int64` cast, from opposite directions. + +**Fix 1 — the "labels must be binary 0/1" guard was vacuous.** It ran *after* +`y.astype(np.int64)`, which truncates toward zero, so the check saw only what the cast had +already produced. Demonstrated, not argued: `temperature_scale(z=[1,1,−1], y=[1.0, 0.4, 0.6])` +**succeeds** and returns `T = 1.4427030259199125`, bit-identical to the fit for `[1,0,0]` — a +target the caller wrote as `0.6` was silently scored a negative. A `NaN` label cast to the +int64 sentinel and surfaced as `range [-9223372036854775808, 1]`, a message pointing at the +wrong problem. The domain check now happens in the original dtype: booleans and integers pass, +floats must be finite and integral, any other dtype is refused by name. + +**Fix 2 — `bool(calib)` closed the NaN hole and left the string hole open.** The guard existed +*because* pandas 3 delivers a null as `NaN` and `bool(nan) is True`; one step further out, +`bool("False")` and `bool("0")` are also `True`. A string-typed `calib` column would therefore +promote its explicit **negatives** into the set `T` is fitted on, with every count agreeing. +Reproduced end to end: `rung_labels(calib=["True","False",…], …)` returned `["calib"]*6`. + +**And the fix for it was already in the repo, which is the actual lesson.** +`stage2/train.py::_bool_or_none` is exactly this parse — tri-state, `masking.is_missing`-based, +`{"true","1"}/{"false","0"}`, raising on anything else — written for the fold columns and +documented as also reading `calib`. It could not be imported here (that module is +torch-adjacent; this one is deliberately numpy-only), so the parse was **promoted down** to +`masking.py`, beside the `is_missing`/`row_text` it completes, with `NULL_TOKENS` moving with +it. `stage2.heads.NULL_TOKENS` is now a re-export and `stage2.train._bool_or_none` a +delegation: one vocabulary, one parse, three call sites. Writing a fourth copy here would have +been the cheaper diff and the wrong one +([[promote-dont-duplicate-is-a-correctness-rule]]). + +**Fixes 3–6, all test-side, all found by sabotage rather than by reading.** (3) The +single-class guard is a **disjunct**, `n_pos == 0 or n_pos == n_fitted`, and only the +all-positive half was tested — with the other half deleted, an all-negative calib carve (the +realistic P3-02 join failure) returns a **cleanly converged, fully certified** `T`. Both halves +are now parametrized. (4) `test_shape_and_domain_violations_raise` used a bare +`pytest.raises(ValueError)`, and both of this module's own guards could be deleted with the +test still green, because the delegated fitter raises its own differently-worded error further +in; it now matches on this module's text. (5) The empty-calib refusal asserted only its first +clause, leaving the census and the `GRADED_RUNGS` count — the only place that constant is read +— unchecked; all three are asserted. (6) The payload's `source_prior`/`target_prior` were +written and never read, so **swapping the two assignments left the suite green** while emitting +an artifact that recorded the deployment prior as the calib prevalence. + +**One finding fixed by deletion.** The docstring cited the known-prior closed form as "Elkan +2001, Theorem 1"; a reviewer put it at Theorem 2 (Theorem 1 being the cost-threshold result). +The numbering could not be verified against the paper from here, so the theorem number is +**gone** rather than guessed — a citation pointing at the wrong theorem is worse than one +pointing at the paper, and the DOI'd Saerens reference is the traceable half anyway. + +**Not fixed, with reasons.** Ten findings were refuted by their own verifiers and are recorded +rather than acted on — among them "the calib certificate is self-referential and vacuous" +(refuted: the selection is by identity against `CALIB_RUNG`, and the inversion test asserts the +swapped fit equals the test arm's own `T`), "`RUNG_VOCABULARY` is not closed because `.strip()` +admits padded tokens" (`" calib "` is a legitimate parquet spelling of `calib`; `" "` is +refused), and a naming divergence between this module's flat `stack_order` and +`temperature.py`'s nested one (the two artifacts have no coupling path and different +validators). + +**Validation.** `test_recalibrate` **138 pass** (was 108); `test_temperature` 81, +`test_masking` + `test_stage2_heads` + `test_stage2_train_smoke` 411 pass together — the +promotion is behaviour-preserving on both sides. Full `unit+golden+ml` tier: the worktree's 14 +failures are a **strict subset** of the 31 that untouched `main` produces in this local runner +(py3.11 / numpy 2.4.2, the only local env with pytest), i.e. zero regressions; CI runs the +pinned stack. ruff 0.15.15 + black 25.11.0 clean. **24 source sabotages, each RED against its +NAMED test, each restored byte-identically** (up from 15; the nine new ones cover the strict +boolean parse, all three label-domain branches, both halves of the single-class disjunct, +`GRADED_RUNGS`, the payload prior swap, the payload provenance, and the 1-D logits guard). + +**Commit.** Feature branch `p3-07-calibration-stack`; squash SHA derived at the phase-exit gate. diff --git a/src/tbox_finder/calib/recalibrate.py b/src/tbox_finder/calib/recalibrate.py index ed7e3596..5ba7c2bb 100644 --- a/src/tbox_finder/calib/recalibrate.py +++ b/src/tbox_finder/calib/recalibrate.py @@ -275,21 +275,25 @@ def assign_rung(*, calib: Any, fold_random: Any) -> str: in one place. Fail-closed in three directions, each of which is a real failure mode rather than a hypothetical: - * a **missing** ``calib`` or ``fold_random`` raises — tested with - :func:`tbox_finder.masking.is_missing`, because under pandas 3 a null string cell - arrives as ``NaN``, which is *truthy*, so ``bool(value)`` would read a missing calib - flag as ``True`` and a missing fold as the string ``"nan"``; + * a **missing** ``calib`` or ``fold_random`` raises, and a ``calib`` cell that is neither + missing nor unambiguously boolean raises too — both read through + :func:`tbox_finder.masking.bool_or_none`, the same tri-state parse ``stage2.train`` + uses for the fold columns. ``bool(value)`` is wrong twice over here: under pandas 3 a + null cell arrives as ``NaN``, which is *truthy*, so a row with **no** flag would be + promoted into the fit; and one step further out ``bool("False")`` and ``bool("0")`` are + *also* ``True``, so a string-typed column would promote its explicit negatives. Both + failures are silent and both look exactly like a working filter; * a ``calib`` row outside the ``train`` rung raises — that is the P3-02 invariant (``splits._assert_calib_disjoint``), re-derived here at the point where violating it would corrupt a gate rather than trusted from upstream; * an unrecognised ``fold_random`` raises, instead of being bucketed as "other". """ - if masking.is_missing(calib): + in_calib = masking.bool_or_none(calib, field="calib") + if in_calib is None: raise ValueError("calib is missing — a row with no calibration flag has no rung") fold = masking.row_text(fold_random).strip() if not fold: raise ValueError("fold_random is missing — a row with no scheme-A fold has no rung") - in_calib = bool(calib) if in_calib: if fold != "train": raise ValueError( @@ -368,6 +372,25 @@ def temperature_scale( raise ValueError(f"logits carries {z.shape[0]} rows but labels carries {y.shape[0]}") if len(rung) != z.shape[0]: raise ValueError(f"logits carries {z.shape[0]} rows but rung carries {len(rung)}") + # Domain-check in the ORIGINAL dtype, before the cast. `astype(np.int64)` truncates + # toward zero, so a range test placed after it is vacuous in both directions: a label of + # 0.6 becomes 0 and a label of 1.7 becomes 1, and the fit then runs — successfully, and + # bit-identically to the truncated targets — on values the caller never wrote. A NaN + # label casts to the int64 sentinel and surfaces as a misleading + # "range [-9223372036854775808, 1]" rather than as the missing value it is. + if y.size: + if np.issubdtype(y.dtype, np.bool_) or np.issubdtype(y.dtype, np.integer): + pass + elif np.issubdtype(y.dtype, np.floating): + if not np.isfinite(y).all(): + raise ValueError("labels carries a non-finite value") + if not np.array_equal(y, np.rint(y)): + raise ValueError( + "labels must be integral 0/1 — a fractional target would truncate " + "silently and move the fit" + ) + else: + raise ValueError(f"labels must be a boolean or numeric 0/1 array, got dtype {y.dtype}") y = y.astype(np.int64, copy=False) if y.size and (int(y.min()) < 0 or int(y.max()) > 1): raise ValueError( @@ -432,10 +455,12 @@ def prior_shift(logits: Any, *, source_prior: float, target_prior: float) -> np. The Saerens/Elkan correction in the log-odds form ADR-0005 D11 names [Saerens, Latinne & Decaestecker, *Neural Computation* 14(1):21–41, DOI:10.1162/089976602753284446 (accessed - 2026-08-03); Elkan, *The foundations of cost-sensitive learning*, IJCAI 2001, Theorem 1]: - under a pure prior shift the class-conditional densities are unchanged, so the likelihood - ratio is unchanged and the whole correction is the additive constant - ``logit(π_t) − logit(π_s)``:: + 2026-08-03); Elkan, *The foundations of cost-sensitive learning*, IJCAI 2001 — cited + without a theorem number, because the base-rate result's numbering could not be verified + against the paper here and a pointer to the wrong theorem is worse than none; the DOI'd + Saerens reference is the traceable one]: under a pure prior shift the class-conditional + densities are unchanged, so the likelihood ratio is unchanged and the whole correction is + the additive constant ``logit(π_t) − logit(π_s)``:: z' = z + log(π_t/(1−π_t)) − log(π_s/(1−π_s)) @@ -496,6 +521,12 @@ def calibrated_posterior( named = posterior_from_logits(tempered) payload: dict[str, Any] = { + # Provenance travels *in* the payload, so a report serialised from it says which step + # and which pinned decision produced the numbers beside it rather than leaving that to + # whoever writes the report ([[pinned-constant-that-nothing-reads]]). + "step": STEP, + "generated_by": GENERATED_BY, + "adr": ADR, "stack_order": list(STACK_ORDER), "temperature": float(temperature), NAMED_POSTERIOR_KEY: named, diff --git a/src/tbox_finder/masking.py b/src/tbox_finder/masking.py index 745de6b3..dfb61dbd 100644 --- a/src/tbox_finder/masking.py +++ b/src/tbox_finder/masking.py @@ -288,6 +288,53 @@ def row_text(value: Any) -> str: return "" if is_missing(value) else str(value) +#: Spellings a *stringified* null arrives as. :func:`is_missing` catches the real sentinels +#: (``None`` / NaN / pandas-NA); these are what is left after something upstream has already +#: rendered one into text. Lives here, beside the missing test it completes, so the several +#: modules that need it share **one** null vocabulary — ``stage2.heads.NULL_TOKENS`` is a +#: re-export of this name, not a second copy. +NULL_TOKENS: frozenset[str] = frozenset({"none", "nan", "na", "", "null", "n/a"}) + + +def bool_or_none(value: Any, *, field: str = "value") -> bool | None: + """A tri-state boolean read of a parquet cell, with pandas' several spellings of missing. + + Returns ``True``/``False`` for an unambiguous boolean, ``None`` when the cell carries no + value at all (a real sentinel, an empty cell, or a :data:`NULL_TOKENS` string), and + **raises** on anything else — so an unrecognised spelling routes to a caller's fail-closed + branch or to an error, never to a silent ``True``. + + ``field`` names the column in the failure message, because a schema migration putting + ``"yes"`` in one column should not raise an error blaming a different one. + + ``is_missing`` rather than ``value or None``: under the ``ml-rna`` env's pandas the string + form of a missing cell is ``"nan"``, which is **truthy**, and that exact difference once + deleted 60 % of a training mix while every clause stayed green + ([[pandas-3-nan-truthy-in-training-env]]). The bare ``bool(value)`` this replaces has the + same shape one step further out: ``bool("False")`` and ``bool("0")`` are both ``True``. + + Promoted here from ``stage2.train`` (P3-07) so the fold readers and the calibration-rung + reader share one parse: a second copy means fixing one and shipping the bug in the other + ([[promote-dont-duplicate-is-a-correctness-rule]]). ``stage2.train._bool_or_none`` + delegates to this function. + """ + if is_missing(value): + return None + if isinstance(value, bool): + return value + # numpy.bool_ / 0 / 1 / "True" all arrive here from parquet round-trips. + text = row_text(value).strip().lower() + if text in {"true", "1"}: + return True + if text in {"false", "0"}: + return False + if not text or text in NULL_TOKENS: + # An empty cell, or one whose STRING form is a null token, carries no assignment — so + # it is missing, and routes to the caller's fail-closed refusal rather than raising. + return None + raise ValueError(f"{field}={value!r} is neither missing nor boolean") + + # --------------------------------------------------------------------------- # # The ADR-0006 D11 / ADR-0005 D14 mining spare-rule (pure predicate) # --------------------------------------------------------------------------- # diff --git a/src/tbox_finder/stage2/heads.py b/src/tbox_finder/stage2/heads.py index a5a60034..62b1b1ba 100644 --- a/src/tbox_finder/stage2/heads.py +++ b/src/tbox_finder/stage2/heads.py @@ -139,7 +139,11 @@ #: Spellings a *stringified* null arrives as. :func:`tbox_finder.masking.is_missing` #: catches the real sentinels (``None`` / NaN / pandas-NA); these are what is left after #: something upstream has already rendered one into text. -NULL_TOKENS: frozenset[str] = frozenset({"none", "nan", "na", "", "null", "n/a"}) +#: +#: **Re-exported, not redefined** (P3-07): the vocabulary lives in +#: :data:`tbox_finder.masking.NULL_TOKENS`, beside the missing test it completes, so the +#: label readers here and the boolean readers in ``masking.bool_or_none`` cannot drift apart. +NULL_TOKENS: frozenset[str] = masking.NULL_TOKENS def _clean(value: Any) -> str: diff --git a/src/tbox_finder/stage2/train.py b/src/tbox_finder/stage2/train.py index 9e1bf29d..bce4041a 100644 --- a/src/tbox_finder/stage2/train.py +++ b/src/tbox_finder/stage2/train.py @@ -193,27 +193,15 @@ def _bool_or_none(value: Any, *, field: str = "value") -> bool | None: pandas the string form of a missing cell is ``"nan"``, which is truthy, and that exact difference once deleted 60% of a training mix while every clause stayed green ([[pandas-3-nan-truthy-in-training-env]]). + + **Promoted to** :func:`tbox_finder.masking.bool_or_none` **at P3-07** and delegated to + here, unchanged in behaviour: the calibration-rung reader needs the identical parse and + cannot import this module (it is torch-adjacent), so the parse moved down to the + stdlib-only layer rather than being copied + ([[promote-dont-duplicate-is-a-correctness-rule]]). This name stays as the in-module + spelling its call sites already use. """ - if masking.is_missing(value): - return None - if isinstance(value, bool): - return value - # numpy.bool_ / 0 / 1 / "True" all arrive here from parquet round-trips. - text = masking.row_text(value).strip().lower() - if text in {"true", "1"}: - return True - if text in {"false", "0"}: - return False - if not text or text in H.NULL_TOKENS: - # An empty cell, or one whose STRING form is a null token, carries no fold - # assignment — so it is missing, and routes to the fail-closed refusal rather than - # raising. The literal `"nan"` matters here and is not hypothetical: it is what a - # missing cell stringifies to under the ml-rna env's pandas, which is exactly how a - # missingness check that only knew about `None` once deleted 60% of a training mix - # ([[pandas-3-nan-truthy-in-training-env]]). `H.NULL_TOKENS` is reused rather than - # respelled so the two null vocabularies cannot drift apart. - return None - raise ValueError(f"{field}={value!r} is neither missing nor boolean") + return masking.bool_or_none(value, field=field) def row_eligibility(row: Mapping[str, Any], *, rung: str, admit_parentless_decoys: bool) -> str: diff --git a/tests/unit/test_recalibrate.py b/tests/unit/test_recalibrate.py index e49002db..a111221f 100644 --- a/tests/unit/test_recalibrate.py +++ b/tests/unit/test_recalibrate.py @@ -300,11 +300,21 @@ def test_the_census_counts_every_rung_including_the_empty_ones(): def test_an_empty_calib_rung_raises_and_names_what_it_saw(): - """The all-``False`` calib column: filters clean, fits on nothing, must not look fine.""" + """The all-``False`` calib column: filters clean, fits on nothing, must not look fine. + + The message is asserted in three parts, not one: the phrase, the **census** (so a + silently-wrong census cannot hide behind a correct headline), and the count of rows on + ``GRADED_RUNGS`` — which is the only place that constant is read, so asserting only the + first clause would leave it a pinned constant nothing checks. + """ spec = [row for row in _MIXED if row[0] != R.CALIB_RUNG] z, y, rungs = _table(spec) - with pytest.raises(ValueError, match="no rows on the 'calib' rung"): + with pytest.raises(ValueError) as excinfo: R.temperature_scale(z, y, rung=rungs) + message = str(excinfo.value) + assert "no rows on the 'calib' rung out of 12" in message + assert "{'calib': 0, 'test': 5, 'train': 4, 'val': 3}" in message + assert "the 8 rows of the graded rungs ['val', 'test']" in message # 5 test + 3 val # Positive control: the identical rows, with the (non-separable) train arm relabelled # calib, fit fine — so the refusal above is about emptiness, not about these rows. @@ -312,8 +322,23 @@ def test_an_empty_calib_rung_raises_and_names_what_it_saw(): assert R.temperature_scale(z, y, rung=rescued).n_fitted == 4 -def test_a_single_class_calib_rung_raises(): - z, y, rungs = _table([("calib", 1.0, 1), ("calib", 2.0, 1), ("test", -1.0, 0)]) +@pytest.mark.parametrize( + "labels,described", + [ + ([1, 1], "all-positive"), + ([0, 0], "all-negative"), + ], +) +def test_a_single_class_calib_rung_raises(labels, described): + """**Both** halves of the compound guard, sabotaged separately. + + ``n_pos == 0 or n_pos == n_fitted`` is a disjunct, and a test that exercises only the + all-positive half leaves the other free to be deleted + ([[sabotage-attribution-names-the-test]]). The all-negative half is the realistic one: a + P3-02 join that drops the positive class yields a calib carve of pure negatives, on which + the fit **converges cleanly** and returns a fully certified `T` if the guard is gone. + """ + z, y, rungs = _table([("calib", -1.0, labels[0]), ("calib", 2.0, labels[1])]) with pytest.raises(ValueError, match="single-class"): R.temperature_scale(z, y, rung=rungs) @@ -338,23 +363,68 @@ def test_an_unrecognised_rung_token_is_refused_not_filtered(token): @pytest.mark.parametrize( - "bad", + "bad,expected", [ - {"labels": np.array([0, 1, 2])}, - {"logits": np.zeros((3, 2))}, + ({"labels": np.array([0, 1, 2])}, "labels must be binary 0/1"), + ({"logits": np.zeros((3, 2))}, "logits must be 1-D"), + ({"labels": np.array([[0], [1], [0]])}, "labels must be 1-D"), ], ) -def test_shape_and_domain_violations_raise(bad): +def test_shape_and_domain_violations_raise(bad, expected): + """``match=`` on **this module's own** message, not a bare ``ValueError``. + + A bare `pytest.raises(ValueError)` passes here even with both of these guards deleted, + because the delegated multi-class fitter raises its own (differently-worded, and for the + 2-D case *wrongly-scoped*) error further in. Matching the text is what makes the guard + this module owns the thing under test. + """ z, y, rungs = _table(_TINY) kwargs = {"logits": z, "labels": y} kwargs.update(bad) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=expected): R.temperature_scale(kwargs["logits"], kwargs["labels"], rung=rungs) # Positive control: the untouched arguments succeed. assert R.temperature_scale(z, y, rung=rungs).n_fitted == 3 +@pytest.mark.parametrize( + "labels,expected", + [ + (np.array([1.0, 0.6, 0.0]), "must be integral"), + (np.array([1.0, 1.7, 0.0]), "must be integral"), + (np.array([1.0, -0.9, 0.0]), "must be integral"), + (np.array([1.0, np.nan, 0.0]), "non-finite"), + (np.array([1.0, np.inf, 0.0]), "non-finite"), + (np.array(["1", "0", "1"]), "boolean or numeric"), + (np.array([None, 1, 0], dtype=object), "boolean or numeric"), + ], +) +def test_a_non_integral_label_is_refused_before_the_int64_cast(labels, expected): + """``astype(np.int64)`` truncates toward zero, so a range test placed *after* it is blind. + + A target written as ``0.6`` becomes ``0`` and the fit then runs — successfully, and + bit-identically to the truncated targets — on values the caller never wrote; a ``NaN`` + label becomes the int64 sentinel and surfaces as a misleading ``range [-9.2e18, 1]``. + The domain check therefore happens in the original dtype. + """ + z = np.array([1.0, 1.0, -1.0]) + with pytest.raises(ValueError, match=expected): + R.temperature_scale(z, labels, rung=["calib"] * 3) + + # Positive control: the same logits with the integral targets the caller meant. + assert R.temperature_scale(z, np.array([1.0, 0.0, 0.0]), rung=["calib"] * 3).n_fitted == 3 + + +@pytest.mark.parametrize("labels", [[True, False, False], np.array([1, 0, 0], dtype=np.int8)]) +def test_boolean_and_integer_label_dtypes_are_accepted(labels): + """The refusal above must not also refuse the dtypes parquet actually delivers.""" + z = np.array([1.0, 1.0, -1.0]) + result = R.temperature_scale(z, labels, rung=["calib"] * 3, grad_tol=1e-8) + # ...and land on the same hand-derived root the float path does: T* = 1/ln2. + assert result.temperature == pytest.approx(1.0 / math.log(2.0), abs=1e-6) + + def test_length_disagreements_raise(): z, y, rungs = _table(_TINY) with pytest.raises(ValueError, match="rung carries"): @@ -412,6 +482,63 @@ def test_a_missing_calib_flag_raises_rather_than_reading_nan_as_true(missing): assert R.assign_rung(calib=True, fold_random="train") == "calib" # positive control +@pytest.mark.parametrize("token", ["False", "false", "0", "FALSE"]) +def test_a_stringified_false_calib_flag_is_read_as_false_not_as_truthy(token): + """``bool("False")`` is ``True`` — the second half of the same fail-open as NaN. + + The NaN guard closes the *missing* spelling; this closes the *rendered* one. A + string-typed `calib` column (an object-dtype widening, a re-materialised join) carrying + an explicit ``"False"`` would otherwise be promoted into the set ``T`` is fitted on, with + every count still looking clean. Read through ``masking.bool_or_none``, the same parse + ``stage2.train`` uses for the fold columns. + """ + assert bool(token) is True # the bite, stated outright + assert R.assign_rung(calib=token, fold_random="train") == "train" + + +@pytest.mark.parametrize("token", ["True", "true", "1", "TRUE"]) +def test_a_stringified_true_calib_flag_is_still_the_calib_rung(token): + """Positive control for the refusal above — the parse is strict, not blanket-rejecting.""" + assert R.assign_rung(calib=token, fold_random="train") == "calib" + + +@pytest.mark.parametrize("bad", ["yes", "no", "maybe", "0.5", [0], (0,), 0.5, -1, 2]) +def test_a_non_boolean_calib_flag_raises_rather_than_being_coerced(bad): + """Anything outside the boolean vocabulary is an error, never a silent ``True``. + + ``0.5``, ``-1``, ``2``, ``[0]`` and ``(0,)`` are all truthy-or-falsy under bare + ``bool()`` and none of them is a calibration flag; a column carrying them is a schema + fault, and a fault must not resolve to a rung. + """ + with pytest.raises(ValueError, match="neither missing nor boolean"): + R.assign_rung(calib=bad, fold_random="train") + assert R.assign_rung(calib=True, fold_random="train") == "calib" # positive control + + +def test_a_string_typed_calib_column_does_not_inflate_the_fit(): + """End to end: three genuine calib rows in a stringified column stay three. + + Before the strict parse this returned ``['calib'] * 6`` and `temperature_scale` reported + `n_fitted=6` — three *training* rows fitted as if they were the carve, with the census + agreeing. The identity assertion is the fitted `T`: it must equal the `T` from the three + intended rows alone, and differ from the six-row one. + """ + calib = ["True", "True", "True", "False", "False", "False"] + fold = ["train"] * 6 + z = np.array([1.0, 1.0, -1.0, 3.0, -3.0, 0.2]) + y = np.array([1, 0, 0, 1, 0, 1]) + + labels = R.rung_labels(calib=calib, fold_random=fold) + assert labels == ["calib", "calib", "calib", "train", "train", "train"] + + got = R.temperature_scale(z, y, rung=labels) + assert got.n_fitted == 3 + intended = R.temperature_scale(z[:3], y[:3], rung=["calib"] * 3) + assert got.temperature == intended.temperature + inflated = R.temperature_scale(z, y, rung=["calib"] * 6) + assert got.temperature != inflated.temperature + + @pytest.mark.parametrize("missing", [None, float("nan"), "", " "]) def test_a_missing_fold_raises(missing): with pytest.raises(ValueError, match="fold_random is missing"): @@ -643,6 +770,37 @@ def test_with_both_priors_the_full_pinned_stack_is_recorded_as_applied(): assert payload["target_prior_in_prd_band"] is True +def test_the_payload_echoes_its_inputs_unswapped_and_carries_its_own_provenance(): + """The recorded priors and ``T`` must be the ones actually used, in the right slots. + + Without this, swapping the two prior assignments produces an internally contradictory + artifact — ``source_prior`` holding the deployment prior, ``target_prior`` the calib + prevalence, and ``target_prior_in_prd_band`` computed from the *real* argument so it + still reads ``True`` — and the whole suite stays green. The provenance scalars are + asserted for the same reason: a constant nothing reads is a constant nothing checks. + """ + z = np.array([-2.0, 0.0, 1.5]) + p_s, p_t = 0.75, R.prior_from_odds_ratio(1e4) + payload = R.calibrated_posterior(z, temperature=1.1, source_prior=p_s, target_prior=p_t) + + assert payload["source_prior"] == p_s + assert payload["target_prior"] == p_t + assert payload["source_prior"] != payload["target_prior"] # asymmetric on purpose + assert payload["temperature"] == 1.1 + assert payload["n_rows"] == z.shape[0] == 3 + # The shift is derived from the recorded pair, so a swap could not stay self-consistent. + assert payload["log_odds_shift"] == pytest.approx( + R.log_odds_shift( + source_prior=payload["source_prior"], target_prior=payload["target_prior"] + ), + rel=0, + abs=1e-15, + ) + assert payload["step"] == "P3-07" + assert payload["generated_by"] == "src/tbox_finder/calib/recalibrate.py" + assert "D11" in payload["adr"] + + @pytest.mark.parametrize("kwargs", [{"source_prior": 0.5}, {"target_prior": 0.001}]) def test_a_half_specified_prior_shift_raises_rather_than_defaulting_one(kwargs): """The module must be unable to invent a deployment prior (§10.3). From 23f9342f6b5a00a1843e76a6c57b250f927ac26b Mon Sep 17 00:00:00 2001 From: bioedca Date: Mon, 3 Aug 2026 16:49:49 -0500 Subject: [PATCH 3/3] =?UTF-8?q?fix(masking):=20P3-07=20review=20r2=20?= =?UTF-8?q?=E2=80=94=20accept=20a=20float-widened=20boolean=20flag,=20stil?= =?UTF-8?q?l=20refuse=20the=20neighbours?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GitHub CodeRabbit check read `pass` on a "Review rate limited" body and no review had run on a721d06 (CLAUDE.md §5.1: a rate-limit notice is an absence, not a pass), so this round came from rung 1 of the ladder — the CodeRabbit CLI, scoped with --committed --base-commit 88879fd. One minor finding, and it is correct. masking.bool_or_none stringifies before matching, so a float flag becomes "1.0"/"0.0", matches no spelling, and raises. That reads like fail-closed behaviour on a schema fault until the shape is checked: pd.Series([True, None, False]).astype(float) and a `boolean`-dtype cast to float64 BOTH produce 1.0 / nan / 0.0. The NaN was already the missing case; the 1.0 and 0.0 beside it are a well-formed nullable-boolean column, and refusing them aborts a legitimate run. The parse now accepts a finite Real equal to exactly 0 or 1 before the text path. 0.5, 1.5, -0.5, 2 and -1 still raise — those are schema faults and the widening does not touch them. This is inherited behaviour, not new: stage2/train.py::_bool_or_none has had it since P3-06, and it only became visible because r1 moved the parse to a shared home where a reviewer read it. Fixing it once fixes it for the fold readers too — the argument for promoting rather than forking, paying off one round later. Validation: test_recalibrate 146 pass; masking + heads + stage2_dataset + stage2_train_smoke + temperature + no_leakage 502 pass beside it. Full unit+golden+ml: 14 failures, still a strict subset of main's 31 in this local runner (comm -13 over the two ^FAILED sets is empty). ruff + black clean. Two more sabotages, both bitten and in both directions: removing the acceptance reddens the float test, and widening it to any finite real (0.5 -> True) reddens the refusal test. --- analyses/phase3_log.qmd | 35 ++++++++++++++++++++++++++++++++++ src/tbox_finder/masking.py | 10 ++++++++++ tests/unit/test_recalibrate.py | 24 ++++++++++++++++++++++- 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/analyses/phase3_log.qmd b/analyses/phase3_log.qmd index d0886ea9..74333f45 100644 --- a/analyses/phase3_log.qmd +++ b/analyses/phase3_log.qmd @@ -1788,3 +1788,38 @@ boolean parse, all three label-domain branches, both halves of the single-class `GRADED_RUNGS`, the payload prior swap, the payload provenance, and the 1-D logits guard). **Commit.** Feature branch `p3-07-calibration-stack`; squash SHA derived at the phase-exit gate. + +## Step P3-07 (review r2) — 2026-08-03: one finding, and the GitHub check that lied about it + +**The gate check said `pass`; no review had run.** After the r1 push, the `CodeRabbit` PR check +went green with the body *"Review rate limited"*, and `pulls/100/reviews` showed exactly one +review, on the **step** commit — `a721d06` was unreviewed. This is the CLAUDE.md §5.1 trap +verbatim: a rate-limit notice is an **absence**, not a pass. Rung 1 of the ladder — the +CodeRabbit **CLI**, a separate path from the throttled GitHub app — ran the round instead, +scoped with `--committed --base-commit 88879fd` to the six files r1 actually touched. + +**One minor finding, and it is right.** `masking.bool_or_none` — the parse r1 had just promoted +— stringifies before matching, so a **float** flag becomes `"1.0"`/`"0.0"`, matches no spelling, +and raises. That looked at first like correct fail-closed behaviour on a schema fault, until the +shape was checked: `pd.Series([True, None, False]).astype(float)` and a `boolean`-dtype cast to +float64 **both** produce `1.0 / nan / 0.0`. The NaN was already the missing case; the `1.0` and +`0.0` beside it are a well-formed nullable-boolean column, and refusing them aborts a legitimate +run. So the parse now accepts a **finite Real equal to exactly 0 or 1** before the text path. +`0.5`, `1.5`, `-0.5`, `2` and `-1` still raise — they are schema faults and nothing about this +widening touches them. + +**Note the inherited-behaviour point.** This was not introduced by P3-07: it is the behaviour +`stage2/train.py::_bool_or_none` has had since P3-06, now visible because the parse moved to a +shared home and got read by a reviewer. Fixing it once fixes it for the fold readers too, which +is the whole argument for having promoted it rather than forked it. + +**Validation.** `test_recalibrate` **146 pass**; `test_masking` + `test_stage2_heads` + +`test_stage2_dataset` + `test_stage2_train_smoke` + `test_temperature` + `test_no_leakage` = 502 +pass beside it. Full `unit+golden+ml`: **14 failures, still a strict subset of `main`'s 31** in +this runner — `comm -13` over the two `^FAILED` sets is empty, so zero new failures, asserted +rather than eyeballed. ruff + black clean. **Two more sabotages, both bitten and both +directions**: removing the acceptance turns the float test red, and widening it to *any* finite +real (so `0.5` reads `True`) turns the refusal test red — a one-sided sabotage would have missed +the second, which is the failure that actually matters here. + +**Commit.** Feature branch `p3-07-calibration-stack`; squash SHA derived at the phase-exit gate. diff --git a/src/tbox_finder/masking.py b/src/tbox_finder/masking.py index dfb61dbd..d98e8633 100644 --- a/src/tbox_finder/masking.py +++ b/src/tbox_finder/masking.py @@ -31,9 +31,11 @@ from __future__ import annotations +import math import re from bisect import bisect_right from collections.abc import Iterable, Mapping, Sequence +from numbers import Real from pathlib import Path from typing import Any @@ -322,6 +324,14 @@ def bool_or_none(value: Any, *, field: str = "value") -> bool | None: return None if isinstance(value, bool): return value + if isinstance(value, Real) and math.isfinite(value) and (value == 0 or value == 1): + # A nullable boolean column that has been through a float widening — which is what + # `pd.Series([True, None, False]).astype(float)` and a `boolean`-dtype cast to + # float64 both produce — delivers `1.0 / nan / 0.0`. The NaN is caught above; the + # 1.0 and 0.0 would otherwise stringify to "1.0"/"0.0", match no spelling, and raise + # on a column that is perfectly well-formed. Exactly 0 and 1 only: 0.5, 2.0 and -1.0 + # are still schema faults and still raise (P3-07 review r2). + return bool(value) # numpy.bool_ / 0 / 1 / "True" all arrive here from parquet round-trips. text = row_text(value).strip().lower() if text in {"true", "1"}: diff --git a/tests/unit/test_recalibrate.py b/tests/unit/test_recalibrate.py index a111221f..0fa1aacc 100644 --- a/tests/unit/test_recalibrate.py +++ b/tests/unit/test_recalibrate.py @@ -502,7 +502,29 @@ def test_a_stringified_true_calib_flag_is_still_the_calib_rung(token): assert R.assign_rung(calib=token, fold_random="train") == "calib" -@pytest.mark.parametrize("bad", ["yes", "no", "maybe", "0.5", [0], (0,), 0.5, -1, 2]) +@pytest.mark.parametrize( + "flag,expected", + [ + (1.0, "calib"), + (0.0, "train"), + (np.float64(1.0), "calib"), + (np.float64(0.0), "train"), + (1, "calib"), + (0, "train"), + ], +) +def test_a_float_widened_boolean_column_is_read_not_refused(flag, expected): + """A nullable boolean column that went through float64 delivers ``1.0 / nan / 0.0``. + + That is what ``pd.Series([True, None, False]).astype(float)`` and a ``boolean``-dtype + cast both produce, and it is a well-formed column, not a schema fault. The NaN is + already the missing case; refusing the ``1.0``/``0.0`` beside it would abort a + legitimate run. Exactly 0 and 1 only — the neighbours below still raise (review r2). + """ + assert R.assign_rung(calib=flag, fold_random="train") == expected + + +@pytest.mark.parametrize("bad", ["yes", "no", "maybe", "0.5", [0], (0,), 0.5, -1, 2, 1.5, -0.5]) def test_a_non_boolean_calib_flag_raises_rather_than_being_coerced(bad): """Anything outside the boolean vocabulary is an error, never a silent ``True``.