From 6c326f5797e01d974db68c18a82947510103bcd0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 17:47:32 +0000 Subject: [PATCH 1/7] feat(leftover): persist leftover-map explained share MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Name e = R̂² / R² of raw residual after two-axis Gabriel reconstruction on leftover pair rows (ADR 0232 / migration 0232). Unexplained leftover share s stays omitted. A finite share greater than 1 is stored, never clamped. Next action opens the named post. --- AGENTS.md | 11 ++- ARCHITECTURE.md | 5 +- .../2.19.0-leftover-map-explained-share.md | 10 ++ CHANGELOG.md | 8 ++ CLAUDE.md | 2 +- backend/app/report_ingestion.py | 20 +++- backend/tests/test_api.py | 21 +++- .../0003-fast-mlsirm-report-integration.md | 7 +- docs/adr/0048-persist-lsirm-leftover-pairs.md | 8 +- docs/adr/0049-leftover-pair-report-ui.md | 19 ++-- docs/adr/0185-leftover-map-cross-share.md | 1 + docs/adr/0201-leftover-map-reconstruction.md | 1 + docs/adr/0232-leftover-map-explained-share.md | 99 +++++++++++++++++++ frontend/package.json | 2 +- frontend/src/App.test.tsx | 12 ++- frontend/src/api.ts | 1 + .../components/LeftoverPairList.stories.tsx | 4 + .../src/components/LeftoverPairList.test.tsx | 50 ++++++++++ frontend/src/components/LeftoverPairList.tsx | 31 +++++- frontend/src/i18n.test.ts | 28 ++++++ frontend/src/i18n.ts | 8 ++ .../src/leftoverMapExplainedShare.test.ts | 18 ++++ frontend/src/leftoverMapExplainedShare.ts | 13 +++ lineageweave/leftover_pairs.py | 58 ++++++++--- ...32_report_leftover_map_explained_share.sql | 14 +++ ...32_report_leftover_map_explained_share.sql | 5 + pyproject.toml | 2 +- scripts/seed_demo_data.py | 6 +- tests/test_leftover_pairs.py | 34 ++++++- tests/test_migration_replay.py | 23 +++++ tests/test_period_report.py | 4 +- tests/test_schema.py | 36 ++++++- uv.lock | 2 +- 33 files changed, 505 insertions(+), 58 deletions(-) create mode 100644 CHANGELOG.d/2.19.0-leftover-map-explained-share.md create mode 100644 docs/adr/0232-leftover-map-explained-share.md create mode 100644 frontend/src/leftoverMapExplainedShare.test.ts create mode 100644 frontend/src/leftoverMapExplainedShare.ts create mode 100644 migrations/0232_report_leftover_map_explained_share.sql create mode 100644 migrations/rollback/0232_report_leftover_map_explained_share.sql diff --git a/AGENTS.md b/AGENTS.md index c927f9e61..42fb304dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -271,16 +271,19 @@ stops startup instead of leaving a healthy-looking partial schema, and application code must not compensate for a missing table. Period leftover pairs (ADR 0017 / 0018 / 0048 / 0049 / 0119 / 0158 / 0162 / -0163 / 0164 / 0182 / 0201) are computed in `lineageweave/leftover_pairs.py` from the +0163 / 0164 / 0182 / 0185 / 0201 / 0232) are computed in `lineageweave/leftover_pairs.py` from the residual after a real GRM/GPCM score, never invented. Distances are Euclidean on the two-dimensional Gabriel leftover map; missing cells stay out of the factorization. Closest and farthest post–criterion pairs persist to `report_leftover_pair` with signed residual `R`, observed `Y`, and expected `E[Y|θ, item]` so `R = Y − E` remains auditable, plus leftover-map rank so rank 0 is not read as structure, -unexplained leftover, and the ADR 0201 reconstruction evidence. ADR 0201 -is the sole normative reconstruction formula, storage, and audit contract; -do not duplicate or reinterpret it here. The pairs sit above the member +unexplained leftover, ADR 0201 reconstruction evidence, leftover-map +cross share `x`, and leftover-map explained share `e = R̂² / R²` of +raw residual (ADR 0232). Unexplained leftover share `s` is not +persisted. ADR 0201 is the sole normative reconstruction formula, +storage, and audit contract; do not duplicate or reinterpret it here. +The pairs sit above the member list so a click opens that post with the leftover criterion current in Post quality (ADR 0158). Leftover-map axis share (ADR 0148) is Gabriel inertia of residual SVD axes 1 and 2 and persists to `report_leftover_map_axis`. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 399894584..95ebb9419 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -609,9 +609,10 @@ information at the group's mean θ (Lord, 1980 max-info CAT). Rankings persist to `report_item_information`. After those IRT main effects, residual SVD leftover pairs on two Gabriel axes (Jeon et al., 2021; ADR 0017 / 0048 / 0049 / 0119 / 0148 / 0158 / 0162 / 0163 / 0164 / 0168 / -0182 / 0185 / 0201) persist to `report_leftover_pair` with signed residual `R`, +0182 / 0185 / 0201 / 0232) persist to `report_leftover_pair` with signed residual `R`, observed `Y`, expected `E[Y|θ, item]`, full leftover-map rank, unexplained -leftover, ADR 0201 reconstruction evidence, and ADR 0185 cross-share evidence. +leftover, ADR 0201 reconstruction evidence, ADR 0185 cross-share evidence, +and leftover-map explained share `e = R̂² / R²` of raw residual (ADR 0232). Those ADRs are the normative mathematical and storage contracts. Leftover-map axis share (Gabriel inertia of residual SVD axes 1 and 2; ADR 0148) persists to `report_leftover_map_axis`. Complete-case leftover-map coverage (ADR diff --git a/CHANGELOG.d/2.19.0-leftover-map-explained-share.md b/CHANGELOG.d/2.19.0-leftover-map-explained-share.md new file mode 100644 index 000000000..b02fc8c8e --- /dev/null +++ b/CHANGELOG.d/2.19.0-leftover-map-explained-share.md @@ -0,0 +1,10 @@ +## 2.19.0 — Leftover-map explained share + +- Persist leftover-map explained share `e = R̂² / R²` of raw residual + on leftover post–criterion pairs (ADR 0232). After `make seed`, + closest and farthest leftover pairs sit above the member list with + `R̂²/R²` next to leftover-map distance `d`; click opens that post. + Omit the badge when the share is missing. A finite share greater than + 1 is shown, never clamped. Never invent a leftover score. Do not + introduce leftover-map unexplained share `s`; ADR 0185 remains + authoritative for `x` and ADR 0201 for `R̂`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 641306055..2bed821c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ All notable changes to this project are documented here. Format follows ### Added +- Period leftover pair rows now name leftover-map explained share + `e = R̂² / R²` of raw residual after two-axis Gabriel reconstruction + next to leftover-map distance `d`, then open that post (Gabriel, 1971; + Jeon et al., 2021, eq. 3; ADR 0232). A missing share omits the badge + rather than inventing a leftover score. A finite share greater than 1 + is stored and shown, never clamped. Unexplained leftover share `s` is + not persisted. + - Persist explicit paragraph, list, table, MathML formula, and caller-parsed conversation-turn semantic-unit kinds without inferring absent boundaries. - Event Lineage now persists each reconstructed connection's independent diff --git a/CLAUDE.md b/CLAUDE.md index eb9e85eab..e612bb29f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,7 +48,7 @@ cutoff. Create/start endpoint rules (ADR 0017 / 0021), tie-vs-miss similarity (ADR 0026), R&R catalog ids (ADR 0019 / 0027), leftover pairs -(ADR 0048–0164 / 0182 / 0201), the text-channel embedding swap and cosine +(ADR 0048–0164 / 0182 / 0185 / 0201 / 0232), the text-channel embedding swap and cosine clamp (ADR 0190), per-edge channel-score persistence (ADR 0195), migration replay (ADR 0166), docstring coverage, and the measurement boundary are all stated in [AGENTS.md](AGENTS.md) -- read it before diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py index f01c15ae0..e44285c66 100644 --- a/backend/app/report_ingestion.py +++ b/backend/app/report_ingestion.py @@ -447,8 +447,8 @@ async def persist_period_report( pair_kind, post_id, criterion_code, leftover_distance, leftover_residual, observed_response, expected_response, leftover_map_rank, leftover_map_unexplained, leftover_map_cross_share, - leftover_map_reconstruction - ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) + leftover_map_reconstruction, leftover_map_explained_share + ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) """, grouping_kind, grouping_key, @@ -465,6 +465,7 @@ async def persist_period_report( pair.leftover_map_unexplained, pair.leftover_map_cross_share, pair.leftover_map_reconstruction, + pair.leftover_map_explained_share, ) for axis in report.leftover_map_axes: await conn.execute( @@ -652,7 +653,8 @@ async def fetch_period_reports( lp.leftover_distance, lp.leftover_residual, lp.observed_response, lp.expected_response, lp.leftover_map_rank, lp.leftover_map_unexplained, lp.leftover_map_cross_share, - lp.leftover_map_reconstruction, p.post_title, + lp.leftover_map_reconstruction, lp.leftover_map_explained_share, + p.post_title, p.visibility_code, p.corporate_entity_id, p.process_unit_id, ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context from report_leftover_pair lp @@ -809,6 +811,11 @@ async def fetch_period_reports( if row["leftover_map_reconstruction"] is None else float(row["leftover_map_reconstruction"]) ), + "leftover_map_explained_share": ( + None + if row["leftover_map_explained_share"] is None + else float(row["leftover_map_explained_share"]) + ), "visibility_code": row["visibility_code"], "corporate_entity_id": str(row["corporate_entity_id"]), "process_unit_id": ( @@ -1010,7 +1017,7 @@ async def fetch_period_comparison( f""" select lp.grouping_kind, lp.grouping_key, lp.pair_kind, lp.post_id, lp.criterion_code, lp.leftover_distance, lp.leftover_residual, - lp.leftover_map_reconstruction, + lp.leftover_map_reconstruction, lp.leftover_map_explained_share, p.post_title, p.visibility_code, p.corporate_entity_id, ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context from report_leftover_pair lp @@ -1065,6 +1072,11 @@ async def fetch_period_comparison( if pair["leftover_map_reconstruction"] is None else float(pair["leftover_map_reconstruction"]) ), + "leftover_map_explained_share": ( + None + if pair["leftover_map_explained_share"] is None + else float(pair["leftover_map_explained_share"]) + ), "visibility_code": pair["visibility_code"], "corporate_entity_id": str(pair["corporate_entity_id"]), "has_real_source_context": bool(pair["has_real_source_context"]), diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 892c8231a..628ab239a 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -181,6 +181,11 @@ / "migrations" / "0206_report_leftover_map_reconstruction.sql" ) +_LEFTOVER_MAP_EXPLAINED_SHARE_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0232_report_leftover_map_explained_share.sql" +) _GLOBAL_ASK_JOB_MIGRATION = ( Path(__file__).resolve().parents[2] / "migrations" @@ -392,6 +397,7 @@ def seeded_db(demo_analyst_token): cur.execute(_LEFTOVER_MAP_UNEXPLAINED_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_CROSS_SHARE_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_RECONSTRUCTION_MIGRATION.read_text()) + cur.execute(_LEFTOVER_MAP_EXPLAINED_SHARE_MIGRATION.read_text()) cur.execute( "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values " "('corporate_entity_level', 'group', 'Group'), " @@ -5700,9 +5706,14 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token, if share is not None: assert not math.isnan(share) assert not math.isinf(share) + explained = pair.get("leftover_map_explained_share") + assert "leftover_map_explained_share" in pair + assert explained is None or isinstance(explained, (int, float)) + if explained is not None: + assert not math.isnan(explained) + assert not math.isinf(explained) if unexplained is not None and reconstruction is not None: assert unexplained + reconstruction == pytest.approx(pair["leftover_residual"]) - assert "leftover_map_explained_share" not in pair assert "leftover_map_unexplained_share" not in pair leftover_axes = high_report.get("leftover_map_axes", []) assert [axis["axis_index"] for axis in leftover_axes] == [1, 2] @@ -5766,6 +5777,14 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token, or isinstance(pair["leftover_map_reconstruction"], (int, float)) for pair in leftover_thread.get("leftover_pairs", []) ) + assert all( + "leftover_map_explained_share" in pair + and ( + pair["leftover_map_explained_share"] is None + or isinstance(pair["leftover_map_explained_share"], (int, float)) + ) + for pair in leftover_thread.get("leftover_pairs", []) + ) def test_seed_period_report_includes_fixture_event_lineage_posts( diff --git a/docs/adr/0003-fast-mlsirm-report-integration.md b/docs/adr/0003-fast-mlsirm-report-integration.md index 11decffc0..944c396a1 100644 --- a/docs/adr/0003-fast-mlsirm-report-integration.md +++ b/docs/adr/0003-fast-mlsirm-report-integration.md @@ -108,9 +108,10 @@ than one large PR: public Rust-backed prediction API (upstream PR #1279); LineageWeave must not reproduce GRM/GPCM parameter conventions locally. 8. **Leftover evidence extensions:** unexplained leftover shipped in 2.12.26 - (ADR 0182), cross-share evidence shipped in 2.12.29 (ADR 0185), and - reconstruction evidence is Unreleased for 2.12.31 (ADR 0201). Do not - persist explained share, unexplained share, or another unsupported alias. + (ADR 0182), cross-share evidence shipped in 2.12.29 (ADR 0185), + reconstruction evidence shipped in 2.12.31 (ADR 0201), and leftover-map + explained share shipped in 2.19.0 (ADR 0232). Do not persist unexplained + leftover share `s` or another unsupported alias. 9. **Leftover-map axis-share slice** (ADR 0148): persist Gabriel inertia `σ_k² / Σ_j σ_j²` of leftover-map axes 1 and 2 on the same residual SVD. Rank-0 residuals emit two zero-share axes. Do not invent a diff --git a/docs/adr/0048-persist-lsirm-leftover-pairs.md b/docs/adr/0048-persist-lsirm-leftover-pairs.md index 613545db4..767600f48 100644 --- a/docs/adr/0048-persist-lsirm-leftover-pairs.md +++ b/docs/adr/0048-persist-lsirm-leftover-pairs.md @@ -6,7 +6,9 @@ [ADR 0163](0163-leftover-observed-expected.md) (observed Y and expected E); [ADR 0164](0164-leftover-map-rank.md) (full map rank); [ADR 0182](0182-leftover-map-unexplained.md) (unexplained leftover U); -[ADR 0185](0185-leftover-map-cross-share.md) (leftover-map cross share) +[ADR 0185](0185-leftover-map-cross-share.md) (leftover-map cross share); +[ADR 0201](0201-leftover-map-reconstruction.md) (signed reconstruction R̂); +[ADR 0232](0232-leftover-map-explained-share.md) (leftover-map explained share) ## Context @@ -49,7 +51,9 @@ read as leftover residual `R`, leftover-map distance `d`, explained leftover share `e`, or unexplained leftover share `s` (ADR 0185). ADR 0201 now persists that same signed reconstruction on the pair row so `U + R̂ = R` remains directly auditable; it does not change this selection or -distance contract. +distance contract. ADR 0232 persists leftover-map explained share +`e = R̂² / R²` of raw residual so `e + s + x = 1` is not read from `x` +alone; unexplained leftover share `s` is still not persisted. Cascade the rows with `report_period_score`. A leftover post must also be a `report_member_score` row, and the leftover criterion diff --git a/docs/adr/0049-leftover-pair-report-ui.md b/docs/adr/0049-leftover-pair-report-ui.md index 4be472ef9..f36e98e6a 100644 --- a/docs/adr/0049-leftover-pair-report-ui.md +++ b/docs/adr/0049-leftover-pair-report-ui.md @@ -8,7 +8,8 @@ [ADR 0182](0182-leftover-map-unexplained.md) (unexplained leftover U); [ADR 0158](0158-leftover-criterion-evaluation-landing.md) (criterion evaluation landing); [ADR 0185](0185-leftover-map-cross-share.md) (leftover-map cross share); -[ADR 0201](0201-leftover-map-reconstruction.md) (signed reconstruction R̂) +[ADR 0201](0201-leftover-map-reconstruction.md) (signed reconstruction R̂); +[ADR 0232](0232-leftover-map-explained-share.md) (leftover-map explained share) ## Context @@ -26,16 +27,17 @@ On each period-report group, render leftover pairs **above** the member list. Each pair is a button: closest or farthest label, post title, criterion short label, signed residual `R`, two-axis leftover-map distance, full map rank, observed `Y`, expected `E` when finite, -unexplained leftover `U`, signed reconstruction `R̂` when finite, and -leftover-map cross share next to distance when finite. The next action names every available +unexplained leftover `U`, signed reconstruction `R̂` when finite, +leftover-map cross share next to distance when finite, and leftover-map +explained share `e = R̂² / R²` of raw residual when finite. The next action names every available measurement before opening the post; no amendment hides another, rank 0 explicitly names no leftover structure, and unexplained leftover names "leftover map leaves unexplained `U` after IRT main effects; open this -post to read the named criterion" when present. When leftover-map cross -share is also present, the next action instead names the identity -remainder `x` two leftover-map axes leave in raw residual after -IRT main effects. A missing or non-finite value falls back in order — -cross share, then reconstruction, then unexplained leftover, then the existing +post to read the named criterion" when present. When leftover-map +explained share is also present, the next action names how much of the +raw residual two leftover-map axes explain after IRT main effects. +A missing or non-finite value falls back in order — +explained share, then cross share, then reconstruction, then unexplained leftover, then the existing closest/farthest next action. Clicking the button opens that post with leftover focus so Post quality marks the named criterion current (ADR 0158). Residual naming is @@ -45,6 +47,7 @@ is [ADR 0164](0164-leftover-map-rank.md), unexplained leftover naming is [ADR 0182](0182-leftover-map-unexplained.md), leftover-map cross share naming is [ADR 0185](0185-leftover-map-cross-share.md). Reconstruction naming is [ADR 0201](0201-leftover-map-reconstruction.md). +Explained-share naming is [ADR 0232](0232-leftover-map-explained-share.md). After `make seed`, closest and farthest leftover pairs sit above the member list. Click a pair to open that post with the leftover diff --git a/docs/adr/0185-leftover-map-cross-share.md b/docs/adr/0185-leftover-map-cross-share.md index 755523b43..3beb0bb5a 100644 --- a/docs/adr/0185-leftover-map-cross-share.md +++ b/docs/adr/0185-leftover-map-cross-share.md @@ -2,6 +2,7 @@ **Decision status:** Draft **Date:** 2026-08-24 +**Amended by:** [ADR 0232](0232-leftover-map-explained-share.md) (leftover-map explained share `e = R̂² / R²`) Amends [ADR 0048](0048-persist-lsirm-leftover-pairs.md) and [ADR 0049](0049-leftover-pair-report-ui.md). diff --git a/docs/adr/0201-leftover-map-reconstruction.md b/docs/adr/0201-leftover-map-reconstruction.md index 049208105..43eae9a7c 100644 --- a/docs/adr/0201-leftover-map-reconstruction.md +++ b/docs/adr/0201-leftover-map-reconstruction.md @@ -2,6 +2,7 @@ **Decision status:** Accepted **Date:** 2026-08-25 +**Amended by:** [ADR 0232](0232-leftover-map-explained-share.md) (leftover-map explained share `e = R̂² / R²`) Amends [ADR 0048](0048-persist-lsirm-leftover-pairs.md), [ADR 0049](0049-leftover-pair-report-ui.md), and diff --git a/docs/adr/0232-leftover-map-explained-share.md b/docs/adr/0232-leftover-map-explained-share.md new file mode 100644 index 000000000..5e6fc971a --- /dev/null +++ b/docs/adr/0232-leftover-map-explained-share.md @@ -0,0 +1,99 @@ +# ADR 0232 — Name leftover-map explained share on period-report pair rows + +**Decision status:** Accepted +**Date:** 2026-08-27 + +Amends [ADR 0048](0048-persist-lsirm-leftover-pairs.md), +[ADR 0049](0049-leftover-pair-report-ui.md), +[ADR 0185](0185-leftover-map-cross-share.md), and +[ADR 0201](0201-leftover-map-reconstruction.md). Independent of leftover-map +unexplained leftover ([ADR 0182](0182-leftover-map-unexplained.md)). + +## Context + +ADR 0185 already persists leftover-map cross share +`x = 2 R̂ U / R²` of raw residual after two-axis Gabriel reconstruction +`R̂ = ξ_{1:2} · ζ_{1:2}` and unexplained leftover `U = R − R̂`. ADR 0201 +already persists signed reconstruction `R̂` so `U + R̂ = R` stays +auditable. The raw-residual cell identity +`R² = R̂² + U² + 2 R̂ U` therefore yields +`e + s + x = 1` with explained leftover share `e = R̂² / R²`, +unexplained leftover share `s = U² / R²`, and leftover-map cross +share `x`. Hiding `e` lets a buyer read `x` (or `R̂`) as the leftover +the two-axis map explains. `e` is nonnegative when finite. Truncated +two-axis reconstruction of a higher-rank cell can make `|R̂| > |R|`, so +a finite explained share may exceed 1; a unit CHECK would reject a +mathematically honest cell. + +This increment does not persist leftover-map unexplained leftover share +`s`, does not persist leftover-map coordinates, does not name leftover-map +inner product, cosine, or length, and does not land Post quality on the +leftover criterion. Leftover-map distance stays two-axis Euclidean. +Reconstruction `R̂` remains the ADR 0201 internal two-axis inner +product already used for `U` and `x`. + +The unprotected-stack reconstructions for neighbouring leftover facts +use 0162–0186. This protected-main increment uses **0232** so it does +not collide with leftover-map reconstruction (0201 / migration 0206), +leftover-map cross share (0185), leftover-map unexplained leftover +(0182), leftover-map unexplained leftover share, leftover residual +disclosure, leftover observed `Y` / expected `E`, leftover-map rank, +two-axis leftover-map distance, leftover coverage, leftover-map axis +share (0148), leftover interaction-map persistence, or shared +token-backed status notice (0214 on an open stack). + +## Decision + +Each leftover pair names `leftover_map_explained_share` — leftover-map +explained share `e = R̂² / R²` of raw residual after two-axis Gabriel +reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}`. Migration `0232` is the +single source of the column on every install path, fresh or existing -- +shipped migrations (`0001` / `0012`) are never edited after the fact. +The column is nullable so older leftover rows keep distance, residual, +unexplained leftover, cross share, and reconstruction without +fabricating a share. Fallback pairs that have no complete-case leftover +map omit the value rather than inventing one. A rank-0 origin cell +stores `0.0` when `R = R̂ = 0`, not a missing value. A non-finite +share stores null rather than inventing a leftover score. A finite +share greater than 1 is stored; do not add a unit or nonnegative CHECK. +This increment does not introduce `leftover_map_unexplained_share`. + +The pair button shows `R̂²/R² {share}` next to leftover-map distance +`d` when the value is a finite number. Next action: leftover map +explains `{share}` of raw residual after IRT main effects; open this +post to read the named criterion. Explained leftover share takes +priority over leftover-map cross share when both are finite. A missing +or non-finite share omits the badge and keeps the existing cross-share +next action. Do not invent a leftover score. Do not invent a theta. + +## Consequences + +`GET /api/reports/{grouping}/{period}` returns +`leftover_map_explained_share`. After `make seed`, closest and farthest +leftover pairs sit above the member list with named `R̂²/R²` next to +`d`; click opens that post. Hidden posts stay hidden. When explained +share, unexplained leftover, reconstruction, and cross share are all +finite and `R ≠ 0`, `e + s + x = 1` with `s = U² / R²` computed only +for the audit identity, not persisted. + +## Related + +Independent of leftover interaction-map persistence, leftover-criterion +evaluation landing, leftover residual disclosure, leftover observed +`Y` / expected `E`, leftover-map complete-case coverage, leftover-map +axis share, leftover pairs on the grouping comparison strip, two-axis +leftover-map distance, leftover-map rank, leftover-map inner product, +leftover-map cosine, leftover-map length, leftover-map unexplained +share, leftover-map unexplained leftover, leftover-map reconstruction, +and leftover-map cross share. + +## References + +Gabriel, K. R. (1971). The biplot graphic display of matrices with +application to principal component analysis. *Biometrika, 58*(3), +453–467. https://doi.org/10.1093/biomet/58.3.453 + +Jeon, M., Jin, I. H., Schweinberger, M., & Baugh, S. (2021). Mapping +unobserved item–respondent interactions: A latent space item response +model with interaction map. *Psychometrika, 86*(2), 378–403. +https://doi.org/10.1007/s11336-021-09762-5 diff --git a/frontend/package.json b/frontend/package.json index 5acf284d7..bf3371abe 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "2.17.0", + "version": "2.19.0", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 2dee4513d..860dcfbfc 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -993,6 +993,7 @@ describe("App, authenticated", () => { leftover_map_rank: 1, leftover_map_cross_share: 0.12, leftover_map_reconstruction: 0.35, + leftover_map_explained_share: 0.76, }, { pair_kind: "farthest", @@ -1007,6 +1008,7 @@ describe("App, authenticated", () => { leftover_map_rank: 1, leftover_map_cross_share: -0.24, leftover_map_reconstruction: -0.85, + leftover_map_explained_share: 0.60, }, ], leftover_map_axes: [ @@ -3979,10 +3981,10 @@ describe("App, authenticated", () => { name: /open leftover farthest pair: specification revision requested/i, }); expect(closestPair).toHaveTextContent("Closest leftover: Public post · sales-lead"); - // Leftover-map cross share is present, so it names the next action - // instead of the rank/observed-expected chain (ADR 0185). + // Leftover-map explained share is present, so it names the next action + // ahead of cross share (ADR 0232). expect(closestPair).toHaveTextContent( - "Two leftover-map axes leave identity remainder 0.12 of raw residual after IRT main effects. Open this post to read sales-lead.", + "Leftover map explains 0.76 of raw residual after IRT main effects. Open this post to read sales-lead.", ); expect(closestPair).toHaveTextContent("R +0.40"); expect(closestPair).toHaveTextContent("Y 2.40 · E 2.00"); @@ -3990,11 +3992,12 @@ describe("App, authenticated", () => { expect(closestPair).toHaveTextContent("U +0.05"); expect(closestPair).toHaveTextContent("2R̂U/R² 0.12"); expect(closestPair).toHaveTextContent("R̂ +0.35"); + expect(closestPair).toHaveTextContent("R̂²/R² 0.76"); expect(closestPair).toHaveTextContent("d 0.12"); expect(closestPair).toHaveAccessibleName("Open leftover closest pair: Public post · sales-lead"); expect(farthestPair).toHaveTextContent("Farthest leftover: Specification revision requested · negative"); expect(farthestPair).toHaveTextContent( - "Two leftover-map axes leave identity remainder -0.24 of raw residual after IRT main effects. Open this post to read negative.", + "Leftover map explains 0.60 of raw residual after IRT main effects. Open this post to read negative.", ); expect(farthestPair).toHaveTextContent("R −1.10"); expect(farthestPair).toHaveTextContent("Y 0.90 · E 2.00"); @@ -4002,6 +4005,7 @@ describe("App, authenticated", () => { expect(farthestPair).toHaveTextContent("U −0.25"); expect(farthestPair).toHaveTextContent("2R̂U/R² -0.24"); expect(farthestPair).toHaveTextContent("R̂ −0.85"); + expect(farthestPair).toHaveTextContent("R̂²/R² 0.60"); expect(farthestPair).toHaveTextContent("d 1.84"); const memberButton = screen.getByRole("button", { name: /open report post: public post/i }); expect(coverageCaption.compareDocumentPosition(closestPair) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); diff --git a/frontend/src/api.ts b/frontend/src/api.ts index fca5882d0..96b6c681a 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -1032,6 +1032,7 @@ export interface LeftoverPair { leftover_map_unexplained?: number | null; leftover_map_cross_share?: number | null; leftover_map_reconstruction?: number | null; + leftover_map_explained_share?: number | null; } export interface LeftoverMapAxis { diff --git a/frontend/src/components/LeftoverPairList.stories.tsx b/frontend/src/components/LeftoverPairList.stories.tsx index 637351074..7e28e4373 100644 --- a/frontend/src/components/LeftoverPairList.stories.tsx +++ b/frontend/src/components/LeftoverPairList.stories.tsx @@ -21,6 +21,8 @@ const meta = { leftover_map_rank: 1, leftover_map_unexplained: 0.05, leftover_map_reconstruction: 0.35, + leftover_map_cross_share: 0.12, + leftover_map_explained_share: 0.76, }, { pair_kind: "farthest", @@ -34,6 +36,8 @@ const meta = { leftover_map_rank: 1, leftover_map_unexplained: -0.25, leftover_map_reconstruction: -0.85, + leftover_map_cross_share: -0.24, + leftover_map_explained_share: 0.60, }, ], }, diff --git a/frontend/src/components/LeftoverPairList.test.tsx b/frontend/src/components/LeftoverPairList.test.tsx index 36c15715a..4ddc0cd01 100644 --- a/frontend/src/components/LeftoverPairList.test.tsx +++ b/frontend/src/components/LeftoverPairList.test.tsx @@ -150,6 +150,56 @@ describe("LeftoverPairList", () => { expect(closest).toHaveTextContent("d 0.12"); }); + it("names leftover-map explained share ahead of cross share and reconstruction", () => { + render( + , + ); + + const closest = screen.getByRole("button"); + expect(closest).toHaveTextContent( + "Leftover map explains 0.76 of raw residual after IRT main effects. Open this post to read sales-lead.", + ); + expect(closest).toHaveTextContent("R̂²/R² 0.76"); + expect(closest).toHaveTextContent("2R̂U/R² 0.12"); + expect(closest).toHaveTextContent("R̂ +0.35"); + expect(closest).toHaveTextContent("U +0.05"); + expect(closest).toHaveTextContent("R +0.40"); + expect(closest).toHaveTextContent("d 0.12"); + }); + + it("keeps cross-share guidance when explained share is missing", () => { + render( + , + ); + + expect(screen.getByRole("button")).toHaveTextContent( + "Two leftover-map axes leave identity remainder 0.12 of raw residual after IRT main effects. Open this post to read sales-lead.", + ); + }); + it("keeps unexplained guidance when reconstruction is missing", () => { render( {unexplained} : null} {crossShareBadge ? {crossShareBadge} : null} {reconstruction ? {reconstruction} : null} + {explainedShareBadge ? {explainedShareBadge} : null} d {pair.leftover_distance.toFixed(2)} diff --git a/frontend/src/i18n.test.ts b/frontend/src/i18n.test.ts index ae085d010..82e9987a6 100644 --- a/frontend/src/i18n.test.ts +++ b/frontend/src/i18n.test.ts @@ -55,6 +55,7 @@ describe("i18n", () => { "Leftover map leaves unexplained U {value} after IRT main effects. Open this post to read {criterion}.", "Leftover map reconstructs R̂ {value} after IRT main effects. Open this post to read {criterion}.", "Two leftover-map axes leave identity remainder {value} of raw residual after IRT main effects. Open this post to read {criterion}.", + "Leftover map explains {value} of raw residual after IRT main effects. Open this post to read {criterion}.", "Read observed Y {observed} and expected E {expected} after IRT main effects, then open this post.", "Leftover map has no leftover structure after IRT main effects. Open this post.", "Leftover map rank {rank} after IRT main effects. Open this post.", @@ -302,6 +303,33 @@ describe("i18n", () => { ), ).toBe(expected); }); + + it.each([ + [ + "ko", + "잔여 지도가 IRT 주효과 이후 원잔차의 0.76를 설명합니다. sales-lead 기준을 읽으려면 이 글을 여세요.", + ], + [ + "zh", + "残差图解释 IRT 主效应后原始残差的 0.76。打开这篇帖子阅读 sales-lead。", + ], + [ + "ja", + "残差マップはIRT主効果後の生の残差の 0.76 を説明します。この投稿を開いて sales-lead を読んでください。", + ], + [ + "vi", + "Bản đồ phần dư giải thích 0.76 của phần dư thô sau hiệu ứng chính IRT. Mở bài viết này để đọc sales-lead.", + ], + ] as const)("formats leftover-map explained share next action in %s", (locale, expected) => { + setLocale(locale); + expect( + tf( + "Leftover map explains {value} of raw residual after IRT main effects. Open this post to read {criterion}.", + { value: "0.76", criterion: "sales-lead" }, + ), + ).toBe(expected); + }); }); describe("locale-aware source labels", () => { diff --git a/frontend/src/i18n.ts b/frontend/src/i18n.ts index 390be07d8..f296534b6 100644 --- a/frontend/src/i18n.ts +++ b/frontend/src/i18n.ts @@ -534,6 +534,8 @@ const TRANSLATIONS: Partial>> = { "잔여 지도가 IRT 주효과 이후 R̂ {value}을(를) 재구성합니다. {criterion} 기준을 읽으려면 이 글을 여세요.", "Two leftover-map axes leave identity remainder {value} of raw residual after IRT main effects. Open this post to read {criterion}.": "잔여 지도의 두 축이 IRT 주효과 이후 원시 잔차의 항등식 나머지 {value}을(를) 남깁니다. {criterion} 기준을 읽으려면 이 글을 여세요.", + "Leftover map explains {value} of raw residual after IRT main effects. Open this post to read {criterion}.": + "잔여 지도가 IRT 주효과 이후 원잔차의 {value}를 설명합니다. {criterion} 기준을 읽으려면 이 글을 여세요.", "Read observed Y {observed} and expected E {expected} after IRT main effects, then open this post.": "IRT 주효과 이후 관측 Y {observed}와 기대 E {expected}를 읽은 다음, 이 글을 여세요.", "Leftover map has no leftover structure after IRT main effects. Open this post.": @@ -1053,6 +1055,8 @@ const TRANSLATIONS: Partial>> = { "残差图在 IRT 主效应后重建 R̂ {value}。打开这篇帖子阅读 {criterion}。", "Two leftover-map axes leave identity remainder {value} of raw residual after IRT main effects. Open this post to read {criterion}.": "残差图的两个轴在 IRT 主效应后留下原始残差的恒等式余项 {value}。打开这篇帖子阅读 {criterion}。", + "Leftover map explains {value} of raw residual after IRT main effects. Open this post to read {criterion}.": + "残差图解释 IRT 主效应后原始残差的 {value}。打开这篇帖子阅读 {criterion}。", "Read observed Y {observed} and expected E {expected} after IRT main effects, then open this post.": "阅读 IRT 主效应后的观测 Y {observed} 与期望 E {expected}，然后打开这篇帖子。", "Leftover map has no leftover structure after IRT main effects. Open this post.": @@ -1575,6 +1579,8 @@ const TRANSLATIONS: Partial>> = { "残差マップはIRT主効果後の R̂ {value} を再構成します。この投稿を開いて {criterion} を読んでください。", "Two leftover-map axes leave identity remainder {value} of raw residual after IRT main effects. Open this post to read {criterion}.": "残差マップの2軸はIRT主効果後の生の残差の恒等式の余り {value} を残します。この投稿を開いて {criterion} を読んでください。", + "Leftover map explains {value} of raw residual after IRT main effects. Open this post to read {criterion}.": + "残差マップはIRT主効果後の生の残差の {value} を説明します。この投稿を開いて {criterion} を読んでください。", "Read observed Y {observed} and expected E {expected} after IRT main effects, then open this post.": "IRT主効果後の観測 Y {observed} と期待 E {expected} を読んでから、この投稿を開いてください。", "Leftover map has no leftover structure after IRT main effects. Open this post.": @@ -2097,6 +2103,8 @@ const TRANSLATIONS: Partial>> = { "Bản đồ phần dư tái dựng R̂ {value} sau hiệu ứng chính IRT. Mở bài viết này để đọc {criterion}.", "Two leftover-map axes leave identity remainder {value} of raw residual after IRT main effects. Open this post to read {criterion}.": "Hai trục của bản đồ phần dư để lại phần giao {value} của phần dư thô sau hiệu ứng chính IRT. Mở bài viết này để đọc {criterion}.", + "Leftover map explains {value} of raw residual after IRT main effects. Open this post to read {criterion}.": + "Bản đồ phần dư giải thích {value} của phần dư thô sau hiệu ứng chính IRT. Mở bài viết này để đọc {criterion}.", "Read observed Y {observed} and expected E {expected} after IRT main effects, then open this post.": "Đọc Y quan sát {observed} và E kỳ vọng {expected} sau hiệu ứng chính IRT, rồi mở bài viết này.", "Leftover map has no leftover structure after IRT main effects. Open this post.": diff --git a/frontend/src/leftoverMapExplainedShare.test.ts b/frontend/src/leftoverMapExplainedShare.test.ts new file mode 100644 index 000000000..a1cec6589 --- /dev/null +++ b/frontend/src/leftoverMapExplainedShare.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { formatLeftoverMapExplainedShare } from "./leftoverMapExplainedShare"; + +describe("formatLeftoverMapExplainedShare", () => { + it("names leftover-map explained share without inventing a leftover score", () => { + expect(formatLeftoverMapExplainedShare(0.76)).toBe("R\u0302\u00b2/R\u00b2 0.76"); + expect(formatLeftoverMapExplainedShare(0)).toBe("R\u0302\u00b2/R\u00b2 0.00"); + expect(formatLeftoverMapExplainedShare(1.25)).toBe("R\u0302\u00b2/R\u00b2 1.25"); + }); + + it("omits the badge when leftover-map explained share is missing or non-finite", () => { + expect(formatLeftoverMapExplainedShare(null)).toBeNull(); + expect(formatLeftoverMapExplainedShare(undefined)).toBeNull(); + expect(formatLeftoverMapExplainedShare(Number.NaN)).toBeNull(); + expect(formatLeftoverMapExplainedShare(Number.POSITIVE_INFINITY)).toBeNull(); + expect(formatLeftoverMapExplainedShare(Number.NEGATIVE_INFINITY)).toBeNull(); + }); +}); diff --git a/frontend/src/leftoverMapExplainedShare.ts b/frontend/src/leftoverMapExplainedShare.ts new file mode 100644 index 000000000..e6b4dca15 --- /dev/null +++ b/frontend/src/leftoverMapExplainedShare.ts @@ -0,0 +1,13 @@ +/** Leftover-map explained share ``e = R̂² / R²`` of raw residual. */ + +export const LEFTOVER_MAP_EXPLAINED_SHARE_ACTION = + "Leftover map explains {value} of raw residual after IRT main effects. Open this post to read {criterion}."; + +export function formatLeftoverMapExplainedShare( + value: number | null | undefined, +): string | null { + if (value == null || !Number.isFinite(value)) { + return null; + } + return `R\u0302\u00b2/R\u00b2 ${value.toFixed(2)}`; +} diff --git a/lineageweave/leftover_pairs.py b/lineageweave/leftover_pairs.py index 070416fe2..32dbebfbe 100644 --- a/lineageweave/leftover_pairs.py +++ b/lineageweave/leftover_pairs.py @@ -1,7 +1,7 @@ """Jeon leftover post–criterion pairs after a main-effect IRT. Implements ADR 0048 as amended by ADR 0119, ADR 0163, ADR 0164, ADR 0182, -and ADR 0185. +ADR 0185, ADR 0201, and ADR 0232. Does not import ``fast_mlsirm`` or ``period_report``. A Gabriel biplot of the residual ``R = Y − E[Y|θ, item]`` supplies person and item @@ -25,10 +25,13 @@ residual after that same truncated two-axis reconstruction, so the identity remainder left by the truncation is not confused with leftover residual ``R``, leftover-map distance ``d``, or unexplained -leftover ``U``. Explained leftover share ``e = R̂² / R²`` and -unexplained leftover share ``s = U² / R²`` are not persisted. Signed -reconstruction ``R̂`` is persisted so ``U + R̂ = R`` stays auditable. ``x`` -may be negative when reconstruction and unexplained leftover have opposite signs. +leftover ``U``. Explained leftover share ``e = R̂² / R²`` is persisted +so ``e + s + x = 1`` is not read from ``x`` alone. Unexplained leftover +share ``s = U² / R²`` is not persisted. Signed reconstruction ``R̂`` is +persisted so ``U + R̂ = R`` stays auditable. ``x`` may be negative when +reconstruction and unexplained leftover have opposite signs. ``e`` is +nonnegative when finite and may exceed 1 when the truncated map +overshoots ``|R|``. """ from __future__ import annotations @@ -59,6 +62,7 @@ class LeftoverPair: leftover_map_unexplained: float | None = None leftover_map_cross_share: float | None = None leftover_map_reconstruction: float | None = None + leftover_map_explained_share: float | None = None @dataclass(frozen=True) @@ -99,11 +103,13 @@ def leftover_pairs_from_residual( expected ``E[Y|θ, item]``. Stored leftover-map rank is the number of Gabriel singular values above the floor. When Gabriel coordinates exist, unexplained leftover ``U = R − R̂`` names the leftover cell - the two-axis map does not reconstruct, and leftover-map cross share + the two-axis map does not reconstruct, leftover-map cross share ``x = 2 R̂ U / R²`` names the identity remainder of raw residual ``R`` after two-axis reconstruction ``R̂ = ξ_{1:2} · ζ_{1:2}`` and - unexplained leftover ``U = R − R̂``. Signed ``R̂`` is persisted with - ``U`` so their raw-residual identity stays auditable. Without a complete-case map there is no pair + unexplained leftover ``U = R − R̂``, and explained leftover share + ``e = R̂² / R²`` names the truncated-map share of that same residual. + Signed ``R̂`` is persisted with ``U`` so their raw-residual identity + stays auditable. Without a complete-case map there is no pair to name (ADR 0168); the caller reads coverage counts instead of a center-distance stand-in pair. """ @@ -156,7 +162,7 @@ def leftover_map_from_residual( candidates: list[ tuple[ float, str, str, float, float, float, - float | None, float | None, float | None, + float | None, float | None, float | None, float | None, ] ] = [] if person_pos is not None and item_pos is not None: @@ -180,6 +186,7 @@ def leftover_map_from_residual( residual_cell = float(residual[person, item]) unexplained = _unexplained_leftover(residual_cell, reconstruction) share = _leftover_map_cross_share(residual_cell, reconstruction) + explained = _leftover_map_explained_share(residual_cell, reconstruction) candidates.append( _candidate_row( post_ids, @@ -193,6 +200,7 @@ def leftover_map_from_residual( unexplained, share, reconstruction if np.isfinite(reconstruction) else None, + explained, ) ) if not candidates: @@ -219,6 +227,25 @@ def _unexplained_leftover(residual: float, reconstruction: float) -> float | Non return float(unexplained) +def _leftover_map_explained_share(residual: float, reconstruction: float) -> float | None: + """Return ``e = R̂² / R²`` when both terms are finite; otherwise omit. + + Truncated two-axis reconstruction of a higher-rank cell can make + ``|R̂| > |R|``, so a finite explained share may exceed 1. Do not + clamp. Origin cells ``R = R̂ = 0`` store ``0.0``, matching cross + share. A zero residual with a nonzero reconstruction is omitted + rather than invented. + """ + if not np.isfinite(residual) or not np.isfinite(reconstruction): + return None + if abs(residual) > _LEFTOVER_SINGULAR_FLOOR: + share = float((reconstruction * reconstruction) / (residual * residual)) + return share if np.isfinite(share) else None + if abs(reconstruction) <= _LEFTOVER_SINGULAR_FLOOR: + return 0.0 + return None + + def _leftover_map_cross_share(residual: float, reconstruction: float) -> float | None: """Return ``x = 2 R̂ U / R²`` when both terms are finite; otherwise omit. @@ -256,11 +283,12 @@ def _candidate_row( leftover_map_unexplained: float | None, leftover_map_cross_share: float | None, leftover_map_reconstruction: float | None, + leftover_map_explained_share: float | None, ) -> tuple[ float, str, str, float, float, float, - float | None, float | None, float | None, + float | None, float | None, float | None, float | None, ]: - """One observed cell: distance, ids, residual, Y, E, U, cross share, R̂.""" + """One observed cell: distance, ids, residual, Y, E, U, cross share, R̂, e.""" leftover_residual = float(residual[person, item]) observed_response = float(matrix[person, item]) expected_response = float(expected[person, item]) @@ -276,6 +304,7 @@ def _candidate_row( leftover_map_unexplained, leftover_map_cross_share, leftover_map_reconstruction, + leftover_map_explained_share, ) @@ -283,7 +312,7 @@ def _pair_from_candidate( pair_kind: str, row: tuple[ float, str, str, float, float, float, - float | None, float | None, float | None, + float | None, float | None, float | None, float | None, ], leftover_map_rank: int, ) -> LeftoverPair: @@ -302,6 +331,7 @@ def _pair_from_candidate( leftover_map_unexplained=row[6], leftover_map_cross_share=row[7], leftover_map_reconstruction=row[8], + leftover_map_explained_share=row[9], ) @@ -438,8 +468,8 @@ def _pad_map_axes(positions: np.ndarray) -> np.ndarray: Unused axes pad with zero rather than inventing a second component. Hidden SVD axes after the second are dropped so reconstruction is ``ξ_{1:2} · ζ_{1:2}``, not the full-rank inner product. That - reconstruction is persisted with unexplained leftover and cross share so - the raw-residual identity remains auditable. + reconstruction is persisted with unexplained leftover, cross share, and + explained leftover share so the raw-residual identity remains auditable. """ padded = np.zeros((positions.shape[0], _LEFTOVER_MAP_AXES), dtype=np.float64) width = min(_LEFTOVER_MAP_AXES, positions.shape[1]) diff --git a/migrations/0232_report_leftover_map_explained_share.sql b/migrations/0232_report_leftover_map_explained_share.sql new file mode 100644 index 000000000..3e7e2e12c --- /dev/null +++ b/migrations/0232_report_leftover_map_explained_share.sql @@ -0,0 +1,14 @@ +-- ADR 0232: persist leftover-map explained share e = R̂² / R² of raw +-- residual after two-axis leftover-map reconstruction +-- (R̂ = ξ_{1:2} · ζ_{1:2}). Distance stays Euclidean leftover-map d. +-- This migration adds only the explained-share column. Upgrade column +-- is nullable so older leftover rows keep distance, residual, +-- unexplained leftover, cross share, and reconstruction without +-- fabricating a share. This migration is the single source of the +-- column on fresh and existing installations. Do not edit shipped +-- migrations 0001 / 0012 after the fact. Do not persist +-- leftover_map_unexplained_share. Do not add a unit or nonnegative +-- CHECK: truncated two-axis reconstruction can make |R̂| > |R|. + +alter table report_leftover_pair + add column if not exists leftover_map_explained_share numeric; diff --git a/migrations/rollback/0232_report_leftover_map_explained_share.sql b/migrations/rollback/0232_report_leftover_map_explained_share.sql new file mode 100644 index 000000000..19216ef8e --- /dev/null +++ b/migrations/rollback/0232_report_leftover_map_explained_share.sql @@ -0,0 +1,5 @@ +-- Reverse 0232. Leftover distance, residual, unexplained leftover, +-- cross share, and reconstruction stay on the pair row. + +alter table report_leftover_pair + drop column if exists leftover_map_explained_share; diff --git a/pyproject.toml b/pyproject.toml index e98205768..8a1288a69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lineageweave" -version = "2.18.0" +version = "2.19.0" description = "Reconstructs git-branch-style lineage DAGs from scattered short records using multi-channel score fusion and LLM adjudication." readme = "README.md" license = { text = "MIT" } diff --git a/scripts/seed_demo_data.py b/scripts/seed_demo_data.py index 53a57e440..f213c55d1 100644 --- a/scripts/seed_demo_data.py +++ b/scripts/seed_demo_data.py @@ -127,6 +127,7 @@ def seed( cur.execute((migrations / "0182_report_leftover_map_unexplained.sql").read_text()) cur.execute((migrations / "0185_report_leftover_map_cross_share.sql").read_text()) cur.execute((migrations / "0206_report_leftover_map_reconstruction.sql").read_text()) + cur.execute((migrations / "0232_report_leftover_map_explained_share.sql").read_text()) cur.execute((migrations / "0060_role_responsibility_agent_type.sql").read_text()) cur.execute((migrations / "0013_person_job_title.sql").read_text()) cur.execute((migrations / "0014_role_responsibility_team_actor_type.sql").read_text()) @@ -1400,8 +1401,8 @@ def _persist_seed_period_report( "pair_kind, post_id, criterion_code, leftover_distance, leftover_residual, " "observed_response, expected_response, leftover_map_rank, " "leftover_map_unexplained, leftover_map_cross_share, " - "leftover_map_reconstruction" - ") values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + "leftover_map_reconstruction, leftover_map_explained_share" + ") values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", ( grouping_kind, grouping_key, @@ -1418,6 +1419,7 @@ def _persist_seed_period_report( pair.leftover_map_unexplained, pair.leftover_map_cross_share, pair.leftover_map_reconstruction, + pair.leftover_map_explained_share, ), ) for axis in report.leftover_map_axes: diff --git a/tests/test_leftover_pairs.py b/tests/test_leftover_pairs.py index a1080e172..db349b169 100644 --- a/tests/test_leftover_pairs.py +++ b/tests/test_leftover_pairs.py @@ -1,7 +1,7 @@ """Leftover post–criterion pairs after the main-effect IRT. Covers ADR 0048 as amended by ADR 0119, ADR 0148, ADR 0163, ADR 0164, -ADR 0182, and ADR 0185. +ADR 0182, ADR 0185, ADR 0201, and ADR 0232. Uses a constructed residual matrix so the closest and farthest pair are known without calling ``fit_polytomous``. Loads @@ -60,8 +60,8 @@ def _assert_residual_reconciles(pair) -> None: def _assert_never_persists_hidden_shares(pair) -> None: - """The cross-share/reconstruction path never persists unsupported shares.""" - assert not hasattr(pair, "leftover_map_explained_share") + """The explained-share path never persists unsupported unexplained share.""" + assert hasattr(pair, "leftover_map_explained_share") assert not hasattr(pair, "leftover_map_unexplained_share") @@ -115,6 +115,10 @@ def test_leftover_residual_biplot_separates_aligned_and_opposed_cells() -> None: assert farthest.leftover_map_cross_share == pytest.approx(0.0, abs=1e-6) assert closest.leftover_map_reconstruction == pytest.approx(0.0, abs=1e-6) assert farthest.leftover_map_reconstruction == pytest.approx(-2.0, abs=1e-6) + # Closest origin cell (R = 0, R̂ = 0): 0/0 stores 0. + assert closest.leftover_map_explained_share == pytest.approx(0.0, abs=1e-6) + # Rank-1 reconstructed opposed cell: |R̂| = |R| so e = 1. + assert farthest.leftover_map_explained_share == pytest.approx(1.0, abs=1e-6) for pair in pairs: _assert_residual_reconciles(pair) _assert_never_persists_hidden_shares(pair) @@ -149,8 +153,11 @@ def test_zero_residual_still_emits_stable_leftover_pairs() -> None: assert pairs[1].leftover_map_cross_share == pytest.approx(0.0) assert pairs[0].leftover_map_reconstruction == pytest.approx(0.0) assert pairs[1].leftover_map_reconstruction == pytest.approx(0.0) + assert pairs[0].leftover_map_explained_share == pytest.approx(0.0) + assert pairs[1].leftover_map_explained_share == pytest.approx(0.0) for pair in pairs: _assert_residual_reconciles(pair) + _assert_never_persists_hidden_shares(pair) assert pair.leftover_map_rank == 0 coverage = leftover_map_coverage_from_residual(post_ids, item_codes, matrix, expected) assert coverage.map_post_count == 2 @@ -177,6 +184,8 @@ def test_rank_zero_nonzero_constant_residual_keeps_raw_identity() -> None: assert pair.leftover_map_unexplained + pair.leftover_map_reconstruction == pytest.approx( pair.leftover_residual ) + assert pair.leftover_map_explained_share == pytest.approx(0.0) + _assert_never_persists_hidden_shares(pair) def test_partial_observation_does_not_treat_missing_as_zero_residual() -> None: @@ -208,6 +217,8 @@ def test_partial_observation_does_not_treat_missing_as_zero_residual() -> None: assert pair.leftover_map_rank == 1 assert pair.leftover_map_unexplained == pytest.approx(0.0, abs=1e-6) assert pair.leftover_map_cross_share == pytest.approx(0.0, abs=1e-6) + assert pair.leftover_map_explained_share == pytest.approx(1.0, abs=1e-6) + _assert_never_persists_hidden_shares(pair) coverage = leftover_map_coverage_from_residual(post_ids, item_codes, matrix, expected) assert coverage.map_post_count == 2 assert coverage.scored_post_count == 3 @@ -281,6 +292,7 @@ def test_leftover_residual_rejects_database_tolerance_boundary() -> None: None, None, None, + None, ) @@ -328,7 +340,9 @@ def test_rank_one_nonzero_center_is_disclosed_by_raw_residual_cross_share() -> N residual = pair.leftover_residual recon = float(reconstruction[post_index[pair.post_id], item_index[pair.criterion_code]]) expected_share = 0.0 if residual == 0.0 and recon == 0.0 else 2.0 * recon * (residual - recon) / residual**2 + expected_explained = 0.0 if residual == 0.0 and recon == 0.0 else (recon * recon) / residual**2 assert pair.leftover_map_cross_share == pytest.approx(expected_share, abs=1e-6) + assert pair.leftover_map_explained_share == pytest.approx(expected_explained, abs=1e-6) assert farthest.leftover_residual != pytest.approx(0.0) assert farthest.leftover_map_cross_share != pytest.approx(farthest.leftover_residual) for pair in pairs: @@ -490,7 +504,11 @@ def test_unexplained_and_cross_share_are_identity_remainder_terms() -> None: explained_share = (recon * recon) / (residual * residual) unexplained_share = (expected_unexplained * expected_unexplained) / (residual * residual) assert pair.leftover_map_cross_share == pytest.approx(expected_share) + assert pair.leftover_map_explained_share == pytest.approx(explained_share) assert explained_share + unexplained_share + expected_share == pytest.approx(1.0) + if abs(residual) > 1e-12: + s = (pair.leftover_map_unexplained ** 2) / (residual * residual) + assert pair.leftover_map_explained_share + s + pair.leftover_map_cross_share == pytest.approx(1.0) if abs(expected_share) > 1e-6: saw_nonzero_cross = True assert pair.leftover_map_cross_share != pytest.approx(pair.leftover_residual) @@ -513,6 +531,16 @@ def test_cross_share_stores_negative_finite_identity_remainder() -> None: assert leftover._leftover_map_cross_share(1.0, float("inf")) is None +def test_explained_share_stores_truncated_map_share_of_raw_residual() -> None: + """Explained leftover share is R̂² / R², never clamped or invented.""" + assert leftover._leftover_map_explained_share(1.0, 2.0) == pytest.approx(4.0) + assert leftover._leftover_map_explained_share(2.0, 2.0) == pytest.approx(1.0) + assert leftover._leftover_map_explained_share(0.0, 0.0) == pytest.approx(0.0) + assert leftover._leftover_map_explained_share(0.0, 1.0) is None + assert leftover._leftover_map_explained_share(float("nan"), 1.0) is None + assert leftover._leftover_map_explained_share(1.0, float("inf")) is None + + def test_pad_map_axes_truncates_hidden_svd_components() -> None: """Axes after the second leftover-map axis do not enter reconstruction.""" padded = leftover._pad_map_axes(np.array([[1.0, 2.0, 9.0]], dtype=np.float64)) diff --git a/tests/test_migration_replay.py b/tests/test_migration_replay.py index 95c5c4fe4..9d4f73ca5 100644 --- a/tests/test_migration_replay.py +++ b/tests/test_migration_replay.py @@ -108,6 +108,29 @@ def test_migrate_sh_replays_leftover_map_axis_migration_on_existing_volumes() -> assert int(migration_name[:4]) >= 12 +def test_migrate_sh_replays_leftover_map_explained_share_on_existing_volumes() -> None: + """migrate.sh's replay window must cover 0232 (leftover-map explained share). + + Volumes created before leftover-map explained share shipped never get + leftover_map_explained_share unless migrate.sh replays 0232 on every + ``docker compose up``. GET /api/reports/{grouping}/{period} then 500s + on undefined_column the first time a period actually has leftover pairs. + + ADR 0166's general four-digit filename boundary covers 0232 without a + per-migration allowlist entry. The column add is nullable and + idempotent so a second start does not invent a leftover score. + """ + migration_name = "0232_report_leftover_map_explained_share.sql" + migration_path = Path(__file__).resolve().parents[1] / "migrations" / migration_name + assert migration_path.exists() + assert re.fullmatch(r"[0-9]{4}_.+\.sql", migration_name) + assert int(migration_name[:4]) >= 12 + sql = migration_path.read_text(encoding="utf-8").casefold() + assert "add column if not exists leftover_map_explained_share" in sql + assert "add column if not exists leftover_map_unexplained_share" not in sql + assert "check (" not in sql + + def test_tenant_settings_migration_is_safe_to_replay() -> None: """The newest migration must survive migrate.sh's every-start replay.""" sql = ( diff --git a/tests/test_period_report.py b/tests/test_period_report.py index 16a42abab..fdb482b9d 100644 --- a/tests/test_period_report.py +++ b/tests/test_period_report.py @@ -276,7 +276,9 @@ def test_calibrated_report_attaches_leftover_pairs() -> None: assert np.isfinite(pair.leftover_map_cross_share) if pair.leftover_map_reconstruction is not None: assert np.isfinite(pair.leftover_map_reconstruction) - assert not hasattr(pair, "leftover_map_explained_share") + if pair.leftover_map_explained_share is not None: + assert np.isfinite(pair.leftover_map_explained_share) + assert hasattr(pair, "leftover_map_explained_share") assert not hasattr(pair, "leftover_map_unexplained_share") assert [axis.axis_index for axis in report.leftover_map_axes] == [1, 2] for axis in report.leftover_map_axes: diff --git a/tests/test_schema.py b/tests/test_schema.py index 49caf6f8c..f3c46f8c6 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -93,6 +93,11 @@ / "migrations" / "0206_report_leftover_map_reconstruction.sql" ) +_LEFTOVER_MAP_EXPLAINED_SHARE_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0232_report_leftover_map_explained_share.sql" +) _LEFTOVER_MAP_AXIS_MIGRATION = ( Path(__file__).resolve().parents[1] / "migrations" @@ -172,6 +177,7 @@ def schema_db(): cur.execute(_LEFTOVER_MAP_UNEXPLAINED_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_CROSS_SHARE_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_RECONSTRUCTION_MIGRATION.read_text()) + cur.execute(_LEFTOVER_MAP_EXPLAINED_SHARE_MIGRATION.read_text()) cur.execute(_SOURCE_EVENT_TIME_MIGRATION.read_text()) # psql sends each statement independently, which is required # by CREATE INDEX CONCURRENTLY. psycopg2 treats a multi- @@ -637,7 +643,7 @@ def test_leftover_pair_names_nullable_cross_share_column(schema_db) -> None: assert columns["leftover_map_cross_share"] == "YES" assert columns["leftover_residual"] == "NO" assert columns["leftover_distance"] == "NO" - assert "leftover_map_explained_share" not in columns + assert columns["leftover_map_explained_share"] == "YES" assert "leftover_map_unexplained_share" not in columns assert columns["leftover_map_reconstruction"] == "YES" with schema_db.cursor() as cur: @@ -652,6 +658,34 @@ def test_leftover_pair_names_nullable_cross_share_column(schema_db) -> None: assert cur.fetchall() == [] +def test_leftover_pair_names_nullable_explained_share_column(schema_db) -> None: + """Every install path preserves legacy pairs while naming leftover-map explained share.""" + with schema_db.cursor() as cur: + cur.execute( + """ + select column_name, is_nullable + from information_schema.columns + where table_name = 'report_leftover_pair' + """ + ) + columns = dict(cur.fetchall()) + assert columns["leftover_map_explained_share"] == "YES" + assert columns["leftover_residual"] == "NO" + assert columns["leftover_distance"] == "NO" + assert "leftover_map_unexplained_share" not in columns + assert columns["leftover_map_reconstruction"] == "YES" + with schema_db.cursor() as cur: + cur.execute( + """ + select conname + from pg_constraint + where conrelid = 'report_leftover_pair'::regclass + and conname like '%explained_share%chk%' + """ + ) + assert cur.fetchall() == [] + + def test_leftover_map_axis_references_period_score(schema_db) -> None: """Axis share is report-level; it must cascade with the period score.""" with schema_db.cursor() as cur: diff --git a/uv.lock b/uv.lock index a29ef2708..59b2a5673 100644 --- a/uv.lock +++ b/uv.lock @@ -685,7 +685,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "2.18.0" +version = "2.19.0" source = { editable = "." } dependencies = [ { name = "certifi" }, From f90f247ac3cda42ac6f3d87ed391b4ca1e3959bd Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 27 Aug 2026 03:02:48 +0900 Subject: [PATCH 2/7] fix(report): align comparison leftover evidence --- backend/app/report_ingestion.py | 8 +++++++- backend/tests/test_api.py | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py index e44285c66..9b2ad2b05 100644 --- a/backend/app/report_ingestion.py +++ b/backend/app/report_ingestion.py @@ -1017,7 +1017,8 @@ async def fetch_period_comparison( f""" select lp.grouping_kind, lp.grouping_key, lp.pair_kind, lp.post_id, lp.criterion_code, lp.leftover_distance, lp.leftover_residual, - lp.leftover_map_reconstruction, lp.leftover_map_explained_share, + lp.leftover_map_cross_share, lp.leftover_map_reconstruction, + lp.leftover_map_explained_share, p.post_title, p.visibility_code, p.corporate_entity_id, ({_SOURCE_CONTEXT_PRESENT_SQL}) as has_real_source_context from report_leftover_pair lp @@ -1072,6 +1073,11 @@ async def fetch_period_comparison( if pair["leftover_map_reconstruction"] is None else float(pair["leftover_map_reconstruction"]) ), + "leftover_map_cross_share": ( + None + if pair["leftover_map_cross_share"] is None + else float(pair["leftover_map_cross_share"]) + ), "leftover_map_explained_share": ( None if pair["leftover_map_explained_share"] is None diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 628ab239a..18831f4f0 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -5777,6 +5777,14 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token, or isinstance(pair["leftover_map_reconstruction"], (int, float)) for pair in leftover_thread.get("leftover_pairs", []) ) + assert all( + "leftover_map_cross_share" in pair + and ( + pair["leftover_map_cross_share"] is None + or isinstance(pair["leftover_map_cross_share"], (int, float)) + ) + for pair in leftover_thread.get("leftover_pairs", []) + ) assert all( "leftover_map_explained_share" in pair and ( From 49885f6019246eb23cc93a31b2f9aa9c059faad8 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 27 Aug 2026 03:36:18 +0900 Subject: [PATCH 3/7] fix(report): project Rust explained share --- backend/tests/test_api.py | 2 +- docs/adr/0232-leftover-map-explained-share.md | 14 +++++--- lineageweave/leftover_pairs.py | 22 ++++++++++-- ...32_report_leftover_map_explained_share.sql | 14 -------- ...36_report_leftover_map_explained_share.sql | 8 +++++ ...32_report_leftover_map_explained_share.sql | 5 --- ...36_report_leftover_map_explained_share.sql | 4 +++ pyproject.toml | 2 +- scripts/seed_demo_data.py | 2 +- tests/test_leftover_pairs.py | 34 +++++++++++++++++++ tests/test_migration_replay.py | 4 +-- tests/test_schema.py | 2 +- uv.lock | 4 +-- 13 files changed, 84 insertions(+), 33 deletions(-) delete mode 100644 migrations/0232_report_leftover_map_explained_share.sql create mode 100644 migrations/0236_report_leftover_map_explained_share.sql delete mode 100644 migrations/rollback/0232_report_leftover_map_explained_share.sql create mode 100644 migrations/rollback/0236_report_leftover_map_explained_share.sql diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index df6f4ad2c..0b845b39e 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -187,7 +187,7 @@ _LEFTOVER_MAP_EXPLAINED_SHARE_MIGRATION = ( Path(__file__).resolve().parents[2] / "migrations" - / "0232_report_leftover_map_explained_share.sql" + / "0236_report_leftover_map_explained_share.sql" ) _GLOBAL_ASK_JOB_MIGRATION = ( Path(__file__).resolve().parents[2] diff --git a/docs/adr/0232-leftover-map-explained-share.md b/docs/adr/0232-leftover-map-explained-share.md index 5e6fc971a..42d5d82d7 100644 --- a/docs/adr/0232-leftover-map-explained-share.md +++ b/docs/adr/0232-leftover-map-explained-share.md @@ -32,9 +32,10 @@ leftover criterion. Leftover-map distance stays two-axis Euclidean. Reconstruction `R̂` remains the ADR 0201 internal two-axis inner product already used for `U` and `x`. -The unprotected-stack reconstructions for neighbouring leftover facts -use 0162–0186. This protected-main increment uses **0232** so it does -not collide with leftover-map reconstruction (0201 / migration 0206), +The feature is stacked over the Rust-boundary work in ADR 0208. The decision +number remains 0232, while the schema increment uses migration **0236** so it +does not collide with the parent stack's migrations. It also does not collide +with leftover-map reconstruction (0201 / migration 0206), leftover-map cross share (0185), leftover-map unexplained leftover (0182), leftover-map unexplained leftover share, leftover residual disclosure, leftover observed `Y` / expected `E`, leftover-map rank, @@ -44,9 +45,14 @@ token-backed status notice (0214 on an open stack). ## Decision +fast-mlsirm's Rust `residual_interaction_map` computes the reconstruction, +unexplained residual, cross share, and explained share in one result envelope. +LineageWeave only binds those returned cells to authorized product identifiers; +it never recalculates these quantities in Python. + Each leftover pair names `leftover_map_explained_share` — leftover-map explained share `e = R̂² / R²` of raw residual after two-axis Gabriel -reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}`. Migration `0232` is the +reconstruction `R̂ = ξ_{1:2} · ζ_{1:2}`. Migration `0236` is the single source of the column on every install path, fresh or existing -- shipped migrations (`0001` / `0012`) are never edited after the fact. The column is nullable so older leftover rows keep distance, residual, diff --git a/lineageweave/leftover_pairs.py b/lineageweave/leftover_pairs.py index 4d91151a3..bf93325bb 100644 --- a/lineageweave/leftover_pairs.py +++ b/lineageweave/leftover_pairs.py @@ -98,13 +98,27 @@ def leftover_map_from_residual( return (), () rank = int(len(result.singular_values)) - candidates: list[tuple[float, str, str, float, float, float, float, float | None, float]] = [] + candidates: list[ + tuple[ + float, + str, + str, + float, + float, + float, + float, + float | None, + float, + float | None, + ] + ] = [] for local_person, person in enumerate(result.person_indices): for local_item, item in enumerate(result.item_indices): observed = float(matrix[int(person), int(item)]) model_expected = float(expected[int(person), int(item)]) residual = float(result.residual[local_person, local_item]) cross_share = float(result.cross_share[local_person, local_item]) + explained_share = float(result.explained_share[local_person, local_item]) candidates.append( ( float(result.distance[local_person, local_item]), @@ -116,6 +130,7 @@ def leftover_map_from_residual( float(result.unexplained[local_person, local_item]), cross_share if np.isfinite(cross_share) else None, float(result.reconstruction[local_person, local_item]), + explained_share if np.isfinite(explained_share) else None, ) ) closest = min(candidates, key=lambda row: (row[0], row[1], row[2])) @@ -166,7 +181,9 @@ def _validate_shapes( def _pair( kind: str, - row: tuple[float, str, str, float, float, float, float, float | None, float], + row: tuple[ + float, str, str, float, float, float, float, float | None, float, float | None + ], rank: int, ) -> LeftoverPair: """Attach product identifiers to one Rust-computed candidate cell.""" @@ -183,4 +200,5 @@ def _pair( leftover_map_unexplained=row[6], leftover_map_cross_share=row[7], leftover_map_reconstruction=row[8], + leftover_map_explained_share=row[9], ) diff --git a/migrations/0232_report_leftover_map_explained_share.sql b/migrations/0232_report_leftover_map_explained_share.sql deleted file mode 100644 index 3e7e2e12c..000000000 --- a/migrations/0232_report_leftover_map_explained_share.sql +++ /dev/null @@ -1,14 +0,0 @@ --- ADR 0232: persist leftover-map explained share e = R̂² / R² of raw --- residual after two-axis leftover-map reconstruction --- (R̂ = ξ_{1:2} · ζ_{1:2}). Distance stays Euclidean leftover-map d. --- This migration adds only the explained-share column. Upgrade column --- is nullable so older leftover rows keep distance, residual, --- unexplained leftover, cross share, and reconstruction without --- fabricating a share. This migration is the single source of the --- column on fresh and existing installations. Do not edit shipped --- migrations 0001 / 0012 after the fact. Do not persist --- leftover_map_unexplained_share. Do not add a unit or nonnegative --- CHECK: truncated two-axis reconstruction can make |R̂| > |R|. - -alter table report_leftover_pair - add column if not exists leftover_map_explained_share numeric; diff --git a/migrations/0236_report_leftover_map_explained_share.sql b/migrations/0236_report_leftover_map_explained_share.sql new file mode 100644 index 000000000..a4be59896 --- /dev/null +++ b/migrations/0236_report_leftover_map_explained_share.sql @@ -0,0 +1,8 @@ +-- ADR 0232: persist leftover-map explained share e = Rhat^2 / R^2 of raw +-- residual after the Rust-owned two-axis leftover-map reconstruction. +-- Distance stays Euclidean leftover-map d. Upgrade column is nullable so +-- older rows retain their evidence without fabricating a share. Do not add a +-- unit or nonnegative CHECK: truncated reconstruction can make |Rhat| > |R|. + +alter table report_leftover_pair + add column if not exists leftover_map_explained_share numeric; diff --git a/migrations/rollback/0232_report_leftover_map_explained_share.sql b/migrations/rollback/0232_report_leftover_map_explained_share.sql deleted file mode 100644 index 19216ef8e..000000000 --- a/migrations/rollback/0232_report_leftover_map_explained_share.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Reverse 0232. Leftover distance, residual, unexplained leftover, --- cross share, and reconstruction stay on the pair row. - -alter table report_leftover_pair - drop column if exists leftover_map_explained_share; diff --git a/migrations/rollback/0236_report_leftover_map_explained_share.sql b/migrations/rollback/0236_report_leftover_map_explained_share.sql new file mode 100644 index 000000000..0c0bae39c --- /dev/null +++ b/migrations/rollback/0236_report_leftover_map_explained_share.sql @@ -0,0 +1,4 @@ +-- Reverse migration 0236. Other leftover-map evidence remains available. + +alter table report_leftover_pair + drop column if exists leftover_map_explained_share; diff --git a/pyproject.toml b/pyproject.toml index 4bd616fe4..7a59a2644 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ backend = [ # Owner arithmetic from protected main (ADR 0208). Product-specific lineage # contracts remain unavailable until a domain-neutral owner contract lands. # PyO3/maturin source builds need the pinned Rust toolchain. - "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@09f762ded35786dd1078222a4577ff09d649816f", + "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@b88012711d6a36e5a0972afb640cddbf655ba925", ] [tool.setuptools.packages.find] diff --git a/scripts/seed_demo_data.py b/scripts/seed_demo_data.py index deebdc571..3e807fca2 100644 --- a/scripts/seed_demo_data.py +++ b/scripts/seed_demo_data.py @@ -129,7 +129,7 @@ def seed( cur.execute((migrations / "0182_report_leftover_map_unexplained.sql").read_text()) cur.execute((migrations / "0185_report_leftover_map_cross_share.sql").read_text()) cur.execute((migrations / "0206_report_leftover_map_reconstruction.sql").read_text()) - cur.execute((migrations / "0232_report_leftover_map_explained_share.sql").read_text()) + cur.execute((migrations / "0236_report_leftover_map_explained_share.sql").read_text()) cur.execute((migrations / "0060_role_responsibility_agent_type.sql").read_text()) cur.execute((migrations / "0013_person_job_title.sql").read_text()) cur.execute((migrations / "0014_role_responsibility_team_actor_type.sql").read_text()) diff --git a/tests/test_leftover_pairs.py b/tests/test_leftover_pairs.py index dfd0eec61..a79701711 100644 --- a/tests/test_leftover_pairs.py +++ b/tests/test_leftover_pairs.py @@ -3,6 +3,7 @@ from __future__ import annotations import inspect +from types import SimpleNamespace import numpy as np import pytest @@ -37,6 +38,39 @@ def test_rust_map_projects_pairs_axes_and_coverage() -> None: ) assert pair.leftover_map_unexplained is not None assert pair.leftover_map_reconstruction is not None + assert pair.leftover_map_explained_share is not None + + +def test_explained_share_is_projected_from_rust_without_recomputation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An upstream value is preserved even when local inputs imply another value.""" + + native_result = SimpleNamespace( + person_indices=np.array([0]), + item_indices=np.array([0]), + scored_person_count=1, + scored_item_count=1, + person_coordinates=np.zeros((1, 2)), + item_coordinates=np.zeros((1, 2)), + singular_values=np.array([1.0]), + axis_shares=np.array([1.0, 0.0]), + residual=np.array([[4.0]]), + distance=np.array([[0.0]]), + reconstruction=np.array([[2.0]]), + explained_share=np.array([[0.375]]), + unexplained=np.array([[2.0]]), + cross_share=np.array([[0.5]]), + ) + monkeypatch.setattr( + leftover, "residual_interaction_map", lambda *_args, **_kwargs: native_result + ) + + pairs, _axes = leftover.leftover_map_from_residual( + ["post-a"], ("item-a",), np.array([[9.0]]), np.array([[1.0]]) + ) + + assert [pair.leftover_map_explained_share for pair in pairs] == [0.375, 0.375] def test_rank_zero_keeps_deterministic_pairs_without_inventing_an_axis() -> None: diff --git a/tests/test_migration_replay.py b/tests/test_migration_replay.py index 94d9fab78..f7a2d0eeb 100644 --- a/tests/test_migration_replay.py +++ b/tests/test_migration_replay.py @@ -112,7 +112,7 @@ def test_migrate_sh_replays_leftover_map_explained_share_on_existing_volumes() - """migrate.sh's replay window must cover 0232 (leftover-map explained share). Volumes created before leftover-map explained share shipped never get - leftover_map_explained_share unless migrate.sh replays 0232 on every + leftover_map_explained_share unless migrate.sh replays 0236 on every ``docker compose up``. GET /api/reports/{grouping}/{period} then 500s on undefined_column the first time a period actually has leftover pairs. @@ -120,7 +120,7 @@ def test_migrate_sh_replays_leftover_map_explained_share_on_existing_volumes() - per-migration allowlist entry. The column add is nullable and idempotent so a second start does not invent a leftover score. """ - migration_name = "0232_report_leftover_map_explained_share.sql" + migration_name = "0236_report_leftover_map_explained_share.sql" migration_path = Path(__file__).resolve().parents[1] / "migrations" / migration_name assert migration_path.exists() assert re.fullmatch(r"[0-9]{4}_.+\.sql", migration_name) diff --git a/tests/test_schema.py b/tests/test_schema.py index ced0f2c00..780644b1d 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -100,7 +100,7 @@ _LEFTOVER_MAP_EXPLAINED_SHARE_MIGRATION = ( Path(__file__).resolve().parents[1] / "migrations" - / "0232_report_leftover_map_explained_share.sql" + / "0236_report_leftover_map_explained_share.sql" ) _LEFTOVER_MAP_AXIS_MIGRATION = ( Path(__file__).resolve().parents[1] diff --git a/uv.lock b/uv.lock index 2946e2b21..afde7d672 100644 --- a/uv.lock +++ b/uv.lock @@ -484,7 +484,7 @@ wheels = [ [[package]] name = "fast-mlsirm" version = "0.9.1" -source = { git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=09f762ded35786dd1078222a4577ff09d649816f#09f762ded35786dd1078222a4577ff09d649816f" } +source = { git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=b88012711d6a36e5a0972afb640cddbf655ba925#b88012711d6a36e5a0972afb640cddbf655ba925" } dependencies = [ { name = "numpy" }, ] @@ -696,7 +696,7 @@ requires-dist = [ { name = "certifi", specifier = ">=2024.0.0" }, { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.6" }, { name = "cryptography", specifier = ">=42.0" }, - { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=09f762ded35786dd1078222a4577ff09d649816f" }, + { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=b88012711d6a36e5a0972afb640cddbf655ba925" }, { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.141.1" }, { name = "httpx2", marker = "extra == 'dev'", specifier = ">=2.12.0" }, { name = "mcp", marker = "extra == 'backend'", specifier = "==2.0.0" }, From 817288a27583340c81f9296a17a4a2adb2e9f1bb Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 27 Aug 2026 03:37:55 +0900 Subject: [PATCH 4/7] build: pin reviewed interaction-map contract --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7a59a2644..3d1b53230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ backend = [ # Owner arithmetic from protected main (ADR 0208). Product-specific lineage # contracts remain unavailable until a domain-neutral owner contract lands. # PyO3/maturin source builds need the pinned Rust toolchain. - "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@b88012711d6a36e5a0972afb640cddbf655ba925", + "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@54bcdf0b1d8a8e98d2265e05e3c03bd42e8b57c2", ] [tool.setuptools.packages.find] diff --git a/uv.lock b/uv.lock index afde7d672..3004bf819 100644 --- a/uv.lock +++ b/uv.lock @@ -484,7 +484,7 @@ wheels = [ [[package]] name = "fast-mlsirm" version = "0.9.1" -source = { git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=b88012711d6a36e5a0972afb640cddbf655ba925#b88012711d6a36e5a0972afb640cddbf655ba925" } +source = { git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=54bcdf0b1d8a8e98d2265e05e3c03bd42e8b57c2#54bcdf0b1d8a8e98d2265e05e3c03bd42e8b57c2" } dependencies = [ { name = "numpy" }, ] @@ -696,7 +696,7 @@ requires-dist = [ { name = "certifi", specifier = ">=2024.0.0" }, { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.6" }, { name = "cryptography", specifier = ">=42.0" }, - { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=b88012711d6a36e5a0972afb640cddbf655ba925" }, + { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=54bcdf0b1d8a8e98d2265e05e3c03bd42e8b57c2" }, { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.141.1" }, { name = "httpx2", marker = "extra == 'dev'", specifier = ">=2.12.0" }, { name = "mcp", marker = "extra == 'backend'", specifier = "==2.0.0" }, From b0b8d108df25a0061d9ff363586772b20d08950a Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 27 Aug 2026 03:58:25 +0900 Subject: [PATCH 5/7] test(schema): apply explained-share migration --- tests/test_schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_schema.py b/tests/test_schema.py index 780644b1d..bc840f244 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -244,6 +244,7 @@ def schema_db(): cur.execute(_LEFTOVER_MAP_UNEXPLAINED_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_CROSS_SHARE_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_RECONSTRUCTION_MIGRATION.read_text()) + cur.execute(_LEFTOVER_MAP_EXPLAINED_SHARE_MIGRATION.read_text()) cur.execute(_OPERATIONS_CASE_MIGRATION.read_text()) cur.execute(_OPERATIONS_CASE_EVIDENCE_MIGRATION.read_text()) cur.execute(_OPERATIONS_CASE_MISSING_MIGRATION.read_text()) From 131a0a9db80a055578fdad0bc69f751e49c7762a Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 27 Aug 2026 04:11:59 +0900 Subject: [PATCH 6/7] build: pin unified Rust envelope head --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3d1b53230..2a326fd7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ backend = [ # Owner arithmetic from protected main (ADR 0208). Product-specific lineage # contracts remain unavailable until a domain-neutral owner contract lands. # PyO3/maturin source builds need the pinned Rust toolchain. - "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@54bcdf0b1d8a8e98d2265e05e3c03bd42e8b57c2", + "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@2c2688a56ce75adddf795333f20d69eaef18e4e5", ] [tool.setuptools.packages.find] diff --git a/uv.lock b/uv.lock index 3004bf819..acee4b8d4 100644 --- a/uv.lock +++ b/uv.lock @@ -484,7 +484,7 @@ wheels = [ [[package]] name = "fast-mlsirm" version = "0.9.1" -source = { git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=54bcdf0b1d8a8e98d2265e05e3c03bd42e8b57c2#54bcdf0b1d8a8e98d2265e05e3c03bd42e8b57c2" } +source = { git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=2c2688a56ce75adddf795333f20d69eaef18e4e5#2c2688a56ce75adddf795333f20d69eaef18e4e5" } dependencies = [ { name = "numpy" }, ] @@ -696,7 +696,7 @@ requires-dist = [ { name = "certifi", specifier = ">=2024.0.0" }, { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.6" }, { name = "cryptography", specifier = ">=42.0" }, - { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=54bcdf0b1d8a8e98d2265e05e3c03bd42e8b57c2" }, + { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=2c2688a56ce75adddf795333f20d69eaef18e4e5" }, { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.141.1" }, { name = "httpx2", marker = "extra == 'dev'", specifier = ">=2.12.0" }, { name = "mcp", marker = "extra == 'backend'", specifier = "==2.0.0" }, From e52a82729fa85a1480d47a953254782390ee37a3 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 27 Aug 2026 04:12:44 +0900 Subject: [PATCH 7/7] build: pin envelope parity proof --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2a326fd7c..117082808 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ backend = [ # Owner arithmetic from protected main (ADR 0208). Product-specific lineage # contracts remain unavailable until a domain-neutral owner contract lands. # PyO3/maturin source builds need the pinned Rust toolchain. - "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@2c2688a56ce75adddf795333f20d69eaef18e4e5", + "fast-mlsirm @ git+https://github.com/ContextualWisdomLab/fast-mlsirm.git@5c7dc9ead1feddcaff0d6c9032fcace4c7028f7c", ] [tool.setuptools.packages.find] diff --git a/uv.lock b/uv.lock index acee4b8d4..b4acf9668 100644 --- a/uv.lock +++ b/uv.lock @@ -484,7 +484,7 @@ wheels = [ [[package]] name = "fast-mlsirm" version = "0.9.1" -source = { git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=2c2688a56ce75adddf795333f20d69eaef18e4e5#2c2688a56ce75adddf795333f20d69eaef18e4e5" } +source = { git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=5c7dc9ead1feddcaff0d6c9032fcace4c7028f7c#5c7dc9ead1feddcaff0d6c9032fcace4c7028f7c" } dependencies = [ { name = "numpy" }, ] @@ -696,7 +696,7 @@ requires-dist = [ { name = "certifi", specifier = ">=2024.0.0" }, { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.6" }, { name = "cryptography", specifier = ">=42.0" }, - { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=2c2688a56ce75adddf795333f20d69eaef18e4e5" }, + { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=5c7dc9ead1feddcaff0d6c9032fcace4c7028f7c" }, { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.141.1" }, { name = "httpx2", marker = "extra == 'dev'", specifier = ">=2.12.0" }, { name = "mcp", marker = "extra == 'backend'", specifier = "==2.0.0" },