diff --git a/AGENTS.md b/AGENTS.md
index 67ea919a8..06999f370 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -209,7 +209,9 @@ list so a click opens that post. Two-axis reconstruction `R̂` is not
persisted. Leftover-map axis share (ADR 0148) is Gabriel inertia of
residual SVD axes 1 and 2 and persists to `report_leftover_map_axis`.
Rank-0 residuals emit two zero-share axes; the shares are report-level
-and are not a leftover score.
+and are not a leftover score. Complete-case coverage (ADR 0168) persists to
+`report_leftover_map_coverage` and captions the pair list with how
+many scored posts entered the map.
`frontend/` has its own toolchain (Node pinned via `frontend/mise.toml`,
pnpm via Corepack -- do not add a second Node package manager or a
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index e7ac85c9c..235f2a216 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -601,7 +601,9 @@ ADR 0017 / 0048 / 0119 / 0162 / 0163 / 0164 / 0182) persist to
`E[Y|θ, item]`, full leftover-map rank, and unexplained leftover
`U = R − R̂` named on the pair row. Leftover-map axis share (Gabriel
inertia of residual SVD axes 1 and 2; ADR 0148) persists to
-`report_leftover_map_axis`. Results persist to
+`report_leftover_map_axis`. Complete-case leftover-map coverage (ADR
+0168) persists to `report_leftover_map_coverage` so readers see how
+many scored posts entered the factorization. Results persist to
`report_period_score` / `report_member_score`.
`GET /api/reports/{grouping}` lists the trend;
`GET /api/reports/{grouping}/{period}` is ABAC-filtered;
@@ -616,7 +618,8 @@ the actual mean θ, the FIPC delta, the CAT-selected item, leftover
closest/farthest pairs (signed residual `R`, observed `Y`, expected
`E`, full rank, and two-axis leftover-map distance `d` after IRT main
effects) above the member list, leftover-map axis share for residual
-SVD axes 1 and 2, and the
+SVD axes 1 and 2, and complete-case coverage captions (map used N of M
+scored posts), plus the
PU / corp / thread comparison -- never a placeholder. TEPP is unchanged.
## Phase 6b: Knowledge Graph as a real Ontology + Semantic Layer
diff --git a/CHANGELOG.d/2.12.17-leftover-map-coverage.md b/CHANGELOG.d/2.12.17-leftover-map-coverage.md
new file mode 100644
index 000000000..02c6858b3
--- /dev/null
+++ b/CHANGELOG.d/2.12.17-leftover-map-coverage.md
@@ -0,0 +1,11 @@
+# 2.12.17 — Leftover map complete-case coverage
+
+## Added
+
+- Period leftover maps now persist how many scored posts entered the
+ complete-case Gabriel factorization (ADR 0168). Missing cells stay
+ out of the map; they are never treated as zero.
+- After seed, the leftover pair list is captioned
+ “Leftover map used N of M scored posts (complete-case)” so a sparse
+ post that was scored but excluded is visible as coverage, not as a
+ fabricated pair.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b2da918ee..b2040d64c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -37,6 +37,11 @@ All notable changes to this project are documented here. Format follows
compatibility vocabulary after validating every mapping's term kind.
- The PROV-O support profile now mints its product class mappings only in the
canonical lowercase namespace while importing the legacy compatibility map.
+- Period leftover maps now persist how many scored posts entered the
+ complete-case Gabriel factorization (ADR 0168). The leftover pair
+ list is captioned “Leftover map used N of M scored posts
+ (complete-case)”; incomplete rows stay excluded, never filled with
+ zero.
### Fixed
diff --git a/backend/app/report_ingestion.py b/backend/app/report_ingestion.py
index deca8d3d7..a00350c81 100644
--- a/backend/app/report_ingestion.py
+++ b/backend/app/report_ingestion.py
@@ -351,7 +351,7 @@ async def persist_period_report(
period_code: str,
report: PeriodReport,
) -> None:
- """Replace the stored report, member scores, leftover pairs, leftover-map axes, and item bank."""
+ """Replace the stored report, member scores, leftover pairs, leftover-map axes, leftover coverage, and item bank."""
await conn.execute(
"""
delete from report_period_score
@@ -478,6 +478,27 @@ async def persist_period_report(
axis.leftover_singular_value,
axis.leftover_share,
)
+ if report.leftover_map_coverage is not None:
+ coverage = report.leftover_map_coverage
+ await conn.execute(
+ """
+ insert into report_leftover_map_coverage (
+ grouping_kind, grouping_key, period_code, rubric_version,
+ map_post_count, scored_post_count, map_item_count, scored_item_count,
+ incomplete_post_count, incomplete_item_count
+ ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
+ """,
+ grouping_kind,
+ grouping_key,
+ period_code,
+ RUBRIC_VERSION,
+ coverage.map_post_count,
+ coverage.scored_post_count,
+ coverage.map_item_count,
+ coverage.scored_item_count,
+ coverage.incomplete_post_count,
+ coverage.incomplete_item_count,
+ )
def _groups_from_rows(
@@ -650,6 +671,18 @@ async def fetch_period_reports(
period_code,
RUBRIC_VERSION,
)
+ leftover_coverage = await conn.fetch(
+ """
+ select grouping_key, map_post_count, scored_post_count,
+ map_item_count, scored_item_count,
+ incomplete_post_count, incomplete_item_count
+ from report_leftover_map_coverage
+ where grouping_kind = $1 and period_code = $2 and rubric_version = $3
+ """,
+ grouping_kind,
+ period_code,
+ RUBRIC_VERSION,
+ )
status_labels = await labels_for_codes(
conn,
[row["ticket_status_code"] for row in members if row["ticket_status_code"]],
@@ -666,6 +699,7 @@ async def fetch_period_reports(
leftover_axes_by_group: dict[str, list[asyncpg.Record]] = defaultdict(list)
for row in leftover_axes:
leftover_axes_by_group[row["grouping_key"]].append(row)
+ leftover_coverage_by_group = {row["grouping_key"]: row for row in leftover_coverage}
payload: list[dict[str, Any]] = []
for header in headers:
grouping_key = header["grouping_key"]
@@ -770,11 +804,28 @@ async def fetch_period_reports(
}
for row in leftover_axes_by_group.get(header["grouping_key"], [])
],
+ "leftover_map_coverage": _leftover_map_coverage_payload(
+ leftover_coverage_by_group.get(header["grouping_key"])
+ ),
}
)
return payload
+def _leftover_map_coverage_payload(row: asyncpg.Record | None) -> dict[str, int] | None:
+ """One complete-case leftover-map coverage row, or None when unpersisted."""
+ if row is None:
+ return None
+ return {
+ "map_post_count": int(row["map_post_count"]),
+ "scored_post_count": int(row["scored_post_count"]),
+ "map_item_count": int(row["map_item_count"]),
+ "scored_item_count": int(row["scored_item_count"]),
+ "incomplete_post_count": int(row["incomplete_post_count"]),
+ "incomplete_item_count": int(row["incomplete_item_count"]),
+ }
+
+
async def list_period_report_summaries(
conn: asyncpg.Connection,
grouping_kind: str,
diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py
index 39d3f179e..4cdf0729b 100644
--- a/backend/tests/test_api.py
+++ b/backend/tests/test_api.py
@@ -142,6 +142,11 @@
/ "migrations"
/ "0169_report_leftover_map_axis.sql"
)
+_LEFTOVER_MAP_COVERAGE_MIGRATION = (
+ Path(__file__).resolve().parents[2]
+ / "migrations"
+ / "0168_report_leftover_map_coverage.sql"
+)
_LEFTOVER_MAP_UNEXPLAINED_MIGRATION = (
Path(__file__).resolve().parents[2]
/ "migrations"
@@ -264,6 +269,7 @@ def seeded_db(demo_analyst_token):
cur.execute(_CHANNEL_WEIGHT_MIGRATION.read_text())
cur.execute(_LEFTOVER_OBSERVED_EXPECTED_MIGRATION.read_text())
cur.execute(_LEFTOVER_MAP_RANK_MIGRATION.read_text())
+ cur.execute(_LEFTOVER_MAP_COVERAGE_MIGRATION.read_text())
cur.execute(_GLOBAL_ASK_JOB_MIGRATION.read_text())
cur.execute(_LEFTOVER_MAP_AXIS_MIGRATION.read_text())
cur.execute(_LEFTOVER_MAP_UNEXPLAINED_MIGRATION.read_text())
@@ -5018,6 +5024,13 @@ def test_seed_period_report_surfaces_on_get_reports(client, demo_analyst_token,
assert [axis["axis_index"] for axis in leftover_axes] == [1, 2]
assert all(axis["leftover_singular_value"] >= 0 for axis in leftover_axes)
assert all(0.0 <= axis["leftover_share"] <= 1.0 for axis in leftover_axes)
+ leftover_coverage = high_report.get("leftover_map_coverage")
+ assert leftover_coverage is not None
+ assert leftover_coverage["map_post_count"] <= leftover_coverage["scored_post_count"]
+ assert leftover_coverage["incomplete_post_count"] == (
+ leftover_coverage["scored_post_count"] - leftover_coverage["map_post_count"]
+ )
+ assert leftover_coverage["scored_post_count"] >= 2
week3 = client.get(
"/api/reports/process_unit/2026-W03",
diff --git a/docs/adr/0048-persist-lsirm-leftover-pairs.md b/docs/adr/0048-persist-lsirm-leftover-pairs.md
index fddc028e2..0bab2fc32 100644
--- a/docs/adr/0048-persist-lsirm-leftover-pairs.md
+++ b/docs/adr/0048-persist-lsirm-leftover-pairs.md
@@ -52,7 +52,7 @@ the IRT matrix is unusable. A rank-0 residual still emits a
stable pair so `make seed` is not empty; the stored distance is
then zero, not a fabricated interaction.
-The UI contract is ADR 0049.
+The UI contract is ADR 0049. Complete-case coverage is ADR 0168.
## Consequences
diff --git a/docs/adr/0049-leftover-pair-report-ui.md b/docs/adr/0049-leftover-pair-report-ui.md
index fdae8f2ee..729857277 100644
--- a/docs/adr/0049-leftover-pair-report-ui.md
+++ b/docs/adr/0049-leftover-pair-report-ui.md
@@ -53,4 +53,5 @@ next action, not only the distance.
## Related
Depends on [ADR 0048](0048-persist-lsirm-leftover-pairs.md) and
-[ADR 0003](0003-fast-mlsirm-report-integration.md).
+[ADR 0003](0003-fast-mlsirm-report-integration.md). Complete-case
+coverage of the leftover map is [ADR 0168](0168-leftover-map-complete-case-coverage.md).
diff --git a/docs/adr/0168-leftover-map-complete-case-coverage.md b/docs/adr/0168-leftover-map-complete-case-coverage.md
new file mode 100644
index 000000000..59f4753fd
--- /dev/null
+++ b/docs/adr/0168-leftover-map-complete-case-coverage.md
@@ -0,0 +1,74 @@
+# ADR 0168 — Name leftover complete-case coverage
+
+**Decision status:** Accepted
+**Date:** 2026-08-24
+
+## Context
+
+ADR 0048 already factorizes the leftover residual
+`R = Y − E[Y|θ, item]` on the **complete-case** rectangle (Gabriel,
+1971). Incomplete rows are dropped; missing cells are never filled
+with zero. ADR 0049 then shows closest and farthest pairs above the
+member list.
+
+The period report still does not tell the buyer how many scored
+posts entered that factorization. A sparse post with one missing
+criterion is excluded from the map, so closest/farthest pairs can
+name two posts while three posts were scored. Without a coverage
+caption, the buyer cannot tell whether the map used every scored
+post or dropped incomplete rows.
+
+This slice does not persist leftover-map coordinates (that is a
+separate increment), does not disclose residual `R` on pair rows,
+and does not change pair click-through.
+
+## Decision
+
+After a real GRM/GPCM score, persist one
+`report_leftover_map_coverage` row per period report (3NF,
+two-or-more-word `snake_case`):
+
+- `map_post_count` — posts that entered the complete-case leftover
+ map
+- `scored_post_count` — posts with at least one observed cell
+- `map_item_count` / `scored_item_count` — the same counts for
+ criteria
+- `incomplete_post_count` / `incomplete_item_count` — scored minus
+ map, stored so the buyer fact is durable and check-constrained
+
+Do not store a second theta. Do not fill missing cells with zero.
+Do not invent coverage when the IRT matrix is unusable.
+
+`GET /api/reports/{grouping}/{period}` returns
+`leftover_map_coverage` next to `leftover_pairs`. The Period
+reports panel renders a caption **above** the leftover pair list:
+
+> Leftover map used N of M scored posts (complete-case)
+
+Missing coverage renders nothing. A hidden post never appears as a
+leftover pair (ADR 0049); coverage counts remain the fitted map,
+not a visibility-filtered recount.
+
+## Consequences
+
+Rebuild and seed write coverage in the same transaction as leftover
+pairs. Migration `0168_report_leftover_map_coverage.sql` upgrades
+volumes that already applied `0001`. `migrate.sh` replays `0168_*`
+on existing volumes.
+
+## References
+
+Gabriel, K. R. (1971). The biplot graphic display of matrices with
+application to principal component analysis. *Biometrika, 58*(3),
+453–467. https://doi.org/10.1093/biomet/58.3.453
+
+Jeon, M., Jin, I. H., Schweinberger, M., & Baugh, S. (2021). Mapping
+unobserved item–respondent interactions: A latent space item response
+model with interaction map. *Psychometrika, 86*(2), 378–403.
+https://doi.org/10.1007/s11336-021-09762-5
+
+## Related
+
+Depends on [ADR 0048](0048-persist-lsirm-leftover-pairs.md) and
+[ADR 0049](0049-leftover-pair-report-ui.md). Complements, and does
+not replace, leftover-map coordinates or residual-row disclosure.
diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx
index fc7e03373..7a2faf0ef 100644
--- a/frontend/src/App.test.tsx
+++ b/frontend/src/App.test.tsx
@@ -997,6 +997,14 @@ describe("App, authenticated", () => {
leftover_share: 0.18,
},
],
+ leftover_map_coverage: {
+ map_post_count: 2,
+ scored_post_count: 3,
+ map_item_count: 2,
+ scored_item_count: 2,
+ incomplete_post_count: 1,
+ incomplete_item_count: 0,
+ },
members: [
{
post_id: "post-1",
@@ -3697,6 +3705,10 @@ describe("App, authenticated", () => {
expect(screen.getByRole("button", { name: /open report post: public post/i })).toHaveTextContent("Open");
expect(screen.getByRole("button", { name: /open report post: public post/i })).toHaveTextContent("due 2026-01-12");
expect(screen.getByLabelText("Leftover pairs")).toBeInTheDocument();
+ expect(screen.getByLabelText("Leftover map coverage")).toHaveTextContent(
+ "Leftover map used 2 of 3 scored posts (complete-case)",
+ );
+ const coverageCaption = screen.getByLabelText("Leftover map coverage");
const closestPair = screen.getByRole("button", { name: /open leftover closest pair: public post/i });
const farthestPair = screen.getByRole("button", {
name: /open leftover farthest pair: specification revision requested/i,
@@ -3720,6 +3732,7 @@ describe("App, authenticated", () => {
expect(farthestPair).toHaveTextContent("U −0.25");
expect(farthestPair).toHaveTextContent("d 1.84");
const memberButton = screen.getByRole("button", { name: /open report post: public post/i });
+ expect(coverageCaption.compareDocumentPosition(closestPair) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy();
expect(closestPair.compareDocumentPosition(memberButton) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy();
expect(
screen.getByRole("button", { name: /open report post: specification revision requested/i }),
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 399485c35..8c1d3b27e 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -3457,6 +3457,14 @@ function ReportsPanel({
{report.selected_items[0].information.toFixed(2)}
)}
+ {report.leftover_map_coverage && report.leftover_map_coverage.scored_post_count > 0 && (
+
+ {tf("Leftover map used {used} of {scored} scored posts (complete-case)", {
+ used: report.leftover_map_coverage.map_post_count,
+ scored: report.leftover_map_coverage.scored_post_count,
+ })}
+
+ )}
{report.leftover_map_axes?.map((axis) => (
{tf("leftover axis {axis} {share}%", {
diff --git a/frontend/src/api.ts b/frontend/src/api.ts
index d90c2e246..a951207b9 100644
--- a/frontend/src/api.ts
+++ b/frontend/src/api.ts
@@ -886,6 +886,15 @@ export interface LeftoverMapAxis {
leftover_share: number;
}
+export interface LeftoverMapCoverage {
+ map_post_count: number;
+ scored_post_count: number;
+ map_item_count: number;
+ scored_item_count: number;
+ incomplete_post_count: number;
+ incomplete_item_count: number;
+}
+
export interface PeriodGroupReport {
grouping_key: string;
grouping_label?: string;
@@ -902,6 +911,7 @@ export interface PeriodGroupReport {
selected_items: SelectedReportItem[];
leftover_pairs: LeftoverPair[];
leftover_map_axes?: LeftoverMapAxis[];
+ leftover_map_coverage?: LeftoverMapCoverage | null;
}
export interface PeriodReports {
diff --git a/frontend/src/i18n.ts b/frontend/src/i18n.ts
index 328bcfdc2..55fed1394 100644
--- a/frontend/src/i18n.ts
+++ b/frontend/src/i18n.ts
@@ -197,6 +197,9 @@ const TRANSLATIONS: Partial>> = {
"Choose a post": "글을 선택하세요",
"Analysis runs": "분석 실행",
"Period reports": "기간 리포트",
+ "Leftover map coverage": "잔여 지도 포함 범위",
+ "Leftover map used {used} of {scored} scored posts (complete-case)":
+ "잔여 지도는 채점된 글 {scored}개 중 {used}개를 사용했습니다(완전사례)",
"Event Lineage": "이벤트 계보",
"Related posts": "관련 글",
"Related to": "관련 대상:",
@@ -643,6 +646,9 @@ const TRANSLATIONS: Partial>> = {
"Choose a post": "选择文章",
"Analysis runs": "分析运行",
"Period reports": "周期报告",
+ "Leftover map coverage": "残差地图覆盖范围",
+ "Leftover map used {used} of {scored} scored posts (complete-case)":
+ "残差地图使用了 {scored} 篇已评分帖文中的 {used} 篇（完全案例）",
"Event Lineage": "事件谱系",
"Related posts": "相关文章",
"Related to": "相关对象：",
@@ -1111,6 +1117,9 @@ const TRANSLATIONS: Partial>> = {
"Choose a post": "投稿を選択",
"Analysis runs": "分析実行",
"Period reports": "期間レポート",
+ "Leftover map coverage": "残差マップの対象範囲",
+ "Leftover map used {used} of {scored} scored posts (complete-case)":
+ "残差マップは採点済み投稿 {scored} 件のうち {used} 件を使いました（完全ケース）",
"Event Lineage": "イベント系譜",
"Related posts": "関連する投稿",
"Related to": "関連対象：",
@@ -1556,6 +1565,9 @@ const TRANSLATIONS: Partial>> = {
"Choose a post": "Chọn bài viết",
"Analysis runs": "Lần chạy phân tích",
"Period reports": "Báo cáo theo kỳ",
+ "Leftover map coverage": "Phạm vi bản đồ phần dư",
+ "Leftover map used {used} of {scored} scored posts (complete-case)":
+ "Bản đồ phần dư dùng {used} trên {scored} bài đã chấm (trường hợp đầy đủ)",
"Event Lineage": "Dòng sự kiện",
"Related posts": "Bài viết liên quan",
"Related to": "Liên quan đến:",
diff --git a/lineageweave/leftover_pairs.py b/lineageweave/leftover_pairs.py
index 7531607e9..cb7ae265d 100644
--- a/lineageweave/leftover_pairs.py
+++ b/lineageweave/leftover_pairs.py
@@ -13,11 +13,12 @@
after the second are dropped. Each pair also names the full leftover-map
rank so a rank-0 collapse is not read as leftover structure. Axis share
is the Gabriel inertia of the first two leftover-map axes (ADR 0148).
-Each pair also names unexplained leftover ``U = R − R̂`` after two-axis
-Gabriel reconstruction ``R̂ = ξ_{1:2} · ζ_{1:2}`` so the leftover cell
-the map does not reconstruct is not confused with leftover residual
-``R`` or leftover-map distance ``d``. Reconstruction is computed
-internally and is not persisted.
+Complete-case coverage (ADR 0168) names how many scored posts entered
+that rectangle. Each pair also names unexplained leftover ``U = R − R̂``
+after two-axis Gabriel reconstruction ``R̂ = ξ_{1:2} · ζ_{1:2}`` so the
+leftover cell the map does not reconstruct is not confused with
+leftover residual ``R`` or leftover-map distance ``d``. Reconstruction
+is computed internally and is not persisted.
"""
from __future__ import annotations
@@ -57,6 +58,18 @@ class LeftoverMapAxis:
leftover_share: float
+@dataclass(frozen=True)
+class LeftoverMapCoverage:
+ """Complete-case counts for the leftover interaction map."""
+
+ map_post_count: int
+ scored_post_count: int
+ map_item_count: int
+ scored_item_count: int
+ incomplete_post_count: int
+ incomplete_item_count: int
+
+
def leftover_pairs_from_residual(
post_ids: list[str],
item_codes: tuple[str, ...],
@@ -158,22 +171,10 @@ def leftover_map_from_residual(
)
)
if not candidates:
- leftover_map_rank = 0
- for person, item in observed:
- distance = abs(float(residual[person, item]) - center)
- candidates.append(
- _candidate_row(
- post_ids,
- item_codes,
- matrix,
- expected,
- residual,
- person,
- item,
- max(distance, 0.0),
- None,
- )
- )
+ # ADR 0168: without a complete-case Gabriel map there is no
+ # leftover pair to name. The report carries coverage counts
+ # instead of a center-distance stand-in pair.
+ return (), ()
closest = min(candidates, key=lambda row: (row[0], row[1], row[2]))
farthest = max(candidates, key=lambda row: (row[0], row[1], row[2]))
pairs = (
@@ -268,6 +269,47 @@ def leftover_map_axes_from_singular(singular: np.ndarray) -> tuple[LeftoverMapAx
return tuple(axes)
+def leftover_map_coverage_from_residual(
+ post_ids: list[str],
+ item_codes: tuple[str, ...],
+ matrix: np.ndarray,
+ expected: np.ndarray,
+) -> LeftoverMapCoverage:
+ """Name how many scored posts entered the complete-case leftover map.
+
+ Gabriel (1971) factorizes the complete-case residual rectangle.
+ Missing cells stay out of that rectangle; they are never filled
+ with zero. ``map_post_count`` is the number of posts that entered
+ the factorization. ``scored_post_count`` is posts with at least
+ one observed cell. Incomplete rows are excluded, never zeroed.
+ """
+ if matrix.shape != (len(post_ids), len(item_codes)):
+ raise ValueError(
+ f"matrix shape {matrix.shape} does not match {len(post_ids)} posts × {len(item_codes)} items"
+ )
+ if expected.shape != matrix.shape:
+ raise ValueError(f"expected shape {expected.shape} does not match matrix {matrix.shape}")
+
+ residual = matrix.astype(np.float64) - expected.astype(np.float64)
+ # Identical mask to leftover_map_from_residual's: a cell the pair map
+ # scores must be exactly a cell coverage counts, so map_post_count and
+ # the caption can never drift apart if expected ever goes non-finite.
+ observed_mask = (~np.isnan(matrix)) & np.isfinite(residual) & np.isfinite(expected)
+ scored_post_count = int(observed_mask.any(axis=1).sum())
+ scored_item_count = int(observed_mask.any(axis=0).sum())
+ keep_person, keep_item = _complete_case_masks(observed_mask)
+ map_post_count = int(keep_person.sum())
+ map_item_count = int(keep_item.sum())
+ return LeftoverMapCoverage(
+ map_post_count=map_post_count,
+ scored_post_count=scored_post_count,
+ map_item_count=map_item_count,
+ scored_item_count=scored_item_count,
+ incomplete_post_count=scored_post_count - map_post_count,
+ incomplete_item_count=scored_item_count - map_item_count,
+ )
+
+
def _complete_case_masks(observed: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
"""Drop incomplete rows, then incomplete columns among remaining rows."""
keep_person = observed.any(axis=1)
@@ -276,6 +318,8 @@ def _complete_case_masks(observed: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
keep_person = keep_person & observed[:, keep_item].all(axis=1)
if np.any(keep_person):
keep_item = keep_item & observed[keep_person, :].all(axis=0)
+ else:
+ keep_item = np.zeros_like(keep_item)
return keep_person, keep_item
diff --git a/lineageweave/period_report.py b/lineageweave/period_report.py
index 6dedaf742..04e4eef4f 100644
--- a/lineageweave/period_report.py
+++ b/lineageweave/period_report.py
@@ -22,7 +22,9 @@
positions. Closest / farthest pairs are the min / max Euclidean
distances on that map (Jeon et al., 2021, eq. 3). Leftover-map axis
share is Gabriel inertia ``σ_k² / Σ_j σ_j²`` of residual SVD axes 1
-and 2 (ADR 0148). ``fast-mlsirm`` has no leftover-pair API; this
+and 2 (ADR 0148). Complete-case coverage (ADR 0168) names how many
+scored posts entered the factorization; incomplete rows are excluded,
+never filled with zero. ``fast-mlsirm`` has no leftover-pair API; this
module does not invent a second IRT fit and does not fork LSIRM.
This module is pure compute. Persistence lives in
@@ -44,7 +46,13 @@
validate_irt_response_matrix,
)
-from .leftover_pairs import LeftoverMapAxis, LeftoverPair, leftover_map_from_residual
+from .leftover_pairs import (
+ LeftoverMapAxis,
+ LeftoverMapCoverage,
+ LeftoverPair,
+ leftover_map_coverage_from_residual,
+ leftover_map_from_residual,
+)
from .leftover_pairs import leftover_pairs_from_residual as leftover_pairs_from_residual
from .leftover_pairs import PAIR_KIND_CLOSEST as PAIR_KIND_CLOSEST
from .leftover_pairs import PAIR_KIND_FARTHEST as PAIR_KIND_FARTHEST
@@ -114,6 +122,7 @@ class PeriodReport:
selected_items: tuple[SelectedItem, ...] = ()
leftover_pairs: tuple[LeftoverPair, ...] = ()
leftover_map_axes: tuple[LeftoverMapAxis, ...] = ()
+ leftover_map_coverage: LeftoverMapCoverage | None = None
def _sigmoid(value: np.ndarray) -> np.ndarray:
@@ -273,6 +282,20 @@ def leftover_pairs_for_fit(
return pairs
+def leftover_map_coverage_for_fit(
+ post_ids: list[str],
+ item_codes: tuple[str, ...],
+ matrix: np.ndarray,
+ model: str,
+ theta: np.ndarray,
+ fit: PolytomousFit,
+) -> LeftoverMapCoverage:
+ """Complete-case leftover-map coverage from the fitted main effects."""
+ probs = _category_probabilities(model, theta, fit)
+ expected = expected_category_matrix(matrix, probs)
+ return leftover_map_coverage_from_residual(post_ids, item_codes, matrix, expected)
+
+
def _member_scores(post_ids: list[str], scores: dict[str, np.ndarray]) -> tuple[MemberScore, ...]:
"""Implement the _member_scores operation for this channel."""
theta = np.asarray(scores["theta_eap"], dtype=np.float64)
@@ -325,6 +348,9 @@ def calibrate_period_report(
leftover_pairs, leftover_map_axes = leftover_map_for_fit(
post_ids, item_codes, matrix, selected, theta, fit
)
+ leftover_map_coverage = leftover_map_coverage_for_fit(
+ post_ids, item_codes, matrix, selected, theta, fit
+ )
return PeriodReport(
selected_model=selected,
mean_theta=mean_theta,
@@ -340,6 +366,7 @@ def calibrate_period_report(
selected_items=rank_items_by_information(item_bank, mean_theta),
leftover_pairs=leftover_pairs,
leftover_map_axes=leftover_map_axes,
+ leftover_map_coverage=leftover_map_coverage,
)
@@ -371,6 +398,9 @@ def score_period_on_bank(
leftover_pairs, leftover_map_axes = leftover_map_for_fit(
post_ids, item_bank.item_codes, matrix, item_bank.model, theta, fit
)
+ leftover_map_coverage = leftover_map_coverage_for_fit(
+ post_ids, item_bank.item_codes, matrix, item_bank.model, theta, fit
+ )
return PeriodReport(
selected_model=item_bank.model,
mean_theta=mean_theta,
@@ -392,6 +422,7 @@ def score_period_on_bank(
selected_items=rank_items_by_information(item_bank, mean_theta),
leftover_pairs=leftover_pairs,
leftover_map_axes=leftover_map_axes,
+ leftover_map_coverage=leftover_map_coverage,
)
diff --git a/migrations/0168_report_leftover_map_coverage.sql b/migrations/0168_report_leftover_map_coverage.sql
new file mode 100644
index 000000000..26d61738d
--- /dev/null
+++ b/migrations/0168_report_leftover_map_coverage.sql
@@ -0,0 +1,27 @@
+-- ADR 0168: persist leftover complete-case coverage (map used N of M
+-- scored posts). CREATE IF NOT EXISTS so a volume that already ran
+-- 0001 still upgrades. Incomplete rows stay excluded; missing cells
+-- are never stored as zero.
+
+create table if not exists report_leftover_map_coverage (
+ grouping_kind text not null,
+ grouping_key text not null,
+ period_code text not null,
+ rubric_version text not null,
+ map_post_count integer not null,
+ scored_post_count integer not null,
+ map_item_count integer not null,
+ scored_item_count integer not null,
+ incomplete_post_count integer not null,
+ incomplete_item_count integer not null,
+ primary key (grouping_kind, grouping_key, period_code, rubric_version),
+ foreign key (grouping_kind, grouping_key, period_code, rubric_version)
+ references report_period_score (grouping_kind, grouping_key, period_code, rubric_version)
+ on delete cascade,
+ check (map_post_count >= 0),
+ check (scored_post_count >= map_post_count),
+ check (map_item_count >= 0),
+ check (scored_item_count >= map_item_count),
+ check (incomplete_post_count = scored_post_count - map_post_count),
+ check (incomplete_item_count = scored_item_count - map_item_count)
+);
diff --git a/migrations/rollback/0168_report_leftover_map_coverage.sql b/migrations/rollback/0168_report_leftover_map_coverage.sql
new file mode 100644
index 000000000..58a5ee541
--- /dev/null
+++ b/migrations/rollback/0168_report_leftover_map_coverage.sql
@@ -0,0 +1 @@
+drop table if exists report_leftover_map_coverage;
diff --git a/scripts/seed_demo_data.py b/scripts/seed_demo_data.py
index cfebeb373..0034b35a6 100644
--- a/scripts/seed_demo_data.py
+++ b/scripts/seed_demo_data.py
@@ -120,6 +120,8 @@ def seed(
cur.execute((migrations / "0012_report_leftover_pair.sql").read_text())
cur.execute((migrations / "0163_report_leftover_observed_expected.sql").read_text())
cur.execute((migrations / "0164_report_leftover_map_rank.sql").read_text())
+ cur.execute((migrations / "0168_report_leftover_map_coverage.sql").read_text())
+ cur.execute((migrations / "0169_report_leftover_map_axis.sql").read_text())
cur.execute((migrations / "0182_report_leftover_map_unexplained.sql").read_text())
cur.execute((migrations / "0060_role_responsibility_agent_type.sql").read_text())
cur.execute((migrations / "0013_person_job_title.sql").read_text())
@@ -1235,6 +1237,27 @@ def _persist_seed_period_report(
axis.leftover_share,
),
)
+ if report.leftover_map_coverage is not None:
+ coverage = report.leftover_map_coverage
+ cur.execute(
+ "insert into report_leftover_map_coverage ("
+ "grouping_kind, grouping_key, period_code, rubric_version, "
+ "map_post_count, scored_post_count, map_item_count, scored_item_count, "
+ "incomplete_post_count, incomplete_item_count"
+ ") values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
+ (
+ grouping_kind,
+ grouping_key,
+ period_code,
+ RUBRIC_VERSION,
+ coverage.map_post_count,
+ coverage.scored_post_count,
+ coverage.map_item_count,
+ coverage.scored_item_count,
+ coverage.incomplete_post_count,
+ coverage.incomplete_item_count,
+ ),
+ )
def _seed_demo_period_report(cur, author_account_id, corporate_entity_id, process_unit_id) -> None:
diff --git a/tests/test_leftover_pairs.py b/tests/test_leftover_pairs.py
index 135ddcd2b..a411e1e13 100644
--- a/tests/test_leftover_pairs.py
+++ b/tests/test_leftover_pairs.py
@@ -49,6 +49,7 @@ def _load_leftover():
leftover_pairs_from_residual = leftover.leftover_pairs_from_residual
leftover_map_from_residual = leftover.leftover_map_from_residual
leftover_map_axes_from_singular = leftover.leftover_map_axes_from_singular
+leftover_map_coverage_from_residual = leftover.leftover_map_coverage_from_residual
def _assert_residual_reconciles(pair) -> None:
@@ -106,6 +107,12 @@ def test_leftover_residual_biplot_separates_aligned_and_opposed_cells() -> None:
for pair in pairs:
_assert_residual_reconciles(pair)
assert pair.leftover_map_rank == 1
+ coverage = leftover_map_coverage_from_residual(post_ids, item_codes, matrix, expected)
+ assert coverage.map_post_count == 3
+ assert coverage.scored_post_count == 3
+ assert coverage.incomplete_post_count == 0
+ assert coverage.map_item_count == 3
+ assert coverage.scored_item_count == 3
def test_zero_residual_still_emits_stable_leftover_pairs() -> None:
@@ -129,6 +136,10 @@ def test_zero_residual_still_emits_stable_leftover_pairs() -> None:
for pair in pairs:
_assert_residual_reconciles(pair)
assert pair.leftover_map_rank == 0
+ coverage = leftover_map_coverage_from_residual(post_ids, item_codes, matrix, expected)
+ assert coverage.map_post_count == 2
+ assert coverage.scored_post_count == 2
+ assert coverage.incomplete_post_count == 0
def test_partial_observation_does_not_treat_missing_as_zero_residual() -> None:
@@ -159,6 +170,13 @@ def test_partial_observation_does_not_treat_missing_as_zero_residual() -> None:
_assert_residual_reconciles(pair)
assert pair.leftover_map_rank == 1
assert pair.leftover_map_unexplained == pytest.approx(0.0, abs=1e-6)
+ coverage = leftover_map_coverage_from_residual(post_ids, item_codes, matrix, expected)
+ assert coverage.map_post_count == 2
+ assert coverage.scored_post_count == 3
+ assert coverage.incomplete_post_count == 1
+ assert coverage.map_item_count == 2
+ assert coverage.scored_item_count == 2
+ assert coverage.incomplete_item_count == 0
def test_leftover_is_empty_without_observed_cells() -> None:
@@ -168,6 +186,13 @@ def test_leftover_is_empty_without_observed_cells() -> None:
matrix = np.array([[np.nan]], dtype=np.float64)
expected = np.array([[0.0]], dtype=np.float64)
assert leftover_pairs_from_residual(post_ids, item_codes, matrix, expected) == ()
+ coverage = leftover_map_coverage_from_residual(post_ids, item_codes, matrix, expected)
+ assert coverage.map_post_count == 0
+ assert coverage.scored_post_count == 0
+ assert coverage.incomplete_post_count == 0
+ assert coverage.map_item_count == 0
+ assert coverage.scored_item_count == 0
+ assert coverage.incomplete_item_count == 0
def test_leftover_residual_equals_observed_minus_expected() -> None:
@@ -384,45 +409,7 @@ def test_rejects_response_and_expectation_shape_mismatches() -> None:
)
-def test_sparse_residual_uses_only_observed_cells_for_fallback_distance() -> None:
- """No complete rectangle still yields finite observed-cell distances."""
- matrix = np.array([[1.0, np.nan], [np.nan, -1.0]], dtype=np.float64)
- pairs = leftover_pairs_from_residual(
- ["post-a", "post-b"],
- ("item-a", "item-b"),
- matrix,
- np.zeros_like(matrix),
- )
- assert [(pair.post_id, pair.criterion_code) for pair in pairs] == [
- ("post-a", "item-a"),
- ("post-b", "item-b"),
- ]
- assert [pair.leftover_distance for pair in pairs] == pytest.approx([1.0, 1.0])
- assert [pair.leftover_map_rank for pair in pairs] == [0, 0]
-
-
-def test_leftover_fallback_omits_unexplained_without_complete_case_map() -> None:
- """No complete-case rectangle: persist distance from |R − center|, omit U."""
- post_ids = ["sparse-a", "sparse-b"]
- item_codes = ("item_near", "item_far")
- matrix = np.array(
- [
- [2.0, np.nan],
- [np.nan, -2.0],
- ],
- dtype=np.float64,
- )
- expected = np.zeros_like(matrix)
- pairs = leftover_pairs_from_residual(post_ids, item_codes, matrix, expected)
- assert [pair.pair_kind for pair in pairs] == [PAIR_KIND_CLOSEST, PAIR_KIND_FARTHEST]
- assert {pair.post_id for pair in pairs} == {"sparse-a", "sparse-b"}
- for pair in pairs:
- assert pair.leftover_map_unexplained is None
- assert pair.leftover_distance >= 0.0
- assert not hasattr(pair, "leftover_map_reconstruction")
-
-
-def test_nonfinite_map_distance_falls_back_to_centered_residual(
+def test_nonfinite_map_distance_emits_no_pair(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""An unusable factorization coordinate cannot become persisted distance."""
@@ -441,8 +428,7 @@ def test_nonfinite_map_distance_falls_back_to_centered_residual(
np.array([[1.0]], dtype=np.float64),
np.array([[0.0]], dtype=np.float64),
)
- assert [pair.leftover_distance for pair in pairs] == [0.0, 0.0]
- assert [pair.leftover_map_rank for pair in pairs] == [0, 0]
+ assert pairs == ()
def test_empty_observation_mask_has_no_complete_case_axes() -> None:
@@ -470,3 +456,20 @@ def test_leftover_map_rank_rejects_negative_rank() -> None:
(0.0, "public-post", "sales_lead_specificity", 0.0, 1.0, 1.0, None),
-1,
)
+
+
+def test_leftover_is_unavailable_without_a_complete_case_rectangle() -> None:
+ """Observed cells alone cannot invent Gabriel positions or map coverage."""
+ post_ids = ["post-a", "post-b"]
+ item_codes = ("item-one", "item-two")
+ matrix = np.array([[1.0, np.nan], [np.nan, 1.0]], dtype=np.float64)
+ expected = np.zeros_like(matrix)
+
+ assert leftover_pairs_from_residual(post_ids, item_codes, matrix, expected) == ()
+ coverage = leftover_map_coverage_from_residual(post_ids, item_codes, matrix, expected)
+ assert coverage.map_post_count == 0
+ assert coverage.scored_post_count == 2
+ assert coverage.map_item_count == 0
+ assert coverage.scored_item_count == 2
+ assert coverage.incomplete_post_count == 2
+ assert coverage.incomplete_item_count == 2
diff --git a/tests/test_period_report.py b/tests/test_period_report.py
index 943321394..95021ac4b 100644
--- a/tests/test_period_report.py
+++ b/tests/test_period_report.py
@@ -280,6 +280,13 @@ def test_calibrated_report_attaches_leftover_pairs() -> None:
assert np.isfinite(axis.leftover_singular_value)
assert np.isfinite(axis.leftover_share)
assert sum(axis.leftover_share for axis in report.leftover_map_axes) <= 1.0 + 1e-9
+ coverage = report.leftover_map_coverage
+ assert coverage is not None
+ assert coverage.map_post_count == 8
+ assert coverage.scored_post_count == 8
+ assert coverage.incomplete_post_count == 0
+ assert coverage.map_item_count == len(items)
+ assert coverage.scored_item_count == len(items)
diff --git a/tests/test_schema.py b/tests/test_schema.py
index f2a295cc6..0f9fd18a3 100644
--- a/tests/test_schema.py
+++ b/tests/test_schema.py
@@ -58,6 +58,11 @@
/ "migrations"
/ "0169_report_leftover_map_axis.sql"
)
+_LEFTOVER_MAP_COVERAGE_MIGRATION = (
+ Path(__file__).resolve().parents[1]
+ / "migrations"
+ / "0168_report_leftover_map_coverage.sql"
+)
_LEFTOVER_MAP_UNEXPLAINED_MIGRATION = (
Path(__file__).resolve().parents[1]
/ "migrations"
@@ -101,6 +106,7 @@ def schema_db():
cur.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text())
cur.execute(_LEFTOVER_OBSERVED_EXPECTED_MIGRATION.read_text())
cur.execute(_LEFTOVER_MAP_RANK_MIGRATION.read_text())
+ cur.execute(_LEFTOVER_MAP_COVERAGE_MIGRATION.read_text())
cur.execute(_LEFTOVER_MAP_AXIS_MIGRATION.read_text())
cur.execute(_LEFTOVER_MAP_UNEXPLAINED_MIGRATION.read_text())
conn.commit()
@@ -147,6 +153,7 @@ def test_migration_applies_cleanly(schema_db) -> None:
"report_item_information",
"report_leftover_pair",
"report_leftover_map_axis",
+ "report_leftover_map_coverage",
"post_summary_result",
"post_summary_event",
"post_summary_role",
@@ -264,6 +271,20 @@ def test_leftover_map_axis_references_period_score(schema_db) -> None:
assert "report_period_score" in targets
+def test_leftover_map_coverage_references_period_score(schema_db) -> None:
+ """Complete-case leftover coverage is 1:1 with the period report."""
+ with schema_db.cursor() as cur:
+ cur.execute(
+ """
+ select confrelid::regclass::text
+ from pg_constraint
+ where conrelid = 'report_leftover_map_coverage'::regclass and contype = 'f'
+ """
+ )
+ targets = {row[0] for row in cur.fetchall()}
+ assert "report_period_score" in targets
+
+
def test_corporate_hierarchy_recursive_query_returns_correct_shape(schema_db) -> None:
"""The real product requirement: 'Acme Group -> Acme Electronics Korea
-> Acme Electronics Gwangju Plant' must be walkable with one query,