fix(lineage): preserve grouping provenance and fail closed weights

backend/app/lineage_ingestion.py

@@ -10,15 +10,22 @@
 
 from __future__ import annotations
 
+import math
+import re
+from collections.abc import Mapping
 from datetime import datetime
-from typing import Any, Mapping
+from typing import Any
 
 import asyncpg
 
 from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL
 from lineageweave.lineage_persistence import lineage_edge_specs
 from lineageweave.models import Edge, Record
 
+# ADR 0145 rejected the unanchored estimator. A future accepted ADR must add
+# its independently validated method code here before persisted weights can run.
+_SUPPORTED_ANCHOR_METHOD_CODES: frozenset[str] = frozenset()
+
 
 def _occurred_at(value: datetime) -> datetime:
     """Reconstruct expects naive datetimes; asyncpg returns timestamptz."""
@@ -72,14 +79,94 @@ async def persist_lineage_edges(conn: asyncpg.Connection, edges: list[Edge]) ->
         )
 
 
+async def load_estimated_channel_weights(
+    conn: asyncpg.Connection, active_channels: set[str]
+) -> dict[str, float] | None:
+    """Load only a complete vector from an independently anchored method.
+
+    ADR 0145 currently authorizes no anchor method. A partial or invalid vector
+    returns ``None`` rather than being repaired. A database that has not applied
+    migration 0135 is likewise an unavailable state, detected without issuing a
+    statement that would abort the caller's outer PostgreSQL transaction.
+    """
+    table_exists = await conn.fetchval(
+        "select to_regclass('public.lineage_channel_weight') is not null"
+    )
+    if not table_exists:
+        return None
+    rows = await conn.fetch(
+        "select channel_code, weight_value, estimation_run_id, "
+        "estimation_method_code, estimator_version, anchor_method_code, "
+        "source_snapshot_sha256, sample_pair_count, knowledge_cutoff "
+        "from lineage_channel_weight"
+    )
+    persisted = {row["channel_code"]: float(row["weight_value"]) for row in rows}
+    if not persisted or set(persisted) != active_channels:
+        return None
+    if any(
+        not math.isfinite(weight) or weight <= 0 or weight > 1
+        for weight in persisted.values()
+    ):
+        return None
+    if not math.isclose(sum(persisted.values()), 1.0, rel_tol=0.0, abs_tol=1e-9):
+        return None
+    provenance = {
+        (
+            row["estimation_run_id"],
+            row["estimation_method_code"],
+            row["estimator_version"],
+            row["anchor_method_code"],
+            row["source_snapshot_sha256"],
+            row["sample_pair_count"],
+            row["knowledge_cutoff"],
+        )
+        for row in rows
+    }
+    if len(provenance) != 1:
+        return None
+    run = next(iter(provenance))
+    (
+        run_id,
+        estimation_method,
+        estimator_version,
+        anchor_method,
+        snapshot_digest,
+        sample_pair_count,
+        knowledge_cutoff,
+    ) = run
+    if (
+        run_id is None
+        or not isinstance(estimation_method, str)
+        or not estimation_method.strip()
+        or not isinstance(estimator_version, str)
+        or not estimator_version.strip()
+        or not isinstance(anchor_method, str)
+        or anchor_method not in _SUPPORTED_ANCHOR_METHOD_CODES
+        or not isinstance(snapshot_digest, str)
+        or re.fullmatch(r"[0-9a-f]{64}", snapshot_digest) is None
+        or not isinstance(sample_pair_count, int)
+        or isinstance(sample_pair_count, bool)
+        or sample_pair_count < 200
+        or not isinstance(knowledge_cutoff, datetime)
+    ):
+        return None
+    return persisted
+
+
 async def rebuild_lineage(conn: asyncpg.Connection) -> list[Edge]:
     """Reconstruct lineage for every ``source_post`` and persist the edges."""
     rows = await conn.fetch(
         "select post_id, post_title, voc_type_code, created_at, corporate_entity_id, "
         "process_unit_id, thread_group_key, secondary_grouping_key "
         f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}"
     )
-    edges = lineage_edge_specs(records_from_source_posts(rows))
+    # No adjudication client is wired on this path, so the active channel
+    # set is the three deterministic channels (reconstruct drops llm when
+    # unavailable rather than faking it).
+    weights = await load_estimated_channel_weights(
+        conn, {"temporal", "secondary_key", "text"}
+    )
+    edges = lineage_edge_specs(records_from_source_posts(rows), weights=weights)
     await persist_lineage_edges(conn, edges)
     return edges
 

backend/tests/test_api.py

@@ -113,6 +113,11 @@
     / "migrations"
     / "0102_project_bound_summary_event.sql"
 )
+_CHANNEL_WEIGHT_MIGRATION = (
+    Path(__file__).resolve().parents[2]
+    / "migrations"
+    / "0135_lineage_channel_weight.sql"
+)
 
 
 def _postgres_available() -> bool:
@@ -226,6 +231,7 @@ def seeded_db(demo_analyst_token):
             cur.execute(_MAJOR_EVENT_ACTION_MIGRATION.read_text())
             cur.execute(_PROJECT_BOUND_ACTION_MIGRATION.read_text())
             cur.execute(_PROJECT_BOUND_EVENT_MIGRATION.read_text())
+            cur.execute(_CHANNEL_WEIGHT_MIGRATION.read_text())
             cur.execute(
                 "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values "
                 "('corporate_entity_level', 'group', 'Group'), "

docker/postgres-init/migrate.sh

@@ -18,7 +18,7 @@ for migration in /opt/lineageweave/migrations/*.sql; do
     migration_name=${migration##*/}
     case "$migration_name" in
         0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;;
-        0060_*|0100_*|0101_*|0102_*) ;;
+        0060_*|0100_*|0101_*|0102_*|0135_*) ;;
         *) continue ;;
     esac
     printf 'Applying %s\n' "$migration_name"

docs/adr/0145-psychometric-channel-weight-estimation.md

@@ -0,0 +1,87 @@
+# ADR 0145 — Channel-weight estimation remains unavailable without an independent anchor
+
+**Decision status:** Rejected proposal
+**Date:** 2026-08-23
+**Reconciles with:** [ADR 0003](0003-fast-mlsirm-report-integration.md)
+
+## Context
+
+`lineageweave.reconstruct` currently uses hand-picked convex weights for its
+temporal, secondary-key, text-similarity, and optional LLM channels. Those
+constants are an explicitly ungrounded historical fallback; a citation does
+not turn them into calibrated measurement.
+
+The rejected proposal treated channels as 2PL items, candidate pairs as
+respondents, and normalized item discriminations as fusion weights. That does
+not establish the product construct. An unanchored IRT fit can describe common
+response structure, but it provides no independent evidence that its latent
+factor is “these posts are genuinely related.” Birnbaum's 2PL item information
+is also conditional on trait location and item difficulty,
+`I_j(theta) = a_j^2 P_j(theta) (1 - P_j(theta))`; it is not a global constant
+proportional only to `a_j`. Normalizing discriminations therefore is not an
+information-optimal convex fusion rule.
+
+The official pinned `fast-mlsirm` contract makes two further boundaries
+explicit: `factor_id` assigns items to latent dimensions, while `cluster_id`
+represents respondent nesting; and a `FitResult` exposes
+`convergence_status` plus package diagnostics that callers must inspect. Those
+contracts can validate a fit's execution, but cannot supply the missing
+criterion validity.
+
+Accepted ADR 0003 assigns temporal/event measurement to TEPP and limits this
+repository's fast-mlsirm integration to the approved LLM-judge/report path.
+This proposed lineage-weight path cannot silently expand that boundary.
+
+## Decision
+
+1. **No unanchored estimate.** LineageWeave does not run the proposed
+   candidate-pair IRT fit and does not persist or activate weights from it.
+   Estimation reports unavailable until an independent lineage anchor and its
+   upstream contract exist.
+2. **No scientific claim for fallback constants.** Existing constants remain
+   unchanged for compatibility, but are neither calibrated nor
+   paper-grounded. This ADR does not promote them to measurement evidence.
+3. **A future proposal must be ADR-first.** It must amend ADR 0003, identify an
+   independent outcome/anchor (for example, an accepted TEPP contract rather
+   than a local proxy), use official fast-mlsirm diagnostics, reject every
+   non-converged fit, and prove criterion validity before product activation.
+4. **Future persisted vectors must be self-consistent and reproducible.** The
+   schema requires a known channel vocabulary, finite positive weights, an
+   exact sum of one at runtime, one estimation run identity, estimator and
+   anchor method versions, sample size, immutable source-snapshot digest, and
+   knowledge cutoff. Current code has no supported anchor method and therefore
+   loads no vector.
+5. **Grouping repair is independent of measurement.** Source row identifiers
+   that were mapped as grouping values are normalized only in derived
+   reconstruction fields. Their caller-mapped raw values remain preserved in
+   separate source-provenance columns across backfill and re-import.
+
+## Consequences
+
+- `scripts/estimate_channel_weights.py` exits without writing because no
+  scientifically authorized anchor exists.
+- Missing migration 0135 remains a normal rollout state and is detected via a
+  non-error PostgreSQL catalog probe, so an outer rebuild transaction is not
+  aborted.
+- The persistence contract is fail-closed: malformed, mixed-provenance, or
+  unsupported-anchor rows are ignored rather than renormalized or repaired.
+- Parameter-recovery tests cannot substitute for criterion validity; they may
+  return only with a future accepted anchored estimator.
+
+## References
+
+Birnbaum, A. (1968). Some latent trait models and their use in inferring an
+examinee's ability. In F. M. Lord & M. R. Novick, *Statistical theories of
+mental test scores* (pp. 397–479). Addison-Wesley.
+
+Fox, J.-P., & Glas, C. A. W. (2001). Bayesian estimation of a multilevel IRT
+model using Gibbs sampling. *Psychometrika, 66*(2), 271–288.
+https://doi.org/10.1007/BF02294839
+
+McNeish, D., & Wolf, M. G. (2020). Thinking twice about sum scores.
+*Behavior Research Methods, 52*(6), 2287–2305.
+https://doi.org/10.3758/s13428-020-01398-0
+
+ContextualWisdomLab. (2026). *fast-mlsirm*, pinned LineageWeave dependency
+contract at commit `5006c38286a4fa1d81bcf57eeed5ce27ae743f50`.
+https://github.com/ContextualWisdomLab/fast-mlsirm/tree/5006c38286a4fa1d81bcf57eeed5ce27ae743f50

lineageweave/channel_weight_estimation.py

@@ -0,0 +1,25 @@
+"""Fail-closed lineage channel-weight boundary (ADR 0145).
+
+Channel-score covariance is not an independent lineage anchor. Until an
+accepted upstream anchor contract exists, this module produces no estimate.
+"""
+
+from __future__ import annotations
+
+
+def estimate_channel_weights(
+    pair_channel_scores: list[dict[str, float]],
+    group_ids: list[int],
+) -> None:
+    """Report unavailable for unanchored channel scores.
+
+    Args:
+        pair_channel_scores: unanchored candidate-pair channel scores.
+        group_ids: corresponding reconstruction-group indexes.
+
+    Returns:
+        Always ``None`` until ADR 0145's independent-anchor requirement is met.
+    """
+    if len(pair_channel_scores) != len(group_ids):
+        raise ValueError("pair_channel_scores and group_ids must align")
+    return None

lineageweave/lineage_persistence.py

@@ -16,7 +16,12 @@
 from .reconstruct import reconstruct
 
 
-def lineage_edge_specs(records: Sequence[Record], *, llm: AdjudicationClient | None = None) -> list[Edge]:
+def lineage_edge_specs(
+    records: Sequence[Record],
+    *,
+    llm: AdjudicationClient | None = None,
+    weights: dict[str, float] | None = None,
+) -> list[Edge]:
     """Run reconstruct and return every resulting parent→child edge.
 
     Callers persist these as ``post_lineage_edge`` rows. Record ids must
@@ -29,6 +34,13 @@ def lineage_edge_specs(records: Sequence[Record], *, llm: AdjudicationClient | N
     (the llm channel is then dropped and the rest renormalized, not
     faked) -- callers that want the highest-weighted reasoning channel
     actually contributing to real reconstructions must pass a real one.
+
+    ``weights`` defaults to ``None``, keeping ``reconstruct()``'s
+    documented fallback constants; callers with a persisted
+    psychometric estimate (ADR 0145) pass it here.
     """
-    trees = reconstruct(list(records), llm=llm)
+    if weights is None:
+        trees = reconstruct(list(records), llm=llm)
+    else:
+        trees = reconstruct(list(records), llm=llm, weights=weights)
     return [edge for tree in trees for edge in tree.edges]

lineageweave/reconstruct.py

@@ -20,10 +20,9 @@
 from .channels import secondary_key_match_score, temporal_score, text_similarity_score
 from .models import Edge, Record, Tree
 
-# Channel weights when every channel is available. llm gets the most weight
-# because it is the only channel that actually reasons about the content
-# instead of approximating it; the rest renormalize when llm is unavailable
-# (see active_weights()).
+# Legacy compatibility weights when every channel is available. These constants
+# are not calibrated measurement evidence (ADR 0145); unavailable channels are
+# dropped and the remainder renormalized by active_weights().
 DEFAULT_CHANNEL_WEIGHTS = {"temporal": 0.15, "secondary_key": 0.15, "text": 0.30, "llm": 0.40}
 
 # ponytail: only the most recent WINDOW prior records in a group are

migrations/0135_lineage_channel_weight.sql

@@ -0,0 +1,41 @@
+-- ADR 0145: reserve an integrity- and provenance-bearing persistence contract.
+-- No anchor method is currently authorized, so application code activates no
+-- stored vector.
+
+alter table source_post
+    add column if not exists source_thread_group_key text,
+    add column if not exists source_secondary_grouping_key text;
+
+comment on column source_post.source_thread_group_key is
+    'Raw caller-mapped source thread field; preserved separately from derived reconstruction grouping.';
+comment on column source_post.source_secondary_grouping_key is
+    'Raw caller-mapped source secondary-group field; preserved separately from derived reconstruction evidence.';
+
+create table if not exists lineage_channel_weight (
+    channel_code text primary key,
+    weight_value double precision not null,
+    estimation_run_id uuid not null,
+    estimation_method_code text not null,
+    estimator_version text not null,
+    anchor_method_code text not null,
+    source_snapshot_sha256 text not null,
+    sample_pair_count bigint not null,
+    knowledge_cutoff timestamptz not null,
+    estimated_at timestamptz not null default now(),
+    constraint lineage_channel_code_check
+        check (channel_code in ('temporal', 'secondary_key', 'text', 'llm')),
+    constraint lineage_weight_value_check
+        check (weight_value > 0 and weight_value <= 1),
+    constraint lineage_estimation_method_check
+        check (btrim(estimation_method_code) <> ''),
+    constraint lineage_estimator_version_check
+        check (btrim(estimator_version) <> ''),
+    constraint lineage_anchor_method_check
+        check (btrim(anchor_method_code) <> ''),
+    constraint lineage_sample_pair_count_check
+        check (sample_pair_count >= 200),
+    constraint lineage_source_snapshot_check
+        check (source_snapshot_sha256 ~ '^[0-9a-f]{64}$'),
+    constraint lineage_knowledge_cutoff_check
+        check (knowledge_cutoff <= estimated_at)
+);

migrations/rollback/0135_lineage_channel_weight.sql

@@ -0,0 +1,8 @@
+-- Rollback for 0135: remove the unavailable channel-weight persistence
+-- contract and the raw grouping provenance columns introduced with it.
+
+drop table if exists lineage_channel_weight;
+
+alter table source_post
+    drop column if exists source_secondary_grouping_key,
+    drop column if exists source_thread_group_key;