feat(lineage): estimate channel-fusion weights psychometrically (ADR 0145)

backend/app/lineage_ingestion.py

@@ -72,14 +72,45 @@ async def persist_lineage_edges(conn: asyncpg.Connection, edges: list[Edge]) ->
         )
 
 
+async def load_estimated_channel_weights(
+    conn: asyncpg.Connection, active_channels: set[str]
+) -> dict[str, float] | None:
+    """Persisted psychometric weights, only on an exact channel-set match.
+
+    ADR 0145: a partial overlap would mix estimated and hand-picked
+    weights into a vector that is neither grounded nor the documented
+    fallback -- so anything other than an exact match falls back
+    entirely (return ``None``). A database that has not applied
+    migration 0135 yet (rollout ordering, rollback) is the same "no
+    estimate persisted" state, not an error -- rebuilds keep working on
+    the fallback constants.
+    """
+    try:
+        rows = await conn.fetch(
+            "select channel_code, weight_value from lineage_channel_weight"
+        )
+    except asyncpg.UndefinedTableError:
+        return None
+    persisted = {row["channel_code"]: float(row["weight_value"]) for row in rows}
+    if not persisted or set(persisted) != active_channels:
+        return None
+    return persisted
+
+
 async def rebuild_lineage(conn: asyncpg.Connection) -> list[Edge]:
     """Reconstruct lineage for every ``source_post`` and persist the edges."""
     rows = await conn.fetch(
         "select post_id, post_title, voc_type_code, created_at, corporate_entity_id, "
         "process_unit_id, thread_group_key, secondary_grouping_key "
         f"from source_post where {SOURCE_POST_ELIGIBILITY_SQL.format(alias='source_post')}"
     )
-    edges = lineage_edge_specs(records_from_source_posts(rows))
+    # No adjudication client is wired on this path, so the active channel
+    # set is the three deterministic channels (reconstruct drops llm when
+    # unavailable rather than faking it).
+    weights = await load_estimated_channel_weights(
+        conn, {"temporal", "secondary_key", "text"}
+    )
+    edges = lineage_edge_specs(records_from_source_posts(rows), weights=weights)
     await persist_lineage_edges(conn, edges)
     return edges
 

backend/tests/test_api.py

@@ -169,6 +169,11 @@
     / "migrations"
     / "0114_semantic_relationship_standard_predicates.sql"
 )
+_CHANNEL_WEIGHT_MIGRATION = (
+    Path(__file__).resolve().parents[2]
+    / "migrations"
+    / "0135_lineage_channel_weight.sql"
+)
 
 
 def _postgres_available() -> bool:
@@ -294,6 +299,7 @@ def seeded_db(demo_analyst_token):
             cur.execute(_SOFTWARE_AGENT_MIGRATION.read_text())
             cur.execute(_SEMANTIC_RELATIONSHIP_MIGRATION.read_text())
             cur.execute(_SEMANTIC_RELATIONSHIP_PREDICATES_MIGRATION.read_text())
+            cur.execute(_CHANNEL_WEIGHT_MIGRATION.read_text())
             cur.execute(
                 "insert into common_lookup_value (lookup_category, lookup_code, lookup_label) values "
                 "('corporate_entity_level', 'group', 'Group'), "

docker/postgres-init/migrate.sh

@@ -18,7 +18,7 @@ for migration in /opt/lineageweave/migrations/*.sql; do
     migration_name=${migration##*/}
     case "$migration_name" in
         0012_*|0013_*|0014_*|0015_*|0016_*|0017_*|0018_*|0019_*|0020_*|0021_*|0022_*|0023_*|0024_*|0025_*|0026_*|0027_*|0028_*|0029_*|0030_*|0031_*|0032_*|0033_*|0034_*|0035_*|0036_*|0037_*|0038_*|0039_*|0040_*|0041_*|0042_*|0043_*|0044_*|0045_*|0046_*|0047_*|0048_*|0049_*|0050_*) ;;
-        0060_*|0100_*|0101_*|0102_*|0103_*|0104_*|0105_*|0106_*|0107_*|0108_*|0109_*|0110_*|0111_*|0112_*|0113_*|0114_*|0130_*|0131_*|0132_*|0133_*|0134_*) ;;
+        0060_*|0100_*|0101_*|0102_*|0103_*|0104_*|0105_*|0106_*|0107_*|0108_*|0109_*|0110_*|0111_*|0112_*|0113_*|0114_*|0130_*|0131_*|0132_*|0133_*|0134_*|0135_*) ;;
         *) continue ;;
     esac
     printf 'Applying %s\n' "$migration_name"

docs/adr/0145-psychometric-channel-weight-estimation.md

@@ -0,0 +1,160 @@
+# ADR 0145 — Lineage channel-fusion weights come from psychometric estimation, not hand-picked constants
+
+**Decision status:** Proposed
+**Date:** 2026-08-23
+
+> Numbering note: parallel branches are assigning ADR numbers concurrently
+> (0143 exists on an unmerged branch). `0145` was the next free number on
+> `docs/customer-master-scope-adr` at time of writing and may need
+> renumbering when branches converge.
+
+## Context
+
+`lineageweave.reconstruct` fuses four evidence channels (temporal,
+secondary-key, text-similarity, llm adjudication) into one convex score
+per candidate parent-child pair. The fusion weights,
+`DEFAULT_CHANNEL_WEIGHTS = {"temporal": 0.15, "secondary_key": 0.15,
+"text": 0.30, "llm": 0.40}`, were hand-picked: the module's own comment
+justifies them with a qualitative argument ("llm ... is the only channel
+that actually reasons about the content"), not with any estimate from
+data. The product's standing requirement — repeated across many
+sessions — is that scoring weights be grounded in published measurement
+methodology through the organization's own psychometric libraries
+(`fast-mlsirm`, TEPP), never asserted by fiat.
+
+The measurement literature gives an exact grounding. Treat each channel
+as an *item* observing the latent trait "these two posts are genuinely
+related", and each scored candidate pair as a *respondent*. Under the
+two-parameter logistic model, the information-optimal scoring weight of
+an item is proportional to its discrimination parameter (Birnbaum, 1968;
+Lord, 1980) — an unweighted or arbitrarily-weighted composite discards
+exactly that information (McNeish & Wolf, 2020). Pairs are nested inside
+reconstruction groups (process unit / corporate entity / thread), so a
+single-level fit would commit the ecological/atomistic inference error
+the standing mandate calls out (Robinson, 1950); `fast-mlsirm`'s MLS2PLM
+is a *multilevel* 2PL whose `factor_id` models exactly this nesting, and
+its `MLSIRMParams.alpha` field is the per-item log-discrimination
+(natural-scale discrimination `exp(alpha)` is positive by construction,
+so normalizing to sum 1 always yields valid convex weights).
+
+`fast-mlsirm` is not yet published to PyPI, so LineageWeave cannot take
+a hard install dependency today. This repository's established pattern
+for every optional capability is fail-closed clients (Null client when
+unconfigured, never a fabricated result); the same pattern applies here.
+
+## Decision
+
+1. **Estimation, not assertion.** A new module,
+   `lineageweave/channel_weight_estimation.py`, estimates channel
+   weights by fitting `fast-mlsirm`'s MLS2PLM over observed channel
+   scores: items = channels, respondents = candidate pairs sampled the
+   same way `reconstruct` forms them (same grouping, same candidate
+   window), `factor_id` = the pair's reconstruction group (multilevel
+   nesting per Robinson, 1950). Estimated weights are the normalized
+   natural-scale discriminations, `exp(alpha_j) / Σ exp(alpha_k)`
+   (Birnbaum, 1968).
+2. **Dichotomization at the fusion floor.** MLS2PLM is dichotomous;
+   channel scores in [0, 1] are dichotomized at
+   `DEFAULT_MIN_FUSED_SCORE` (0.3) — the same threshold `reconstruct`
+   already treats as the boundary between "evidence of a link" and
+   "no plausible candidate", so the measurement model observes the same
+   binary event the fusion decision acts on.
+3. **Fail closed, never fabricate.** When `fast-mlsirm` is not
+   importable, the sample is too small, or the fit degenerates (any
+   non-finite alpha), estimation returns nothing and callers keep the
+   documented fallback constants — now explicitly labeled as
+   *ungrounded fallback* in `reconstruct.py`'s docstring, not as a
+   justified default.
+4. **Persisted, provenance-bearing weights.** An operator script
+   (`scripts/estimate_channel_weights.py`) runs the estimation against
+   the real corpus and upserts one row per channel into a new
+   `lineage_channel_weight` table (migration 0135) carrying the weight,
+   the estimation method code, the sample size, and the estimation
+   timestamp. `rebuild_lineage` loads these rows and passes them to
+   `reconstruct`; it uses them **only when the persisted channel set
+   exactly matches the active channel set** (no partial mixing of
+   estimated and hand-picked weights — a mixed vector is neither
+   grounded nor the documented fallback), otherwise it falls back
+   entirely.
+5. **The llm channel is estimated only when adjudication is
+   configured.** Scoring sampled pairs through the adjudication client
+   costs provider calls; the script includes the llm channel when a
+   client is available and skips it otherwise (the exact-match rule in
+   (4) then keeps rebuilds on the fallback until a full estimate
+   exists). Accuracy over speed, per the standing mandate.
+
+## Consequences
+
+**Positive.** Fusion weights become an estimable, auditable quantity
+with a citation trail instead of a code comment: the persisted row
+records how many pairs supported the estimate and when. Re-running the
+operator script after corpus growth re-calibrates the fusion without a
+code change. The multilevel fit respects group nesting rather than
+pooling pairs atomistically.
+
+**Negative.** A new optional dependency surface (fast-mlsirm via git
+until it reaches PyPI) and a new operator step. Dichotomizing at 0.3
+discards within-interval score variation; a graded/continuous-response
+model (fast-mlsirm ships `grm.py`/`crm.py`) is the natural upgrade once
+this loop is validated end-to-end — deliberately out of scope for the
+first landing. TEPP-side calibration (event-level theta as the latent
+anchor) is a further integration this ADR does not attempt while TEPP
+remains non-production (see the standing `tepp_readiness_watch`).
+
+## Rejected Alternatives
+
+- **Keep hand-picked constants.** The standing product requirement
+  explicitly forbids this; no citation supports the current 0.15/0.15/
+  0.30/0.40 split.
+- **Reciprocal Rank Fusion for this surface.** RRF (Cormack et al.,
+  2009) is parameter-free and already grounds RankWeave's *rank* fusion
+  constant (η = 60), but reconstruct's decision is a thresholded
+  *score* over at most `candidate_window` candidates, not a deep
+  ranked-list merge; discarding score magnitude here would also discard
+  `DEFAULT_MIN_FUSED_SCORE`'s "no plausible parent" semantics.
+- **Supervised weight learning (logistic regression on labeled pairs).**
+  There is no labeled corpus: the source system carries no ground-truth
+  thread links (their absence is why this library exists). IRT estimates
+  discriminations from response structure without per-pair labels.
+
+## Implementation Notes
+
+1. Migration `0135_lineage_channel_weight.sql`:
+   `lineage_channel_weight(channel_code text primary key, weight_value
+   double precision not null, estimation_method_code text not null,
+   sample_pair_count bigint not null, estimated_at timestamptz not null
+   default now())`, plus rollback. Two-word snake_case per ADR 0120.
+2. `estimate_channel_weights()` returns `None` on: import failure,
+   fewer than `_MIN_SAMPLE_PAIRS` pairs, any channel with fewer than two
+   distinct dichotomized responses, or any non-finite estimated alpha.
+3. `lineage_edge_specs` and `rebuild_lineage` gain an optional
+   `weights` pass-through; `None` keeps today's behavior exactly.
+4. Tests: fail-closed paths run everywhere; a parameter-recovery test
+   (planted discriminations recovered within tolerance, the
+   organization's RMSE standard) runs when `fast_mlsirm` is importable
+   and skips honestly otherwise, same as this repo's live-service
+   skips.
+
+## References (APA 7th)
+
+Birnbaum, A. (1968). Some latent trait models and their use in
+inferring an examinee's ability. In F. M. Lord & M. R. Novick,
+*Statistical theories of mental test scores* (pp. 397–479).
+Addison-Wesley.
+
+Cormack, G. V., Clarke, C. L. A., & Buettcher, S. (2009). Reciprocal
+rank fusion outperforms Condorcet and individual rank learning methods.
+*Proceedings of the 32nd International ACM SIGIR Conference on Research
+and Development in Information Retrieval*, 758–759.
+https://doi.org/10.1145/1571941.1572114
+
+Lord, F. M. (1980). *Applications of item response theory to practical
+testing problems*. Lawrence Erlbaum Associates.
+
+McNeish, D., & Wolf, M. G. (2020). Thinking twice about sum scores.
+*Behavior Research Methods, 52*(6), 2287–2305.
+https://doi.org/10.3758/s13428-020-01398-0
+
+Robinson, W. S. (1950). Ecological correlations and the behavior of
+individuals. *American Sociological Review, 15*(3), 351–357.
+https://doi.org/10.2307/2087176

lineageweave/channel_weight_estimation.py

@@ -0,0 +1,134 @@
+"""Psychometric estimation of lineage channel-fusion weights (ADR 0145).
+
+The convex weights `reconstruct()` fuses its evidence channels with were
+historically hand-picked constants. This module replaces assertion with
+estimation: each channel is treated as an *item* observing the latent
+trait "these two posts are genuinely related", each scored candidate
+pair as a *respondent*, and the pair's reconstruction group as the
+multilevel nesting factor (Robinson, 1950, on why pooling nested
+observations atomistically misleads). Under the two-parameter logistic
+model the information-optimal scoring weight of an item is proportional
+to its discrimination (Birnbaum, 1968; Lord, 1980), so the estimated
+weights are the normalized natural-scale discriminations from
+`fast-mlsirm`'s multilevel 2PL (`MLS2PLM`), whose ``MLSIRMParams.alpha``
+holds per-item log-discriminations.
+
+Fail-closed like every optional capability in this codebase: when
+`fast_mlsirm` is not importable, the sample is too small, any channel is
+degenerate (fewer than two distinct dichotomized responses), or the fit
+produces a non-finite estimate, :func:`estimate_channel_weights` returns
+``None`` and the caller keeps the documented fallback constants -- it
+never fabricates a "grounded" weight.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+
+from .reconstruct import DEFAULT_MIN_FUSED_SCORE
+
+# Below this many scored pairs a 2PL discrimination estimate is noise,
+# not measurement -- refuse rather than persist an unstable weight.
+_MIN_SAMPLE_PAIRS = 200
+
+
+@dataclass(frozen=True)
+class ChannelWeightEstimate:
+    """One estimation run's convex weights plus its provenance."""
+
+    weights: dict[str, float]
+    sample_pair_count: int
+    estimation_method_code: str
+
+
+def dichotomize(score: float, threshold: float = DEFAULT_MIN_FUSED_SCORE) -> int:
+    """Binary "evidence of a link" event at the fusion floor.
+
+    `reconstruct` already treats ``DEFAULT_MIN_FUSED_SCORE`` as the
+    boundary between a plausible parent and no candidate at all, so the
+    measurement model observes the same event the fusion decision acts
+    on (ADR 0145 point 2).
+    """
+    return 1 if score >= threshold else 0
+
+
+def estimate_channel_weights(
+    pair_channel_scores: list[dict[str, float]],
+    group_ids: list[int],
+) -> ChannelWeightEstimate | None:
+    """Estimate convex fusion weights from observed channel scores.
+
+    Args:
+        pair_channel_scores: one dict per candidate pair mapping every
+            active channel name to its score in [0, 1]. Every dict must
+            carry the same channel set -- a pair missing a channel is a
+            caller bug, not missing data to impute.
+        group_ids: the reconstruction-group index of each pair (same
+            length/order), used as MLS2PLM's multilevel ``factor_id``.
+
+    Returns:
+        The estimate, or ``None`` whenever a grounded estimate cannot be
+        produced (fail closed -- see module docstring for the cases).
+    """
+    if len(pair_channel_scores) != len(group_ids):
+        raise ValueError("pair_channel_scores and group_ids must align")
+    if len(pair_channel_scores) < _MIN_SAMPLE_PAIRS:
+        return None
+    channels = sorted(pair_channel_scores[0])
+    if not channels:
+        return None
+    for scores in pair_channel_scores:
+        if sorted(scores) != channels:
+            raise ValueError("every pair must score the same channel set")
+
+    responses = [
+        [dichotomize(scores[channel]) for channel in channels]
+        for scores in pair_channel_scores
+    ]
+    for column, channel in enumerate(channels):
+        observed = {row[column] for row in responses}
+        if len(observed) < 2:
+            # A channel that always (or never) clears the floor carries no
+            # discriminating information; a 2PL slope for it is undefined
+            # in practice. Refuse rather than estimate around it.
+            return None
+
+    try:
+        import numpy
+        from fast_mlsirm import FitConfig, fit
+    except ImportError:
+        return None
+
+    # One latent "relatedness" trait loads every channel (factor_id maps
+    # items to latent dimensions); pairs are nested in reconstruction
+    # groups via cluster_id -- fast-mlsirm's multilevel random-intercept
+    # structure (Fox & Glas, 2001), which requires the marginal (mmle)
+    # estimator.
+    result = fit(
+        responses=numpy.asarray(responses, dtype=float),
+        factor_id=numpy.zeros(len(channels), dtype=numpy.int64),
+        cluster_id=numpy.asarray(group_ids, dtype=numpy.int64),
+        config=FitConfig(model="MLS2PLM", latent_dim=1, estimator="mmle"),
+    )
+    log_discriminations = list(numpy.asarray(result.params.alpha, dtype=float).ravel())
+    if len(log_discriminations) != len(channels):
+        return None
+    if any(not math.isfinite(alpha) for alpha in log_discriminations):
+        return None
+
+    # exp(alpha) is the natural-scale discrimination -- positive by
+    # construction, so the normalization always yields valid convex
+    # weights (Birnbaum, 1968: optimal weight proportional to a_j).
+    discriminations = [math.exp(alpha) for alpha in log_discriminations]
+    total = sum(discriminations)
+    if not math.isfinite(total) or total <= 0:
+        return None
+    return ChannelWeightEstimate(
+        weights={
+            channel: discrimination / total
+            for channel, discrimination in zip(channels, discriminations)
+        },
+        sample_pair_count=len(pair_channel_scores),
+        estimation_method_code="mls2plm_discrimination",
+    )