From dd362618864684cab1897bf9356c18dd26194d68 Mon Sep 17 00:00:00 2001 From: jinhongkuan Date: Wed, 15 Apr 2026 00:33:41 -0400 Subject: [PATCH] =?UTF-8?q?mcp:=20v0.4.14=20=E2=80=94=20surface=20source?= =?UTF-8?q?=5Fexcerpt=20+=20meeting=5Fdate=20in=20read=20responses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "tie meeting context to code" value prop only worked at write time. At read time, brief / drift / search returned the decision text and source_ref string but stripped the raw source passage that produced the decision — even though source_span.text was sitting in the ledger waiting to be surfaced. Surfaced during demo gallery work when the visual was forced to either invent meeting context or look thin. Added: - DecisionMatch.source_excerpt + meeting_date (search responses) - DriftEntry.source_excerpt + meeting_date (drift responses) - BriefDecision.source_excerpt + meeting_date (brief responses) - search_by_bm25 pulls source_span.{text, meeting_date} via <-yields<-source_span reverse traversal in the same query — no extra DB roundtrip - get_decisions_for_file does a single follow-up batched query against matched intent IDs to backfill the same fields - Synthetic-span filter: _reground_ungrounded writes placeholder source_spans where text == intent.description to trigger lazy grounding. Both query paths filter those out so the excerpt reflects the original meeting passage, never the bookkeeping placeholder. Tests: - 4 new cases in tests/test_v0414_source_excerpt.py covering search/brief/drift/empty paths - Test isolation fixes for v0.4.13 dedup tests that were sharing state with v0.4.14 tests via the in-memory ledger singleton Full v0.4.14 regression: 215 passed. Co-Authored-By: Claude Opus 4.6 (1M context) --- CHANGELOG.md | 44 ++++++ contracts.py | 12 ++ handlers/brief.py | 3 + handlers/detect_drift.py | 2 + handlers/search_decisions.py | 2 + ledger/queries.py | 84 ++++++++++- pyproject.toml | 2 +- tests/test_v0413_canonical_dedup.py | 2 + tests/test_v0414_source_excerpt.py | 224 ++++++++++++++++++++++++++++ 9 files changed, 368 insertions(+), 7 deletions(-) create mode 100644 tests/test_v0414_source_excerpt.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f580957..d60b5070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,50 @@ All notable changes to bicameral-mcp are tracked here. Format loosely follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## 0.4.14 — 2026-04-15 — Source Excerpt + Meeting Date in Read Responses + +The "tie meeting context to code" value prop only worked at write time. +At read time, the brief / drift / search responses returned the +decision text and the source_ref string but stripped the raw source +passage that produced the decision — even though `source_span.text` +was sitting in the ledger waiting to be surfaced. Surfaced during +demo gallery work when the visual was forced to either invent +context or look thin. + +### Added + +- **`DecisionMatch.source_excerpt` + `meeting_date`** (search responses) +- **`DriftEntry.source_excerpt` + `meeting_date`** (drift responses) +- **`BriefDecision.source_excerpt` + `meeting_date`** (brief responses) +- **`search_by_bm25`** now pulls source_span.text + meeting_date via + `<-yields<-source_span.{text, meeting_date}` reverse traversal in + the same query — no extra DB roundtrip. +- **`get_decisions_for_file`** does a follow-up batched query against + the matched intent IDs to backfill the same fields. Single round-trip + regardless of how many intents touch the file. +- **Synthetic-span filter**: `_reground_ungrounded` writes + placeholder source_spans where `text == intent.description` to + trigger lazy grounding. Both query paths filter those out so the + excerpt always reflects the original meeting passage, never the + bookkeeping placeholder. + +### Tests + +- 4 new cases in `tests/test_v0414_source_excerpt.py`: + - search response surfaces source_excerpt + meeting_date + - brief response surfaces source_excerpt + meeting_date + - drift response surfaces source_excerpt + meeting_date (via the + follow-up batched query) + - empty source_span text → empty source_excerpt (graceful, no + leak from synthetic reground spans) + +### Migration + +No schema changes. `source_span.text` and `source_span.meeting_date` +were already stored at ingest. v0.4.14 just plumbs them through to +the read responses. Pre-v0.4.14 clients that ignore `source_excerpt` +and `meeting_date` see no change. + ## 0.4.13 — 2026-04-14 — Content-Addressable Dedup (Team Mode Hardening) Closes the team-mode dedup gap. Previously, when two developers diff --git a/contracts.py b/contracts.py index 96e04351..3d776748 100644 --- a/contracts.py +++ b/contracts.py @@ -71,6 +71,12 @@ class DecisionMatch(BaseModel): code_regions: list[CodeRegionSummary] drift_evidence: str = "" related_constraints: list[str] = [] + # v0.4.14: meeting context — the raw passage from the source that + # produced this decision, plus the meeting date if known. Pulled + # from source_span.text via the yields reverse edge. Empty when + # the source_span has no text or no link to this intent. + source_excerpt: str = "" + meeting_date: str = "" class LinkCommitResponse(BaseModel): @@ -157,6 +163,9 @@ class DriftEntry(BaseModel): lines: tuple[int, int] drift_evidence: str = "" source_ref: str + # v0.4.14: meeting context tied to this decision (see DecisionMatch). + source_excerpt: str = "" + meeting_date: str = "" class DetectDriftResponse(BaseModel): @@ -285,6 +294,9 @@ class BriefDecision(BaseModel): code_regions: list[CodeRegionSummary] = [] severity_tier: int = 1 # 1=L1, 2=L2, 3=L3 — populated by v0.4.7 severity config drift_evidence: str = "" + # v0.4.14: meeting context tied to this decision (see DecisionMatch). + source_excerpt: str = "" + meeting_date: str = "" class BriefGap(BaseModel): diff --git a/handlers/brief.py b/handlers/brief.py index ef82405f..bcc5f695 100644 --- a/handlers/brief.py +++ b/handlers/brief.py @@ -262,6 +262,9 @@ def _to_brief_decision(m: DecisionMatch) -> BriefDecision: ], severity_tier=1, # v0.4.6: no severity config, all decisions default L1 drift_evidence=m.drift_evidence, + # v0.4.14: forward meeting context from the matched DecisionMatch + source_excerpt=m.source_excerpt, + meeting_date=m.meeting_date, ) diff --git a/handlers/detect_drift.py b/handlers/detect_drift.py index f6c624ae..d314ad91 100644 --- a/handlers/detect_drift.py +++ b/handlers/detect_drift.py @@ -58,6 +58,8 @@ async def handle_detect_drift( lines=tuple(region.get("lines", (0, 0))), drift_evidence=drift_evidence, source_ref=d.get("source_ref", ""), + source_excerpt=d.get("source_excerpt", ""), + meeting_date=d.get("meeting_date", ""), )) source = "working_tree" if use_working_tree else "HEAD" diff --git a/handlers/search_decisions.py b/handlers/search_decisions.py index 093afe71..240b6913 100644 --- a/handlers/search_decisions.py +++ b/handlers/search_decisions.py @@ -62,6 +62,8 @@ async def handle_search_decisions( code_regions=regions, drift_evidence=m.get("drift_evidence", ""), related_constraints=m.get("related_constraints", []), + source_excerpt=m.get("source_excerpt", ""), + meeting_date=m.get("meeting_date", ""), )) ungrounded_count = sum(1 for m in matches if m.status == "ungrounded") diff --git a/ledger/queries.py b/ledger/queries.py index 976b0870..a936abdb 100644 --- a/ledger/queries.py +++ b/ledger/queries.py @@ -176,7 +176,14 @@ async def search_by_bm25( max_results: int = 10, min_confidence: float = 0.5, ) -> list[dict]: - """BM25 search on intent.description.""" + """BM25 search on intent.description. + + v0.4.14: also pulls ``source_span.text`` (raw passage) + ``meeting_date`` + via the ``yields`` reverse edge so callers can render the meeting + excerpt that produced the decision. The first source_span linked to + the intent is used; multiple spans (same decision mentioned in + multiple meetings) collapse to the earliest one for stability. + """ rows = await client.query( """ SELECT @@ -193,16 +200,14 @@ async def search_by_bm25( end_line, purpose, content_hash - } AS code_regions + } AS code_regions, + <-yields<-source_span.{text, meeting_date} AS source_spans FROM intent WHERE description @0@ $query LIMIT $n """, {"query": query, "n": max_results}, ) - # @0@ already filtered to matching documents. - # Assign position-based confidence (1.0 for first match, decreasing). - # Note: embedded SurrealDB v2 always returns search::score=0.0 — use count instead. total = len(rows) for i, row in enumerate(rows): ca = row.pop("created_at", None) @@ -211,6 +216,21 @@ async def search_by_bm25( for region in (row.get("code_regions") or []): if region and "symbol_name" in region: region["symbol"] = region.pop("symbol_name") + # v0.4.14: collapse source_spans → top-level source_excerpt + meeting_date. + # Filter out: + # - empty-text spans (placeholder rows from ingests with no passage) + # - synthetic spans created by _reground_ungrounded, which writes + # text=description as a placeholder. Those aren't real meeting + # context; the original ingest's span is. + spans = row.pop("source_spans", None) or [] + description = row.get("description", "") + real_spans = [ + s for s in spans + if s and s.get("text") and s.get("text") != description + ] + first_span = real_spans[0] if real_spans else None + row["source_excerpt"] = (first_span.get("text") if first_span else "") or "" + row["meeting_date"] = (first_span.get("meeting_date") if first_span else "") or "" return _normalize_decisions(rows) @@ -294,7 +314,13 @@ async def get_decisions_for_file( client: LedgerClient, file_path: str, ) -> list[dict]: - """Reverse traversal: code_region → symbol → intent for a given file.""" + """Reverse traversal: code_region → symbol → intent for a given file. + + v0.4.14: also pulls ``source_excerpt`` + ``meeting_date`` per intent + via a follow-up batch query against the ``yields`` reverse edge so + the drift handler can render the meeting passage that produced + each decision. + """ rows = await client.query( """ SELECT @@ -322,6 +348,7 @@ async def get_decisions_for_file( # Flatten: one row per (region, intent) pair results = [] seen_intent_ids: set[str] = set() + intent_id_set: set[str] = set() for region_row in rows: region = { "file_path": region_row.get("file_path", ""), @@ -337,16 +364,61 @@ async def get_decisions_for_file( if iid in seen_intent_ids: continue seen_intent_ids.add(iid) + intent_id_set.add(iid) results.append({ "intent_id": iid, "description": intent.get("description", ""), "source_type": intent.get("source_type", ""), "source_ref": intent.get("source_ref", ""), + "source_excerpt": "", + "meeting_date": "", "speaker": "", "ingested_at": str(intent.get("created_at", "")), "status": intent.get("status", "ungrounded"), "code_region": region, }) + + # v0.4.14: backfill source_excerpt + meeting_date via yields reverse edge. + # Single batched query keeps this O(1) extra DB roundtrip regardless of + # how many intents matched the file. Compares stringified IDs because + # SurrealDB's bind for record-ref IN clause is finicky in embedded mode. + if intent_id_set: + excerpt_rows = await client.query( + """ + SELECT + type::string(id) AS intent_id, + <-yields<-source_span.{text, meeting_date} AS source_spans + FROM intent + WHERE type::string(id) IN $ids + """, + {"ids": list(intent_id_set)}, + ) + excerpt_by_intent: dict[str, tuple[str, str]] = {} + # Build a description lookup so we can filter synthetic + # _reground_ungrounded spans (text == description) the same way + # search_by_bm25 does. + desc_by_intent = {e["intent_id"]: e.get("description", "") for e in results} + for r in (excerpt_rows or []): + iid = str(r.get("intent_id", "")) + desc = desc_by_intent.get(iid, "") + spans = r.get("source_spans") or [] + real_spans = [ + s for s in spans + if s and s.get("text") and s.get("text") != desc + ] + first = real_spans[0] if real_spans else None + if first: + excerpt_by_intent[iid] = ( + str(first.get("text") or ""), + str(first.get("meeting_date") or ""), + ) + for entry in results: + iid = entry["intent_id"] + if iid in excerpt_by_intent: + excerpt, mdate = excerpt_by_intent[iid] + entry["source_excerpt"] = excerpt + entry["meeting_date"] = mdate + return results diff --git a/pyproject.toml b/pyproject.toml index ec927c0c..b487473c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "bicameral-mcp" -version = "0.4.13" +version = "0.4.14" description = "Decision ledger MCP server — ingests meeting transcripts, maps decisions to code, tracks drift" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_v0413_canonical_dedup.py b/tests/test_v0413_canonical_dedup.py index 6a41f792..9abad717 100644 --- a/tests/test_v0413_canonical_dedup.py +++ b/tests/test_v0413_canonical_dedup.py @@ -193,6 +193,7 @@ async def test_upsert_intent_collapses_whitespace_variant(monkeypatch, surreal_u intent row, dedupe via canonical_id.""" monkeypatch.setenv("USE_REAL_LEDGER", "1") monkeypatch.setenv("SURREAL_URL", surreal_url) + reset_ledger_singleton() ledger = get_ledger() if hasattr(ledger, "connect"): @@ -252,6 +253,7 @@ async def test_upsert_intent_distinguishes_real_differences(monkeypatch, surreal """Different decisions on the same source produce different rows.""" monkeypatch.setenv("USE_REAL_LEDGER", "1") monkeypatch.setenv("SURREAL_URL", surreal_url) + reset_ledger_singleton() ledger = get_ledger() if hasattr(ledger, "connect"): diff --git a/tests/test_v0414_source_excerpt.py b/tests/test_v0414_source_excerpt.py new file mode 100644 index 00000000..8dd6db91 --- /dev/null +++ b/tests/test_v0414_source_excerpt.py @@ -0,0 +1,224 @@ +"""v0.4.14 — source excerpt + meeting_date plumbing tests. + +Verifies that the meeting context bicameral stores at ingest time +(source_span.text + source_span.meeting_date) is now surfaced in the +brief/drift/search read paths via the yields reverse edge. + +Pre-v0.4.14 the data was in the ledger but the handlers stripped it +out. The v0.4.14 fix pulls it back through into: + + - DecisionMatch.source_excerpt + meeting_date + - DriftEntry.source_excerpt + meeting_date + - BriefDecision.source_excerpt + meeting_date + +Tests: + 1. Ingest a payload with span.text + meeting_date populated, then + search/brief/drift back and verify the fields come through. + 2. Ingest with empty span.text — fields default to "" (graceful). + 3. Multiple intents per file — each intent's excerpt is its own. +""" + +from __future__ import annotations + +import pytest + +from adapters.ledger import get_ledger, reset_ledger_singleton +from context import BicameralContext +from handlers.brief import handle_brief +from handlers.detect_drift import handle_detect_drift +from handlers.search_decisions import handle_search_decisions + + +@pytest.mark.phase2 +@pytest.mark.asyncio +async def test_search_response_includes_source_excerpt(monkeypatch, surreal_url): + """search response surfaces source_span.text and meeting_date.""" + monkeypatch.setenv("USE_REAL_LEDGER", "1") + monkeypatch.setenv("SURREAL_URL", surreal_url) + reset_ledger_singleton() # isolate from prior test's ledger state + + ledger = get_ledger() + if hasattr(ledger, "connect"): + await ledger.connect() + + payload = { + "query": "rate limiting strategy", + "repo": "test-repo", + "mappings": [ + { + "span": { + "span_id": "span-1", + "source_type": "transcript", + "text": ( + "Alex: I think we should use a token bucket rate " + "limiter on the checkout endpoint, capped at 100 " + "requests per minute per IP." + ), + "source_ref": "sprint-13-arch-review", + "meeting_date": "2026-03-30", + }, + "intent": "Use token bucket rate limiter on checkout, 100 RPM per IP", + "symbols": [], + "code_regions": [], + } + ], + } + await ledger.ingest_payload(payload) + + ctx = BicameralContext.from_env() + response = await handle_search_decisions( + ctx, query="token bucket rate limit", max_results=5, min_confidence=0.3, + ) + assert response.matches, "Expected at least one match for the ingested decision" + match = response.matches[0] + assert "token bucket" in match.source_excerpt.lower(), ( + f"source_excerpt should contain the meeting passage; got {match.source_excerpt!r}" + ) + assert "Alex:" in match.source_excerpt, ( + "speaker prefix should be preserved in the raw passage" + ) + assert match.meeting_date == "2026-03-30", ( + f"meeting_date should round-trip; got {match.meeting_date!r}" + ) + + +@pytest.mark.phase2 +@pytest.mark.asyncio +async def test_brief_response_includes_source_excerpt(monkeypatch, surreal_url): + """brief.decisions[i] surfaces source_excerpt via _to_brief_decision.""" + monkeypatch.setenv("USE_REAL_LEDGER", "1") + monkeypatch.setenv("SURREAL_URL", surreal_url) + reset_ledger_singleton() # isolate from prior test's ledger state + + ledger = get_ledger() + if hasattr(ledger, "connect"): + await ledger.connect() + + payload = { + "query": "session caching", + "repo": "test-repo", + "mappings": [ + { + "span": { + "span_id": "span-2", + "source_type": "slack", + "text": ( + "Brian: let's cache user sessions in Redis for " + "horizontal scaling — local memory will break " + "when we add the second worker." + ), + "source_ref": "slack:payments:1726113809330439", + "meeting_date": "2026-04-02", + }, + "intent": "Cache user sessions in Redis for horizontal scaling", + "symbols": [], + "code_regions": [], + } + ], + } + await ledger.ingest_payload(payload) + + ctx = BicameralContext.from_env() + brief = await handle_brief(ctx, topic="session caching Redis") + assert brief.decisions, "Expected at least one matched decision in brief" + decision = brief.decisions[0] + assert "Redis" in decision.source_excerpt, ( + f"source_excerpt should contain the meeting passage; got {decision.source_excerpt!r}" + ) + assert decision.meeting_date == "2026-04-02" + + +@pytest.mark.phase2 +@pytest.mark.asyncio +async def test_empty_source_excerpt_is_graceful(monkeypatch, surreal_url): + """Ingest with empty span.text → response has empty source_excerpt + (no crash, no KeyError).""" + monkeypatch.setenv("USE_REAL_LEDGER", "1") + monkeypatch.setenv("SURREAL_URL", surreal_url) + reset_ledger_singleton() # isolate from prior test's ledger state + + ledger = get_ledger() + if hasattr(ledger, "connect"): + await ledger.connect() + + payload = { + "query": "empty span test", + "repo": "test-repo", + "mappings": [ + { + "span": { + "span_id": "span-3", + "source_type": "manual", + "text": "", # explicitly empty + "source_ref": "manual-entry-1", + }, + "intent": "Empty span test decision", + "symbols": [], + "code_regions": [], + } + ], + } + await ledger.ingest_payload(payload) + + ctx = BicameralContext.from_env() + response = await handle_search_decisions( + ctx, query="empty span test", max_results=5, min_confidence=0.3, + ) + assert response.matches + assert response.matches[0].source_excerpt == "" + assert response.matches[0].meeting_date == "" + + +@pytest.mark.phase2 +@pytest.mark.asyncio +async def test_drift_entry_carries_source_excerpt(monkeypatch, surreal_url): + """DriftEntry from detect_drift includes source_excerpt + meeting_date.""" + monkeypatch.setenv("USE_REAL_LEDGER", "1") + monkeypatch.setenv("SURREAL_URL", surreal_url) + reset_ledger_singleton() # isolate from prior test's ledger state + + ledger = get_ledger() + if hasattr(ledger, "connect"): + await ledger.connect() + + payload = { + "query": "discount logic", + "repo": "test-repo", + "mappings": [ + { + "span": { + "span_id": "span-4", + "source_type": "transcript", + "text": ( + "Alex: discounts are 10% on orders of $100 or more. " + "Below that, no discount." + ), + "source_ref": "sprint-14-planning", + "meeting_date": "2026-03-12", + }, + "intent": "10% discount on orders over $100", + "symbols": ["calculate_discount"], + "code_regions": [ + { + "file_path": "src/pricing/discount.py", + "symbol": "calculate_discount", + "type": "function", + "start_line": 1, + "end_line": 4, + } + ], + } + ], + } + await ledger.ingest_payload(payload) + + ctx = BicameralContext.from_env() + drift = await handle_detect_drift( + ctx, file_path="src/pricing/discount.py", use_working_tree=False, + ) + assert drift.decisions, "Expected at least one decision from detect_drift" + entry = drift.decisions[0] + assert "10%" in entry.source_excerpt or "$100" in entry.source_excerpt, ( + f"source_excerpt should contain the meeting passage; got {entry.source_excerpt!r}" + ) + assert entry.meeting_date == "2026-03-12"