diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3a55015f5..399894584 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -201,9 +201,11 @@ it fetches a genuine access token from a live Keycloak, verifies the allow/deny ABAC boundary against a throwaway migrated Postgres database (a private post scoped to a *different* corporate entity is proven excluded from the list and 403s on direct fetch), and proves a forged -token is rejected. Its dev-only FastAPI `TestClient` uses Starlette with the -project's official `httpx` dev dependency; no alternate transport package is -introduced. `scripts/seed_demo_data.py` populates the docker-compose +token is rejected. Its dev-only FastAPI `TestClient` uses Starlette's +supported `httpx2` transport alongside the project's official `httpx` +dependency. The transport package is dev-only and exists solely for the +current Starlette integration contract; production runtime dependencies remain +unchanged. `scripts/seed_demo_data.py` populates the docker-compose stack itself with the same shape of synthetic data for manual/frontend use. `CORSMiddleware` (`backend/app/main.py`) allows exactly the frontend's origin(s) (`FRONTEND_ORIGINS`), `GET` and `POST` (the extract-keymen diff --git a/CHANGELOG.d/2.20.0-backend-contract-regressions.md b/CHANGELOG.d/2.20.0-backend-contract-regressions.md new file mode 100644 index 000000000..a391d991b --- /dev/null +++ b/CHANGELOG.d/2.20.0-backend-contract-regressions.md @@ -0,0 +1,3 @@ +### Fixed + +- Restored the runtime-only TEPP credential setting, kept API integration fixtures collision-free, aligned asynchronous Ask tests with semantic retrieval, replaced deprecated FastAPI 422 constants, and moved Starlette integration tests to its supported `httpx2` transport. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cf0eb0f8..641306055 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ All notable changes to this project are documented here. Format follows ### Added +- Persist explicit paragraph, list, table, MathML formula, and caller-parsed + conversation-turn semantic-unit kinds without inferring absent boundaries. - Event Lineage now persists each reconstructed connection's independent channel scores, the normalized weights actually used, and their contributions. The Event Lineage DAG discloses those exact values as inferred diff --git a/backend/app/config.py b/backend/app/config.py index 4dba383e8..827441648 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -59,6 +59,7 @@ class Settings: valkey_url: str searxng_base_url: str tepp_transport_url: str + tepp_api_key: str caldav_base_url: str naruon_calendar_base_url: str naruon_calendar_service_token: str @@ -171,6 +172,7 @@ def load_settings() -> Settings: valkey_url=os.environ.get("VALKEY_URL", "redis://localhost:16379/0"), searxng_base_url=os.environ.get("SEARXNG_BASE_URL", ""), tepp_transport_url=os.environ.get("TEPP_TRANSPORT_URL", ""), + tepp_api_key=os.environ.get("TEPP_API_KEY", ""), caldav_base_url=os.environ.get("CALDAV_BASE_URL", "").strip(), naruon_calendar_base_url=os.environ.get("NARUON_CALENDAR_BASE_URL", "").strip(), naruon_calendar_service_token=os.environ.get( diff --git a/backend/app/main.py b/backend/app/main.py index b53907977..08ff32428 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1230,7 +1230,7 @@ async def resolve_customer_master_hint( ) from exc if resolution is None: raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, + status.HTTP_422_UNPROCESSABLE_CONTENT, "this hint could not be resolved to a corroborated organization name", ) return resolution @@ -1589,7 +1589,7 @@ async def read_post( as_of_clock = parse_as_of_clock(as_of) except ValueError as exc: raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, + status.HTTP_422_UNPROCESSABLE_CONTENT, "as_of must be an ISO-8601 timestamp. Use the run cutoff, " "then compare the known body with the live body.", ) from exc @@ -2213,7 +2213,7 @@ async def read_ontology_neighborhood( try: cutoff_clock = parse_as_of_clock(knowledge_cutoff) except ValueError as exc: - raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(exc)) from exc + raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, str(exc)) from exc try: async with pool.acquire() as conn: neighborhood = await visible_ontology_neighborhood( @@ -2629,7 +2629,7 @@ async def compare_period_groupings( try: parse_period_code(period_code) except ValueError as exc: - raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(exc)) from exc + raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, str(exc)) from exc async with pool.acquire() as conn: rows = await fetch_period_comparison(conn, period_code) demo_entity_ids: set[str] = set() @@ -2685,7 +2685,7 @@ async def list_period_reports( """Available calibrated periods for one grouping kind (FIPC trend).""" _require_post_read(account) if grouping_kind not in GROUPING_KINDS: - raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "unknown grouping_kind") + raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, "unknown grouping_kind") async with pool.acquire() as conn: summaries = await list_period_report_summaries(conn, grouping_kind) demo_entity_ids: set[str] = set() @@ -2715,11 +2715,11 @@ async def read_period_reports( """Calibrated IRT scores for one grouping kind and calendar period.""" _require_post_read(account) if grouping_kind not in GROUPING_KINDS: - raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "unknown grouping_kind") + raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, "unknown grouping_kind") try: parse_period_code(period_code) except ValueError as exc: - raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(exc)) from exc + raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, str(exc)) from exc async with pool.acquire() as conn: reports = await fetch_period_reports(conn, grouping_kind, period_code) demo_entity_ids: set[str] = set() @@ -2792,11 +2792,11 @@ async def rebuild_period_report_endpoint( """Refit or FIPC-score every group in the period. post_admin only.""" _require_post_admin(account) if grouping_kind not in GROUPING_KINDS: - raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "unknown grouping_kind") + raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, "unknown grouping_kind") try: parse_period_code(period_code) except ValueError as exc: - raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(exc)) from exc + raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, str(exc)) from exc async with pool.acquire() as conn: async with conn.transaction(): reports = await rebuild_period_reports(conn, grouping_kind, period_code) @@ -3001,7 +3001,7 @@ async def chat_about_post( question = request.question.strip() if not question: raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, "question is required" + status.HTTP_422_UNPROCESSABLE_CONTENT, "question is required" ) post = await _load_visible_post(post_id, account, pool) post_metadata = build_post_llm_metadata(post_id, post) @@ -3594,7 +3594,7 @@ async def read_calendar( _require_post_read(account) if (window_start is None) ^ (window_end is None): raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, + status.HTTP_422_UNPROCESSABLE_CONTENT, "window_start and window_end must be supplied together", ) settings = load_settings() diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index df1744dd2..45ed5d9d4 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -28,6 +28,7 @@ from lineageweave.http_client import HttpClientError, get_json, post_form from lineageweave.knowledge_graph import knowledge_graph_edges_for_post +from lineageweave.post_chat import ChatSourceDocument from lineageweave.post_summary import POST_SUMMARY_CONTRACT_VERSION _POSTGRES_ADMIN_DSN = os.environ.get( @@ -568,7 +569,7 @@ def _seed_analysis_run( ) other_account_id = cur.fetchone()[0] visible_run_id = _seed_analysis_run( - "a" * 64, + "0" * 64, "visible-own-corp", account_id, "analysis_scope_corporate_entity", @@ -3900,7 +3901,9 @@ def answer(self, question: str, sources) -> None: ) assert response.status_code == 503 - assert "no complete evidence object" in response.json()["detail"] + assert response.json()["detail"] == ( + "Post chat is temporarily unavailable. Saved evidence is still available." + ) def test_live_chat_provider_error_does_not_leak_raw_error( @@ -3943,6 +3946,14 @@ class _FailingAskClient: def answer(self, question: str, sources) -> object: raise Exception("raw-global-provider-secret") + async def _source(*_args, **_kwargs): + return [ + ChatSourceDocument( + seeded_db["own_private_post_id"], "Authorized source", "Evidence" + ) + ] + + monkeypatch.setattr("backend.app.global_ask_queue.gather_global_chat_sources", _source) monkeypatch.setattr("backend.app.main._post_chat_client", lambda **_kwargs: _FailingAskClient()) headers = {"Authorization": f"Bearer {demo_analyst_token}"} @@ -5093,6 +5104,14 @@ def answer(self, question, sources): # noqa: ARG002 - contract shape cited_post_ids=(sources[0].post_id,), ) + async def _source(*_args, **_kwargs): + return [ + ChatSourceDocument( + seeded_db["own_private_post_id"], "Authorized source", "Evidence" + ) + ] + + monkeypatch.setattr("backend.app.global_ask_queue.gather_global_chat_sources", _source) monkeypatch.setattr("backend.app.main._post_chat_client", lambda **_kwargs: _FakeChatClient()) headers = {"Authorization": f"Bearer {demo_analyst_token}"} submitted = client.post( diff --git a/backend/tests/test_config.py b/backend/tests/test_config.py index 2a80f1fa4..e3c182a0b 100644 --- a/backend/tests/test_config.py +++ b/backend/tests/test_config.py @@ -41,7 +41,10 @@ def test_tepp_transport_url_defaults_empty_and_is_not_a_score(monkeypatch) -> No monkeypatch.delenv("TEPP_TRANSPORT_URL", raising=False) assert load_settings().tepp_transport_url == "" monkeypatch.setenv("TEPP_TRANSPORT_URL", "https://tepp.example/v1/analysis-runs") - assert load_settings().tepp_transport_url == "https://tepp.example/v1/analysis-runs" + monkeypatch.setenv("TEPP_API_KEY", "runtime-only-secret") + settings = load_settings() + assert settings.tepp_transport_url == "https://tepp.example/v1/analysis-runs" + assert settings.tepp_api_key == "runtime-only-secret" def test_keyverse_issuer_overrides_local_keycloak_and_uses_oidc_discovery(monkeypatch) -> None: diff --git a/docs/adr/0223-explicit-semantic-content-unit-kinds.md b/docs/adr/0223-explicit-semantic-content-unit-kinds.md new file mode 100644 index 000000000..cfc325c21 --- /dev/null +++ b/docs/adr/0223-explicit-semantic-content-unit-kinds.md @@ -0,0 +1,43 @@ +# ADR 0223: Explicit semantic content unit kinds + +**Status:** Accepted +**Date:** 2026-08-26 + +## Context + +PRD-FR-4 requires ordered paragraph, list, table, formula, +conversation-turn, and image-region semantic units. The source parser already +kept those boundaries, but `post_content_unit.unit_kind_code` collapsed every +textual DOM unit to `dom` and every markup-free unit to `plain_text`. A stored +row therefore could not disclose which source boundary produced its embedding. + +## Decision + +1. PostgreSQL admits `paragraph`, `list`, `table`, `formula`, and + `conversation_turn` as governed `post_content_unit_kind` values. Existing + `plain_text`, `dom`, and `image` values remain valid historical values. +2. New writes classify only explicit source boundaries: paragraph/plain-text + chunks, `li`, table rows, top-level MathML `math`, and caller-parsed + conversation turns. Unknown DOM blocks remain `dom`; no prose pattern or + model guess manufactures a kind. +3. MathML is retained as one ordered formula boundary. This decision does not + parse, evaluate, or assign mathematical meaning to the expression. +4. Image regions remain normalized children of their document-order image + unit under ADR 0091 rather than duplicating them as top-level content units. +5. A source adapter may pass already parsed `Chunk` units to persistence. + LineageWeave does not infer RFC 5322 sender boundaries from an opaque body. + +## Consequences + +- Embedding rows remain attached to the same ordered source unit while their + stored kind becomes inspectable and stable. +- Existing rows are not rewritten, so provenance is preserved. +- Formula evaluation and formula ontology remain outside LineageWeave. + +## References + +World Wide Web Consortium. (2025). *MathML Core* (Candidate Recommendation +Snapshot, June 24, 2025). https://www.w3.org/TR/2025/CR-mathml-core-20250624/ + +Resnick, P. W. (Ed.). (2008). *Internet message format* (RFC 5322). Internet +Engineering Task Force. https://www.rfc-editor.org/rfc/rfc5322 diff --git a/docs/adr/README.md b/docs/adr/README.md index 88ffb3abc..eadef2874 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -11,7 +11,7 @@ decision from them. |---|---| | [`product-requirements.md`](../product-requirements.md) | Product requirements projection across the ADR set; ADRs remain normative | | [`product-technical-gap-baseline.md`](../product-technical-gap-baseline.md) | Product/technical traceability projection across the ADR set; ADRs remain normative | -| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0024](0024-rankweave-fusion-fail-closed.md), [0165](0165-quantity-script-display.md), [0167](0167-rankweave-ranking-channel-evidence.md), [0169](0169-ask-batched-lineage-graph.md), [0172](0172-event-lineage-channel-evidence.md), [0202](0202-ask-event-time-filter.md) | +| [`lineage-bi-research-notes.md`](../lineage-bi-research-notes.md) | [0084](0084-lineage-research-grounding.md), [0062](0062-semantic-unit-embedding.md), [0064](0064-lineage-evidence-and-tree-assembly.md), [0024](0024-rankweave-fusion-fail-closed.md), [0165](0165-quantity-script-display.md), [0167](0167-rankweave-ranking-channel-evidence.md), [0169](0169-ask-batched-lineage-graph.md), [0172](0172-event-lineage-channel-evidence.md), [0202](0202-ask-event-time-filter.md), [0223](0223-explicit-semantic-content-unit-kinds.md) | | [`PROV_O_IMPLEMENTATION.md`](../PROV_O_IMPLEMENTATION.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`PROV_O_IMPLEMENTATION_MATRIX.md`](../PROV_O_IMPLEMENTATION_MATRIX.md) | [0065](0065-prov-o-provenance-boundary.md) | | [`ONTOLOGY_NAMESPACE_INVENTORY.md`](../doctoring/ONTOLOGY_NAMESPACE_INVENTORY.md) | [0207](0207-repository-case-ontology-namespace-canonical.md), [0157](0157-public-ontology-namespace-identity.md) | diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 9a65c2eb6..aa1bcf250 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,7 +1,7 @@ # Product & Technical Gap Baseline -> Dashboard delivery snapshot: 2026-08-25 21:34 KST. Protected `main` was -> `d7d5eeb310b055b5e138060cf2dfb929b03090a6`. This local branch is not +> Dashboard delivery snapshot: 2026-08-26 05:01 KST. Protected `main` was +> `04e6b610655d0db91d5f7ba9486bdda1440e0b19`. This local branch is not > protected-main release evidence. ## Operations Dashboard PRD/TRD traceability @@ -60,18 +60,14 @@ only aggregate, non-identifying evidence to this repository. ### Exact open-PR boundary -At this snapshot there were 3 open PRs and 11 open issues. Exact observed heads -were `#628 d07d212f` (this branch's observed parent), `#627 9e0528a6`, and -`#579 1c209c85`. PR #579 is open; its ADR 0211 reservation is why this branch's -filter-option decision is ADR 0212. PRs #612, #614, #615, #616, and #626 -reached protected `main`; the superseded baseline PR #613 closed without merge -and its PRD was recreated on protected main. The open heads remain blocked on -hosted gates and/or independent review. These -observations are not merge readiness. Re-fetch exact heads, +At this snapshot there were 16 open PRs and 10 open issues. The exact-head +inventory in section 1 is authoritative for this snapshot. Every open head +remained blocked on hosted gates and/or independent review. These observations +are not merge readiness. Re-fetch exact heads, unresolved threads, checks, approvals, rulesets, and merge SHA before any lifecycle claim. -> Audit snapshot: 2026-08-25 21:34 KST (refreshed by the autonomous merge +> Audit snapshot: 2026-08-26 05:01 KST (refreshed by the autonomous merge > loop). This repository records synthetic fixtures and aggregate, > non-identifying runtime evidence only. Open PRs and local checks are not > protected-default-branch release evidence. Identifying post identifiers, @@ -80,23 +76,40 @@ lifecycle claim. ## 1. Exact-head and governance evidence -The protected default branch was `d7d5eeb310b055b5e138060cf2dfb929b03090a6` -when this baseline was refreshed. The live queue contained 3 open PRs and 11 +The protected default branch was `04e6b610655d0db91d5f7ba9486bdda1440e0b19` +when this baseline was refreshed. The live queue contained 16 open PRs and 10 open issues. The exact-head inventory below supersedes older per-PR snapshots elsewhere in this document; those older rows remain useful historical delivery context only. | PR | Exact observed head | Merge/check state at this snapshot | | ---: | --- | --- | -| #628 | `d07d212f` (observed parent) | this row is updated by #628 itself, so its exact head advances after the snapshot is encoded; ADR 0212 combines complete ABAC-visible filter options into one database round trip, while hosted gates and independent review remain required | -| #627 | `9e0528a6` | repairs k6 lifecycle evidence preservation; hosted gates remain required | -| #579 | `1c209c85` | persists leftover interaction-map coordinates and owns ADR 0211; hosted gates and independent review remain required | +| #663 | `be361f10` | project ontology traversal plus cutoff/snapshot-frozen project focus and labels | +| #662 | `92534118` | fail-closed injectable TEPP status/read boundary; the owning executable HTTP route remains unavailable | +| #661 | `11206c58` | Rust-estimated reconstruction fixture weights replace hand-authored reconstruction-test dictionaries | +| #660 | `c10cae01` (observed parent) | backend runtime repair plus the #664 semantic-unit stack; this documentation commit advances the head after capture | +| #659 | `e948bd27` | ontology node readability and tokenized UI fills | +| #658 | `fe830b0a` | evidence-honest Global Ask cutoff with revision-interval live-after semantics | +| #657 | `64f48679` | Dashboard case-metric contract work | +| #644 | `d9ff9980` | native-surface code splitting with modal-focus regression coverage | +| #643 | `0a1f8ec1` | accessible status-notice surfaces | +| #640 | `2d50fa01` | operations-dashboard contract alignment | +| #639 | `aee02dca` | terminal checks observed; exact-head independent approval still required | +| #636 | `f7b9a65f` | terminal checks observed; exact-head independent approval still required | +| #632 | `32c7d359` | active semantic provenance repair head; hosted checks and independent review required | +| #631 | `c0022c97` | terminal checks observed; exact-head independent approval still required | +| #629 | `4b4d6707` | terminal checks observed; exact-head independent approval still required | +| #579 | `689a21b6` | leftover interaction-map coordinate persistence; exact-head independent approval still required | No row above is merge evidence. Immediately before any lifecycle action, re-fetch the head, unresolved threads, formal reviews, rulesets, and same-head check conclusions. In particular, queued checks are infrastructure state and do not transfer evidence from an earlier SHA. +PR #664 merged as `b2e48d5b0db59f5aa434e2a293cd182ee810c019` +into #660's non-default branch. Its semantic-unit implementation is therefore +stack evidence only until #660 passes the protected-`main` gate. + PR #607 first merged as `61fd631c7bb3c57113fd19763c2c43161eeb2824` into #606's non-default branch. PR #606 subsequently passed the protected gate, so the combined TEPP-consumer and operations-dashboard implementation is now @@ -369,7 +382,7 @@ this file per §3.5 of the prior snapshot). | Authorized-corpus runtime | Repository tests use synthetic fixtures; private records remain outside git | Authenticated runtime validation returning only aggregate, non-identifying evidence | | Concurrent web responsiveness | ADR 0204 releases pooled transactions during provider work, and the synthetic Compose boundary has an authenticated k6 E2E harness for Ask enqueue, concurrent reads, and job polling. An older-image local observation found repeated post-filter queries while `/api/posts` exceeded 30 seconds; ADR 0212 combines two authorized filter-option queries into one round trip without narrowing ABAC-visible options. The observation is not exact-head evidence or a product guarantee, and no physical scan reduction is claimed without an exact-head plan | Rebuild an exact-head application image, run `make load-http` with declared environment concurrency/window, and retain raw distributions and resource configuration. Compare the post-list database plan and latency with ADR 0212 while preserving the complete authorized filter set; set no SLO until representative capacity evidence is approved | | Image understanding | Region, OCR, and description work exists across active heads (#405, #419), but current runtime acceptance has not yet proved table-image structure, complete region coverage, or summary/image readiness together | Orchestrator-backed rendered workflow, original/derived asset provenance, region-before-OCR processing, and honest unsupported states; reconcile ADR 0052's image-bearing summary readiness with ADR 0098 before changing sequencing | -| Semantic source rendering | Paragraph, table, list, formula, and indentation work exists across stacks (#394, #427, #448–#450); #515 adds synthetic backend/frontend parity for deterministic rows/cells, footnote boundaries, and encoded scripts | Land the #427 → #515 stack, then gather authenticated browser evidence that list nesting, continuation alignment, and formula units render without authoring-layout artifacts | +| Semantic source rendering | ADR 0223 and migration 0221 give new paragraph, list, table, MathML formula, and caller-parsed conversation-turn units explicit persisted kinds without rewriting historical rows; image regions remain ordered normalized children under ADR 0091. This branch is candidate evidence, not protected-main delivery | Land the exact-head candidate, then prove an authorized semantic-only query retrieves each persisted unit kind and gather authenticated browser evidence that nesting, continuation alignment, formula units, and image regions retain source order | | Event and project semantics | Multi-project mentions, project-bound actions, 5W1H, requester/processor, and semantic relations exist in ADR 0036/0052/0100/0111/0129 and active stacks | Aggregate authenticated evidence must show distinct projects and events, explicit requester/processor and real R&R, normalized relative time, and product/entity relations without promoting attendance or co-occurrence | | Knowledge Graph readability | The black evidence-node root cause is an undefined-token fallback; the design-token repair and long-label/evidence-table coverage remain only on closed, unmerged #490, not protected `main` | Recreate the token repair on a current base and deliver it through protected `main`, then verify light/dark contrast, keyboard graph navigation, full labels, and evidence tables in the authenticated rendered surface | | Source-code lookup UX | Source state/detail codes remain evidence-bearing machine values and current detail presentation is dense | Catalog-backed display labels with raw-code provenance, compact 5W1H/source-detail hierarchy, keyboard access, and no unsupported customer/project binding | @@ -385,7 +398,7 @@ this file per §3.5 of the prior snapshot). | External integrations | Search, Zotero, calendar, Keyverse, orchestrator, RankWeave, ThreadWeave, TEPP, DiskSage, wardnet | Provider conformance, failure/reconciliation behavior, and provenance-bearing integration evidence | | MSA / modular reuse | LineageWeave must run standalone and as a consumer of org packages | Do not reimplement RankWeave/TEPP/orchestrator/ThreadWeave/Keyverse; fix upstream and PR there | | Product contract authority | This branch recreates the first LineageWeave PRD after superseded #613 closed without merge and records an exact-case ecosystem authority register; TEPP, fast-mlsirm, keyverse, and ThreadWeave have standalone PRDs, while contextual-orchestrator, RankWeave, DiskSage, and wardnet currently rely on product-planning/architecture documents and naruon has only a scoped Topic Intelligence PRD | Land the LineageWeave PRD, keep ADRs normative, and add standalone PRDs in each owning repository before making cross-product release claims beyond its documented boundary | -| Release quality | Local focused/full suites have passed on individual PR heads | Repository-wide coverage, docstrings, Storybook, security, browser, and release evidence on one exact head | +| Release quality | The #664/#660 stacked tree passed the complete Python suite (1,352 passed, 17 skipped) after #660 removed a duplicate synthetic snapshot digest and restored the TEPP settings contract. This is local candidate evidence, not protected-main delivery | Repository-wide coverage, frontend/Storybook, security, browser, and protected merge evidence on one exact head | | PII | Masking would paralyze the product; ADR 0001 forbids identifying artifacts in git | ABAC + authorized runtime; synthetic fixtures in git; no mask-in-place that drops names the operator must read | | Database | PostgreSQL, 3NF, snake_case ≥ two words, hot-partition and lock policy | No file DBs; read/write split if lock management fails; whitelist every migration | diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index e5789a1ea..edad0f59c 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -68,6 +68,7 @@ "ol", "ul", "li", + "math", "footnote", "endnote", "w:footnote", diff --git a/lineageweave/post_content_persistence.py b/lineageweave/post_content_persistence.py index 7fa97a832..5fb752cc4 100644 --- a/lineageweave/post_content_persistence.py +++ b/lineageweave/post_content_persistence.py @@ -33,6 +33,21 @@ _LOGGER = logging.getLogger(__name__) +def _persisted_unit_kind(chunk: Chunk) -> str: + """Map explicit source boundaries onto the governed semantic-unit vocabulary.""" + if chunk.unit_type in {"image", "conversation_turn"}: + return chunk.unit_type + if chunk.label == "math": + return "formula" + if chunk.label in {"tr", "w:tr"}: + return "table" + if chunk.label == "li": + return "list" + if chunk.unit_type in {"plain_text", "paragraph"} or chunk.label in {"p", "w:p"}: + return "paragraph" + return "dom" + + def _bounded_unit_batches( # noqa: UP047 - retain Python 3.10 compatibility. units: list[tuple[_BatchKey, str | dict[str, object]]], ) -> list[list[tuple[_BatchKey, str | dict[str, object]]]]: @@ -112,15 +127,18 @@ async def persist_post_content( normalized_result: Any | None = None, structure_client: PostStructureClient | None = None, post_title: str = "", + semantic_units: list[Chunk] | None = None, ) -> int: """Replace one post's normalized content artifacts and return unit count. Provider calls happen before the short database transaction. A failed or unavailable embedding call writes no vector row; it never writes a zero or guessed vector. The raw body remains in ``source_post`` for future retry. + ``semantic_units`` admits caller-parsed source boundaries such as RFC 5322 + conversation turns without inferring them from an opaque body string. """ normalized = normalized_result or normalize_post_body(body, vision_client) - chunks = chunk_by_source_body(body) + chunks = semantic_units if semantic_units is not None else chunk_by_source_body(body) image_results = {result.chunk_index: result for result in normalized.image_results} formatting = {hint.chunk_index: hint.style for hint in normalized.formatting_hints} @@ -300,7 +318,7 @@ async def persist_post_content( """, post_id, chunk.index, - chunk.unit_type, + _persisted_unit_kind(chunk), chunk.label, unit_text, style, diff --git a/migrations/0221_semantic_content_unit_kinds.sql b/migrations/0221_semantic_content_unit_kinds.sql new file mode 100644 index 000000000..16113079b --- /dev/null +++ b/migrations/0221_semantic_content_unit_kinds.sql @@ -0,0 +1,13 @@ +begin; + +insert into common_lookup_value + (lookup_category, lookup_code, lookup_label, display_order) +values + ('post_content_unit_kind', 'paragraph', 'Paragraph', 3), + ('post_content_unit_kind', 'list', 'List item', 4), + ('post_content_unit_kind', 'table', 'Table row', 5), + ('post_content_unit_kind', 'formula', 'Formula', 6), + ('post_content_unit_kind', 'conversation_turn', 'Conversation turn', 7) +on conflict (lookup_code) do nothing; + +commit; diff --git a/migrations/rollback/0221_semantic_content_unit_kinds.sql b/migrations/rollback/0221_semantic_content_unit_kinds.sql new file mode 100644 index 000000000..7387a87a2 --- /dev/null +++ b/migrations/rollback/0221_semantic_content_unit_kinds.sql @@ -0,0 +1,9 @@ +delete from common_lookup_value lookup +where lookup.lookup_category = 'post_content_unit_kind' + and lookup.lookup_code in + ('paragraph', 'list', 'table', 'formula', 'conversation_turn') + and not exists ( + select 1 + from post_content_unit unit + where unit.unit_kind_code = lookup.lookup_code + ); diff --git a/pyproject.toml b/pyproject.toml index e3b7a5b18..84a6e7566 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,10 @@ dev = [ # against lineageweave-kg-shapes.ttl (ADR 0207 decision 10). Pure # Python; OWL-RL reasoning included. "pyshacl>=0.26.0", + "httpx2>=2.12.0", ] backend = [ - "fastapi>=0.115.0", + "fastapi>=0.141.1", "uvicorn[standard]>=0.30.0", "asyncpg>=0.29.0", "pyjwt[crypto]>=2.8.0", diff --git a/tests/test_chunking.py b/tests/test_chunking.py index b7be87013..20fab16e5 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -73,6 +73,18 @@ def test_chunk_by_dom_nested_blocks_do_not_duplicate_text() -> None: assert chunks[0].label == "p" +def test_chunk_by_dom_keeps_mathml_as_a_formula_boundary() -> None: + chunks = chunk_by_dom( + "

Before.

x+1

After.

" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Before."), + ("math", "x+1"), + ("p", "After."), + ] + + def test_chunk_by_dom_groups_table_cells_by_row_instead_of_flattening() -> None: """Live bug (2026-08-19): each used to push its own independent chunk with no row grouping, so a real table (headers + N data rows) diff --git a/tests/test_migration_replay.py b/tests/test_migration_replay.py index c170f73e8..0c7fccfd2 100644 --- a/tests/test_migration_replay.py +++ b/tests/test_migration_replay.py @@ -60,6 +60,18 @@ def test_interval_relation_backfill_uses_utc_created_day() -> None: assert "created_at at time zone 'UTC'" in sql +def test_semantic_content_unit_kind_migration_is_replay_safe() -> None: + sql = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0221_semantic_content_unit_kinds.sql" + ).read_text(encoding="utf-8").lower() + + assert "on conflict (lookup_code) do nothing" in sql + for unit_kind in ("paragraph", "list", "table", "formula", "conversation_turn"): + assert f"'{unit_kind}'" in sql + + def test_interval_relation_foreign_key_validation_is_separate() -> None: """Installing the FK must not scan a large existing edge table.""" diff --git a/tests/test_post_content_persistence_edges.py b/tests/test_post_content_persistence_edges.py index 0542d9897..9a1f845fc 100644 --- a/tests/test_post_content_persistence_edges.py +++ b/tests/test_post_content_persistence_edges.py @@ -6,7 +6,7 @@ import pytest -from lineageweave.chunking import chunk_by_dom +from lineageweave.chunking import Chunk, chunk_by_dom from lineageweave.image_content import ImageRegion from lineageweave.post_content_normalization import ( FormattingHint, @@ -17,6 +17,7 @@ from lineageweave.post_content_persistence import ( _bounded_structure_batches, _bounded_unit_batches, + _persisted_unit_kind, _render_image_text, persist_post_content, ) @@ -178,6 +179,29 @@ def test_render_image_text_preserves_unavailable_and_caption_variants() -> None: ) +def test_persisted_unit_kind_uses_explicit_source_boundaries() -> None: + assert _persisted_unit_kind(Chunk("Paragraph", "plain_text", 0)) == "paragraph" + assert _persisted_unit_kind(Chunk("Item", "dom", 0, label="li")) == "list" + assert _persisted_unit_kind(Chunk("A | B", "dom", 0, label="tr")) == "table" + assert _persisted_unit_kind(Chunk("x + y", "dom", 0, label="math")) == "formula" + assert ( + _persisted_unit_kind(Chunk("Reply", "conversation_turn", 0, label="sender")) + == "conversation_turn" + ) + + +def test_persists_explicit_conversation_turn_units() -> None: + conn = _Connection() + units = [ + Chunk("Question", "conversation_turn", 0, label="sender-a"), + Chunk("Answer", "conversation_turn", 1, label="sender-b"), + ] + + assert _persist(conn, "post-id", "source body", semantic_units=units) == 2 + inserted = [args for query, args in conn.fetchvals if "insert into post_content_unit" in query] + assert [args[2] for args in inserted] == ["conversation_turn", "conversation_turn"] + + def test_persists_image_tags_formatting_and_embeddings() -> None: body = '

before

after

' chunks = chunk_by_dom(body) diff --git a/uv.lock b/uv.lock index 6190681ba..b714796ba 100644 --- a/uv.lock +++ b/uv.lock @@ -535,6 +535,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/74/d370e55600d9bcfa0d9794b0166126d49291a3d2b20c268fc98c453a4948/httpcore2-2.12.0-py3-none-any.whl", hash = "sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb", size = 83074, upload-time = "2026-08-18T13:22:05.854Z" }, +] + [[package]] name = "httptools" version = "0.8.0" @@ -586,6 +599,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/f8/579a8b51e42e38ee32647df9f08aa25643ae788e275cc625b199829c4671/httpx2-2.12.0.tar.gz", hash = "sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf", size = 100040, upload-time = "2026-08-18T13:22:09.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/95/411ba65569158e862368917aaf56597f3e5fa3b91b0502919638465a08f3/httpx2-2.12.0-py3-none-any.whl", hash = "sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36", size = 95427, upload-time = "2026-08-18T13:22:06.834Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -632,6 +671,7 @@ backend = [ dev = [ { name = "coverage" }, { name = "httpx" }, + { name = "httpx2" }, { name = "psycopg2-binary" }, { name = "pyjwt", extra = ["crypto"] }, { name = "pyshacl" }, @@ -645,8 +685,9 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.6" }, { name = "cryptography", specifier = ">=42.0" }, { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=d025b7d237d8db7ca97a5611606c6285d5870895" }, - { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.115.0" }, + { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.141.1" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.0" }, + { name = "httpx2", marker = "extra == 'dev'", specifier = ">=2.12.0" }, { name = "opentelemetry-api", specifier = ">=1.30.0" }, { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.30.0" }, { name = "opentelemetry-sdk", specifier = ">=1.30.0" }, @@ -1272,6 +1313,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/e0/ffbc0d61d68304602120998a5d660c8108464064bdedc814dc4be8410425/threadweave-0.1.0-py3-none-any.whl", hash = "sha256:03c31fa21873a9493687d81eab4ec067bf169dade7cff077b80df46fd0db3aaf", size = 14967, upload-time = "2026-07-12T03:59:57.088Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0"