diff --git a/AGENTS.md b/AGENTS.md index e704c9058..936c7f1c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -173,8 +173,12 @@ never a fabricated theta or a local psychometric substitute. Buyer Board **Weekly VOC** is an ISO-8601 week list filter (ADR 0092). Opening that filtered post focuses Event Lineage (ADR 0093). Do not invent a week, a theta, or a cutoff body. -Opening a Calendar commitment uses the same focus path (ADR 0094). Do not +Opening a Calendar commitment uses the same focus path (ADR 0134). Do not invent a week, a theta, a cutoff body, or a CalDAV event. +Opening a Customer master related post uses the same focus path (ADR 0095). +Do not invent a week, a theta, a cutoff body, a CalDAV event, or a customer. +Opening an Ask Agent cited post uses the same focus path (ADR 0096). Do not +invent a cited post. ## Tests diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 461ebacba..2c895860b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -69,6 +69,7 @@ flowchart LR | `rankweave_client.py` | Fail-closed RankWeave ranking port (`weighted_reciprocal_rank_fuse` in-process; never invent a fused score or a theta) | | `reconstruct.py` | The pipeline: group → candidate window → score → fuse → thread | | `lineage_persistence.py` | Flattens reconstruct trees into `post_lineage_edge` row specs (parent, child, fused_score) | +| `lineage_contract.py` | Versioned, store-agnostic provider boundary for bounded authorized evidence and opaque-reference results | | `knowledge_graph.py` | Random-walk-with-restart relevance + per-node adaptive related-node cutoff (Tong et al., 2006) -- pure graph math, no Postgres | | `keyman_extraction.py` | Pluggable LLM extraction of two-sided (our-side/counterparty) person mentions + N:N org affiliations from a post | | `entity_relationship_classification.py` | Pluggable LLM classification of a named organization's relationship to the post author (`rel_voc`/`rel_vom`/`rel_vop`/`rel_vocc`/`rel_voco`/`rel_vos`) | @@ -282,8 +283,10 @@ Keycloak issued; `src/App.tsx` renders a git-branch SVG of `GET /api/lineage` (click a node to open that post; `post_admin` can rebuild), the post list with a named Weekly VOC ISO-8601 week filter (ADR 0092; opening that filtered post focuses Event Lineage, ADR 0093), -Calendar commitments use the same Event Lineage focus path (ADR 0094), and -the full detail popup includes Korean +Calendar commitments use the same Event Lineage focus path (ADR 0134), +Customer master related posts use the same Event Lineage focus path +(ADR 0095). Ask Agent cited posts use the same Event Lineage focus path +(ADR 0096), and the full detail popup includes Korean summary/key-events/R&R, VOC evidence excerpts, an Event Lineage panel (direct vs. indirect links; a link opens that post), the Keyman affiliate tree (resolved ancestors plus unresolved org roots), Keyman + diff --git a/CHANGELOG.d/2.12.7-korean-search-boundary.md b/CHANGELOG.d/2.12.7-korean-search-boundary.md new file mode 100644 index 000000000..9ea0359d8 --- /dev/null +++ b/CHANGELOG.d/2.12.7-korean-search-boundary.md @@ -0,0 +1,7 @@ +# 2.12.7 — Preserve Korean search corroboration boundaries + +## Fixed + +- Accept a Korean organization token followed by a grammatical particle in a + natural Searxng snippet while still rejecting the token inside a longer + unrelated Hangul word. diff --git a/CHANGELOG.d/2.12.8-ontology-org-shacl.md b/CHANGELOG.d/2.12.8-ontology-org-shacl.md new file mode 100644 index 000000000..c8f5cb417 --- /dev/null +++ b/CHANGELOG.d/2.12.8-ontology-org-shacl.md @@ -0,0 +1,13 @@ +# 2.12.8 Organization ontology and SHACL boundaries + +The core Knowledge Graph ontology now distinguishes real organizations from +the concepts used to classify them. `CorporateEntity` and `Team` reuse W3C ORG +organization, organizational-unit, containment, and unit-membership semantics; +SKOS remains responsible for Group, Company, and Plant level concepts and +canonical/alternative labels. + +The ontology now publishes stable version/import metadata and a companion +versioned SHACL graph. Regression tests cover the ORG/SKOS boundary, direct +parent and team-owner cardinalities, and the continued relational lookup-code +round trip. PostgreSQL remains authoritative and ontology imports are never +network-dereferenced at runtime. diff --git a/CHANGELOG.d/2.15.0-customer-hierarchy-boundary.md b/CHANGELOG.d/2.15.0-customer-hierarchy-boundary.md new file mode 100644 index 000000000..9fcae3ac6 --- /dev/null +++ b/CHANGELOG.d/2.15.0-customer-hierarchy-boundary.md @@ -0,0 +1,7 @@ +# 2.15.0 — Keep unresolved customer parents non-authoritative + +## Fixed + +- A self-parent or cyclic customer relation now remains visibly unresolved in + the focus workspace instead of being presented as a valid parent. Review the + source hierarchy before using that relationship for navigation. diff --git a/CHANGELOG.d/2.15.0-customer-master-open-event-lineage.md b/CHANGELOG.d/2.15.0-customer-master-open-event-lineage.md new file mode 100644 index 000000000..99b6ba65d --- /dev/null +++ b/CHANGELOG.d/2.15.0-customer-master-open-event-lineage.md @@ -0,0 +1,11 @@ +# 2.15.0 Opening a Customer master related post focuses Event Lineage + +Customer master names authorized customer entities as current and to open a +related post to read Event Lineage. That open focuses the popup Event Lineage +heading. Home-list opens do not. No TEPP theta is invented. + +Customer Master now keeps one customer at the center of a responsive three-pane +workspace: authorized hierarchy, visible parent/direct-child relationships, and +source-backed linked evidence. Closing evidence no longer loses the selected +customer context. Desktop, tablet, and phone layouts use the shared UI tokens +and the 1024 px / 768 px responsive boundaries. diff --git a/CHANGELOG.d/2.16.0-ask-agent-open-event-lineage.md b/CHANGELOG.d/2.16.0-ask-agent-open-event-lineage.md new file mode 100644 index 000000000..d60ab9945 --- /dev/null +++ b/CHANGELOG.d/2.16.0-ask-agent-open-event-lineage.md @@ -0,0 +1,5 @@ +# 2.16.0 Opening an Ask Agent cited post focuses Event Lineage + +Ask Agent names authorized cited posts as current after an answer and to +open one to read Event Lineage. That open focuses the popup Event Lineage +heading. Home-list opens do not. No TEPP theta is invented. diff --git a/CHANGELOG.d/2.16.0-pr262-hardening.md b/CHANGELOG.d/2.16.0-pr262-hardening.md new file mode 100644 index 000000000..a8501f8ca --- /dev/null +++ b/CHANGELOG.d/2.16.0-pr262-hardening.md @@ -0,0 +1,6 @@ +### Fixed + +- Keep embedded source images visible when a post also contains a Markdown + table, and remove the privileged self-modifying contract-repair workflow. +- Align the Python package version and ontology ADR reference with the 2.16.0 + release. diff --git a/CHANGELOG.d/cross-repo-lineage-provider-contract.md b/CHANGELOG.d/cross-repo-lineage-provider-contract.md new file mode 100644 index 000000000..812117b0e --- /dev/null +++ b/CHANGELOG.d/cross-repo-lineage-provider-contract.md @@ -0,0 +1,14 @@ +# Unreleased: bounded cross-repository lineage provider contract + +LineageWeave now exposes a versioned, store-agnostic contract for authorized +Naruon-shaped evidence. It preserves opaque evidence references, separate RFC +email fields, project hints, cutoff exclusion, channel limitations, and a +deterministic request digest without reading Naruon's database or claiming +Naruon's authoritative project status. Nested email references and project +hints have explicit request-level ceilings so callers cannot bypass the +bounded-work contract with oversized collections. + +Provider failures now become an explicit unavailable-channel limitation and +rerun with renormalized non-LLM weights. Long body text is retained in the +request envelope without being flattened into the short pairwise comparison +label. diff --git a/CHANGELOG.d/customer-master-tree-projection.md b/CHANGELOG.d/customer-master-tree-projection.md new file mode 100644 index 000000000..3d3f8991f --- /dev/null +++ b/CHANGELOG.d/customer-master-tree-projection.md @@ -0,0 +1,14 @@ +### Fixed + +- Customer Master now preserves the authorized Group → Company → Plant hierarchy while promoting + missing-parent, self-parent, and cyclic relations to visible `unresolved` roots instead of silently + dropping those customers. +- Customer hierarchy rendering now follows the ontology's W3C ORG containment and separate SKOS level + classification instead of conflating organization instances with taxonomy concepts. +- Late related-post responses can no longer replace evidence for a newly selected customer entity. + +### Accessibility + +- Added a reusable WAI-ARIA customer tree with one roving focus target, branch expansion, + Arrow/Home/End navigation, Enter/Space evidence activation, exact level/position metadata, and an + independently owned source-post evidence panel. diff --git a/CHANGELOG.d/external-lineage-contract.md b/CHANGELOG.d/external-lineage-contract.md new file mode 100644 index 000000000..713cd12c5 --- /dev/null +++ b/CHANGELOG.d/external-lineage-contract.md @@ -0,0 +1,13 @@ +# External email/project lineage contract + +- Add a strict, versioned external analysis contract for future Naruon and separately governed consumer use. +- Export immutable request/result types, strict parsing, canonical serialization, deterministic digests, stable errors, and the store-agnostic `analyze_external_lineage` package entry point. +- Accept only bounded caller-authorized opaque evidence references; no provider credentials, mailbox access, persistence, provider mutation, or direct application-database integration is introduced. +- Preserve caller-observed RFC/provider/manual parent relations separately from inferred reconstructed continuation. +- Exclude caller-observed children from alternative inferred-parent scoring, optional model disclosure, and inferred-pair budget while retaining them as candidate history for later records. +- Enforce available-time knowledge cutoffs and disclose excluded evidence without substituting later facts. +- Reject explicit-parent cycles and candidate-pair work above the caller-approved limit before optional LLM/provider activity. +- Expose exact active channel scores, weights, contributions, LLM availability state, proposed project groupings, and deterministic result digests. +- Add JSON Schema Draft 2020-12, one canonical ADR 0133, APA 7th doctoring, and focused TDD coverage. +- Publish `not_invoked` in the JSON Schema and ADR when an available optional + LLM has no candidate pair to judge. diff --git a/CHANGELOG.md b/CHANGELOG.md index a3559c0b9..5241e2f6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,26 @@ All notable changes to this project are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.16.0] - 2026-08-19 + +### Added + +- Opening an Ask Agent cited post now focuses Event Lineage and names Keyman + and evaluation as the next read. After an authorized answer, Ask Agent + names cited posts as current before that open. Home-list opens do not add + that focus or copy. No TEPP theta is invented. No cited post is invented +(ADR 0096 / ADR 0039 / ADR 0016). + +## [2.15.0] - 2026-08-19 + +### Added + +- Opening a Customer master related post now focuses Event Lineage and names + Keyman and evaluation as the next read. Customer master names authorized + customer entities as current before that open. Home-list opens do not add + that focus or copy. No TEPP theta is invented. No customer is invented +(ADR 0095 / ADR 0037 / ADR 0016). + ## [2.14.0] - 2026-08-19 ### Added @@ -12,7 +32,7 @@ All notable changes to this project are documented here. Format follows and evaluation as the next read. Calendar names authorized commitments as current before that open. Home-list opens do not add that focus or copy. No TEPP theta is invented. No cutoff body is invented - (ADR 0094 / ADR 0016). + (ADR 0134 / ADR 0016). ## [2.13.0] - 2026-08-19 @@ -29,7 +49,7 @@ All notable changes to this project are documented here. Format follows - Board now names Weekly VOC as an ISO-8601 week list filter. The control keeps Voice of Customer posts for the latest week present in the loaded - list (UTC Thursday rule) and tells the buyer to open a post to read + list (UTC Thursday rule) and tells the reader to open a post to read Event Lineage. Reset filters returns every VOC type and every week. No TEPP theta is invented (ADR 0092). @@ -38,30 +58,26 @@ All notable changes to this project are documented here. Format follows ### Changed - Renamed "Buyer" terminology to reader/workspace naming across the frontend - shell, backend evidence helpers, and living docs (ADR 0119). LineageWeave - has no explicit buyer role, so `BuyerNav`/`BuyerDestination` became - `WorkspaceNav`/`WorkspaceDestination`, `.buyer-gnb*` CSS became - `.workspace-gnb*`, and prose referring to the reading user now says - "reader" instead of "buyer". Historical ADRs and changelog entries keep - their original wording as a point-in-time record. + shell, backend evidence helpers, and living docs (ADR 0119). Historical ADRs + and changelog entries retain their point-in-time wording. ### Fixed +- Preserve source-order nested list units, numeric superscript footnotes, + HTML/OOXML table rows, and recognizable Markdown table rows across the + semantic-unit parser and reader body renderer. See the [product and + technical gap baseline](docs/product-technical-gap-baseline.md) and + [ADR 0103](docs/adr/0103-semantic-document-evidence-contract.md). +- Preserve multiline VISION table rows, render parent and region OCR tables + accessibly, and request source-visible entity, relationship, layout, and + document-purpose evidence instead of a generic image caption. VISION calls + now share the structure channel's 600-second deep-agent runtime boundary; + an empty same-image retry can no longer erase previously observed OCR. - Removed the completed one-shot Global Ask package-manager repair workflow; normal product CI remains the only branch validation path. - `make smoke` and `make seed` now run through the locked project `uv` environment, so local OIDC and synthetic-data workflows resolve the same pinned dependencies as CI. -- The workspace Event Lineage global Search action now retries focus after the - board finishes loading, so navigation from Customer master, Calendar, or - Ask Agent lands the cursor in the search box. The handled request is consumed, - so later board navigation does not steal focus, and Search closes an open - mobile drawer like every other destination change. -- Mobile Event Lineage evidence cards now read their translated column labels - from the rendered cells instead of hardcoded English CSS, and the two drawer - close controls have distinct accessible names. -- Event Lineage SVG edges now retain their instance-specific direction markers, - so parent-to-child arrows remain visible when multiple lineage groups render. - All OpenAI-compatible chat-completion consumers now validate the shared response envelope before parsing it, preventing malformed provider bodies from escaping as raw `KeyError` or response-shape details. diff --git a/CLAUDE.md b/CLAUDE.md index 40e173ead..b71583083 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,10 +78,23 @@ ISO-8601 week stay; other VOC types and older weeks drop out. The Board names Event Lineage as the next read (ADR 0092). Open a remaining post: Event Lineage takes focus and names Keyman and evaluation next (ADR 0093). A home-list open does not. Do not invent a theta. - ## Calendar open (v2.14.0) Open Calendar. Authorized commitments are current. Open a commitment: Event Lineage takes focus and names Keyman and evaluation next -(ADR 0094). A home-list open does not. Do not invent a theta or a +(ADR 0134). A home-list open does not. Do not invent a theta or a CalDAV event. + +## Customer master open (v2.15.0) + +Open Customer master. Authorized customer entities are current. Open a +related post: Event Lineage takes focus and names Keyman and evaluation +next (ADR 0095). A home-list open does not. Do not invent a theta or a +customer. + +## Ask Agent open (v2.16.0) + +Open Ask Agent. After an authorized answer, cited posts are current. Open +a cited post: Event Lineage takes focus and names Keyman and evaluation +next (ADR 0096). A home-list open does not. Do not invent a theta or a +cited post. diff --git a/backend/app/analysis_run_start.py b/backend/app/analysis_run_start.py index 324c6cd67..180a660cf 100644 --- a/backend/app/analysis_run_start.py +++ b/backend/app/analysis_run_start.py @@ -11,7 +11,7 @@ import hashlib import json -from datetime import datetime, timezone +from datetime import UTC, datetime from typing import Any from uuid import UUID @@ -21,15 +21,15 @@ AnalysisRunCreateError, fetch_visible_analysis_run, ) -from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL from backend.app.analysis_run_outbox import ( latest_outbox_delivery_is_claimed, latest_outbox_delivery_is_delivered, outbox_request_digest, ) from backend.app.lineage_ingestion import records_from_source_posts +from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL from lineageweave.adjudication_client import AdjudicationClient -from lineageweave.http_client import HttpClientError, post_json +from lineageweave.http_client import post_json from lineageweave.lineage_persistence import lineage_edge_specs from lineageweave.models import Edge from lineageweave.tepp_client import AnalysisRunRequest, TeppClient, TeppNotAvailable @@ -103,8 +103,8 @@ def transport(payload: dict[str, Any]) -> dict[str, Any]: try: headers = {"authorization": f"Bearer {api_key}"} if api_key.strip() else {} return post_json(url, payload, headers=headers, timeout=30.0) - except (HttpClientError, OSError, ValueError, TypeError) as exc: - raise TeppNotAvailable("TEPP transport unavailable") from exc + except Exception as exc: + raise TeppNotAvailable("TEPP transport request failed") from exc return TeppClient(transport=transport) @@ -119,12 +119,12 @@ def tepp_run_request( """Build TEPP's published request from the frozen run, never a theta.""" cutoff = knowledge_cutoff if cutoff.tzinfo is None: - cutoff = cutoff.replace(tzinfo=timezone.utc) + cutoff = cutoff.replace(tzinfo=UTC) return AnalysisRunRequest( idempotency_key=idempotency_key, tenant_workspace_id=str(corporate_entity_id), snapshot_id=snapshot_sha256, - knowledge_cutoff=cutoff.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + knowledge_cutoff=cutoff.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), model_contract_version=_TEPP_MODEL_CONTRACT, output_profile=_TEPP_OUTPUT_PROFILE, ) @@ -610,7 +610,7 @@ async def deliver_queued_analysis_run( return await _visible_or_404( conn, analysis_run_id, account_id, affiliated_entity_ids ) - now = datetime.now(timezone.utc) + now = datetime.now(UTC) try: if not latest_outbox_delivery_is_claimed(latest): await _append_outbox_delivery( @@ -636,9 +636,8 @@ async def deliver_queued_analysis_run( affiliated_entity_ids=affiliated_entity_ids, adjudication_client=adjudication_client, ) - finished = datetime.now(timezone.utc) - if finished < now: - finished = now + finished = datetime.now(UTC) + finished = max(finished, now) await _append_outbox_delivery( conn, analysis_run_id, @@ -699,7 +698,7 @@ async def _deliver_lineage_reconstruction( adjudication_client: AdjudicationClient | None = None, ) -> None: """Persist ThreadWeave parent choices for the frozen bag.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) member_rows = await _snapshot_member_posts( conn, locked["analysis_source_snapshot_id"], @@ -715,9 +714,8 @@ async def _deliver_lineage_reconstruction( ) edges = lineage_edge_specs(records_from_source_posts(rows), llm=adjudication_client) digest = reconstruction_result_digest(edges) - finished = datetime.now(timezone.utc) - if finished < now: - finished = now + finished = datetime.now(UTC) + finished = max(finished, now) await conn.execute( """ insert into analysis_run_reconstruction @@ -759,7 +757,7 @@ async def _deliver_tepp_measurement( tepp_client: TeppClient, ) -> None: """Submit the frozen snapshot through ``tepp_client``. Never persist a theta.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) request = tepp_run_request( idempotency_key=str(locked["idempotency_key"]), snapshot_sha256=str(locked["snapshot_sha256"]), @@ -767,17 +765,15 @@ async def _deliver_tepp_measurement( corporate_entity_id=str(locked["corporate_entity_id"]), ) status_code, failure_code, envelope = _tepp_submission(tepp_client, request) - if status_code == _SUCCEEDED and envelope is not None: - if not await _persist_tepp_result( - conn, - analysis_run_id=analysis_run_id, - envelope=envelope, - ): - status_code = _FAILED - failure_code = "tepp_result_not_persisted" - finished = datetime.now(timezone.utc) - if finished < now: - finished = now + if status_code == _SUCCEEDED and envelope is not None and not await _persist_tepp_result( + conn, + analysis_run_id=analysis_run_id, + envelope=envelope, + ): + status_code = _FAILED + failure_code = "tepp_result_not_persisted" + finished = datetime.now(UTC) + finished = max(finished, now) await _append_status( conn, analysis_run_id, diff --git a/docker/contextual-orchestrator/start.py b/docker/contextual-orchestrator/start.py index e35c9a436..9ceef0ad2 100644 --- a/docker/contextual-orchestrator/start.py +++ b/docker/contextual-orchestrator/start.py @@ -14,9 +14,11 @@ def _pop_first_env(*names: str) -> str: - """Read the first configured alias without leaving credentials in the environment.""" + """Read the first alias, removing quotes preserved by Docker env files.""" for name in names: value = os.environ.pop(name, "").strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] if value: return value return "" @@ -27,7 +29,7 @@ def main() -> None: provider_key = _pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY", "NVIDIA_NIM_API_KEY") if not provider_key: raise SystemExit("LLM_GATEWAY_API_KEY or LLM_API_KEY is required to start the real LLM service") - auth_token = os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN", "").strip() + auth_token = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_TOKEN") if not auth_token: raise SystemExit("CONTEXTUAL_ORCHESTRATOR_TOKEN is required to start the authenticated LLM service") @@ -36,14 +38,14 @@ def main() -> None: raise SystemExit("LLM_GATEWAY_API_URL or LLM_GATEWAY_URL is required to start the gateway") if not provider_url.rstrip("/").endswith("/v1"): provider_url = provider_url.rstrip("/") + "/v1" - raw_limit = os.environ.pop("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "4096").strip() + raw_limit = _pop_first_env("LLM_GATEWAY_MAX_OUTPUT_TOKENS") or "4096" try: max_output_tokens = int(raw_limit) except ValueError as exc: raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be an integer") from exc if not 64 <= max_output_tokens <= 4096: raise SystemExit("LLM_GATEWAY_MAX_OUTPUT_TOKENS must be between 64 and 4096") - raw_body_limit = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", str(8 * 1024 * 1024)).strip() + raw_body_limit = _pop_first_env("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES") or str(8 * 1024 * 1024) try: max_body_bytes = int(raw_body_limit) except ValueError as exc: @@ -56,7 +58,7 @@ def main() -> None: agent["base_url"] = provider_url agent["credential_key"] = "LLM_GATEWAY_API_KEY" agent.setdefault("provider_protocol", "auto") - embedding_model = os.environ.get("LLM_GATEWAY_EMBEDDING_MODEL", "").strip() + embedding_model = _pop_first_env("LLM_GATEWAY_EMBEDDING_MODEL") if embedding_model: embedding_agents = [ agent diff --git a/docs/adr/0005-relation-verification-agent.md b/docs/adr/0005-relation-verification-agent.md index ed5bd8108..f9f6214c0 100644 --- a/docs/adr/0005-relation-verification-agent.md +++ b/docs/adr/0005-relation-verification-agent.md @@ -38,10 +38,14 @@ observed from LLM classification), not "this specific relationship claim is definitely true" -- a genuinely false relationship between two REAL organizations still returns results about each organization separately, so this is an existence/plausibility check, not a full -relationship-truth adjudicator. That is a real upgrade path once real -usage shows the coarser signal under- or over-trusting results in -practice, not implemented here because nothing yet demonstrates the -need for it over this cheaper stage. +relationship-truth adjudicator. A result is accepted only when every +distinctive token in the proposed organization name occurs in the +result's non-search host or content snippet; the result title is excluded +because search engines echo the query there. This prevents one generic +word in an unrelated result from validating an invented multi-token name. +That is a real upgrade path once real usage shows the coarser signal under- +or over-trusting results in practice, not implemented here because +nothing yet demonstrates the need for full NLI over this cheaper stage. The real implementation, `SearxngRelationVerificationClient`, queries a **self-hosted** Searxng instance (`docker/searxng/`), never a @@ -55,6 +59,12 @@ keeps the channel unavailable (never fabricates a verification result) when `SEARXNG_BASE_URL` is unset, same discipline as every other pluggable client in this repo. +Evidence token selection excludes generic fixture descriptors such as +`fictitious`, `nonexistent`, `placeholder`, `sample`, `example`, and `demo`. +Those words can appear in unrelated search results and are not organization +identity. This keeps synthetic demo data out of the corroboration signal while +retaining distinctive organization tokens and cited URLs. + Persistence: `post_counterparty_entity` gains `verification_status_code` (`common_lookup_value` category `relation_verification_status`: `verify_pending` / `verify_corroborated` diff --git a/docs/adr/0022-authorized-tepp-start.md b/docs/adr/0022-authorized-tepp-start.md index 84fc71635..fd541a58a 100644 --- a/docs/adr/0022-authorized-tepp-start.md +++ b/docs/adr/0022-authorized-tepp-start.md @@ -37,6 +37,8 @@ authorized transaction: 4. submits through `TeppClient`. An empty `TEPP_TRANSPORT_URL` keeps the default unavailable transport. A set URL POSTs the published wire payload through the http(s)-only helper. File URLs stay unavailable; + `AnalysisRunRequest` rejects non-v1 versions and blank/non-text required + fields before the transport is called, matching TEPP's v1 JSON Schema; 5. appends Failed / `tepp_not_available` when the transport is missing or refused, or Failed / `tepp_result_not_persisted` when TEPP accepts an envelope this product cannot store yet. @@ -93,3 +95,7 @@ World Wide Web Consortium. https://www.w3.org/TR/prov-dm/ World Wide Web Consortium. (2013). *PROV-O: The PROV ontology* (W3C Recommendation). https://www.w3.org/TR/prov-o/ + +ContextualWisdomLab. (2026). *TEPP API and modular integration contract* +([Computer software documentation]). GitHub. +https://github.com/ContextualWisdomLab/TEPP/blob/main/docs/API_CONTRACT.md diff --git a/docs/adr/0095-internal-relation-evidence.md b/docs/adr/0094-internal-relation-evidence.md similarity index 96% rename from docs/adr/0095-internal-relation-evidence.md rename to docs/adr/0094-internal-relation-evidence.md index c5e44e4b3..31d8e8f22 100644 --- a/docs/adr/0095-internal-relation-evidence.md +++ b/docs/adr/0094-internal-relation-evidence.md @@ -1,4 +1,4 @@ -# ADR 0095: Preserve authorized internal evidence for relation verification +# ADR 0094: Preserve authorized internal evidence for relation verification - Status: Accepted - Date: 2026-08-18 diff --git a/docs/adr/0095-customer-master-open-focuses-event-lineage.md b/docs/adr/0095-customer-master-open-focuses-event-lineage.md new file mode 100644 index 000000000..c782cc6dc --- /dev/null +++ b/docs/adr/0095-customer-master-open-focuses-event-lineage.md @@ -0,0 +1,35 @@ +# ADR 0095: Opening a Customer master related post focuses Event Lineage + +- Status: Accepted +- Date: 2026-08-19 + +## Context + +Board Weekly VOC and Calendar commitment opens already focus Event Lineage +(ADR 0093 / ADR 0134). Customer master is the remaining buyer GNB destination +that opens an authorized related post. That open was a home-list open: the +popup body appeared and Event Lineage did not take focus. + +## Decision + +Opening a related post on Customer master is a `fromCustomerMaster` open. +That open reuses the Event Lineage focus path used by report-member, +Weekly VOC, and Calendar opens: + +- Customer master names the next action: authorized customer entities are + current; open a related post to read Event Lineage. +- The popup Event Lineage heading takes focus. +- The popup names the opened post as current in Event Lineage and tells + the buyer to read Keyman and evaluation next. + +A Board home-list open does not focus Event Lineage and does not add that +copy. A `?post=` deep link is still a home-list open. + +No TEPP theta is invented. No cutoff body is invented (ADR 0016). Customer +master does not invent a customer or a parent (ADR 0037 / ADR 0010). + +## Consequences + +- Customer master, Calendar, Weekly VOC, and report-member opens share one + focus contract. +- Closing the popup clears the Customer master open flag. diff --git a/docs/adr/0096-ask-agent-open-focuses-event-lineage.md b/docs/adr/0096-ask-agent-open-focuses-event-lineage.md new file mode 100644 index 000000000..a16adb8f9 --- /dev/null +++ b/docs/adr/0096-ask-agent-open-focuses-event-lineage.md @@ -0,0 +1,36 @@ +# ADR 0096: Opening an Ask Agent cited post focuses Event Lineage + +- Status: Accepted +- Date: 2026-08-19 + +## Context + +Board Weekly VOC, Calendar, and Customer master opens already focus Event +Lineage (ADR 0093 / ADR 0134 / ADR 0095). Ask Agent is the remaining buyer +GNB destination that opens an authorized cited post. That open was a +home-list open: the popup body appeared and Event Lineage did not take +focus. + +## Decision + +Opening a cited post on Ask Agent is a `fromAskAgent` open. That open +reuses the Event Lineage focus path used by report-member, Weekly VOC, +Calendar, and Customer master opens: + +- After an authorized answer, Ask Agent names the next action: cited posts + are current; open a cited post to read Event Lineage. +- The popup Event Lineage heading takes focus. +- The popup names the opened post as current in Event Lineage and tells + the buyer to read Keyman and evaluation next. + +A Board home-list open does not focus Event Lineage and does not add that +copy. A `?post=` deep link is still a home-list open. + +No TEPP theta is invented. No cutoff body is invented (ADR 0016). Ask Agent +does not invent a cited post (ADR 0039). + +## Consequences + +- Ask Agent, Customer master, Calendar, Weekly VOC, and report-member + opens share one focus contract. +- Closing the popup clears the Ask Agent open flag. diff --git a/docs/adr/0103-semantic-document-evidence-contract.md b/docs/adr/0103-semantic-document-evidence-contract.md new file mode 100644 index 000000000..88cbcd6a9 --- /dev/null +++ b/docs/adr/0103-semantic-document-evidence-contract.md @@ -0,0 +1,82 @@ +# ADR 0103 — Preserve semantic document evidence across source and buyer views + +**Decision status:** Proposed on the stacked product-gap branch +**Date:** 2026-08-20 +**Figma File ID:** `1Su3lDRmiZdcUs47t1QwIX` +**Related baseline:** [Product and Technical Gap Baseline](../product-technical-gap-baseline.md) + +## Context + +Live aggregate inspection identified recurring buyer-visible loss at the +source boundary: superscript footnotes, nested list order/depth, table rows, +and Markdown tables were not represented consistently between Python +ingestion, PostgreSQL units, and the React popup. A flattened string cannot +reconstruct a table row, a branch in a list, or the position of an image. It +also makes a later LLM summary less auditable. + +The HTML Living Standard defines the semantic elements used by the source +boundary, including `ol`, `li`, `table`, `tr`, and `sup`. CommonMark provides a +versioned baseline for Markdown block parsing; table syntax remains an +extension in many Markdown dialects, so the implementation accepts only a +recognizable header/separator/data shape and otherwise preserves plain text. + +## Decision + +1. Parse source content into ordered semantic units before embedding or + summarization. A unit retains its source label, source order, indentation + metadata, and image position. +2. Group HTML and OOXML table cells into row units. Markdown tables are + recognized only when a header row is immediately followed by a separator + row; the separator itself is not evidence content. +3. Treat explicit CSS/OOXML indentation as authoritative. List-container + nesting contributes structural depth but must not double-count an explicit + source width. +4. Mark a numeric footnote only when the source uses a numeric `sup` marker; + a numeric table cell or ordinary numbered text is not a footnote by itself. +5. Keep the frontend's raw-source fallback aligned with the persisted unit + labels. Persisted row units render as accessible tables; unresolved + structure remains visibly unresolved and actionable. +6. Apply the same narrow Markdown-table renderer to persisted image OCR. + VISION output may use multiple `TEXT` lines so row boundaries survive; its + caption names only visible entities, relationships, layout, and document + purpose rather than offering a generic one-sentence description. The + client allows 600 seconds for deep orchestrator work; a 180-second local + cutoff already terminated a valid live response before delivery. +7. Serialize replacement per source post and reject a same-image retry when + its content hash matches non-empty persisted OCR but the retry returns no + OCR. Provider completion is transport evidence, not permission to erase a + stronger prior observation. During an operator backfill, this typed + preservation failure skips only the affected post, records it in the + aggregate result, and allows the remaining selected posts to continue. + +## Rejected alternatives + +- Flattening all bodies into one embedding string: loses row, list, and image + boundaries and cannot be repaired at display time. +- Treating every leading number as a footnote: mislabels table rows and + numbered instructions. +- Calling a provider directly from the parser: violates the orchestrator + trust boundary and makes evidence/cost lineage incomplete. +- Creating a separate parsing service: the existing shared chunker and + persistence boundary are sufficient; Ponytail favors the smaller change. + +## Consequences + +- Search and summaries receive smaller, meaningful units without exposing raw + markup or image base64. +- The database keeps the existing normalized unit tables; this decision adds + no denormalized JSON field or new service. +- A weaker same-image VISION retry fails before replacement, leaving the + prior committed evidence available for a later orchestrator retry. +- A protected retry does not abort an entire operator batch; the skipped-post + count is visible to the operator without exposing raw post content. +- Markdown dialects outside the narrow recognized shape remain plain text and + are reported as a future parser extension rather than guessed. + +## Verification + +The baseline's synthetic tests cover numeric superscript footnotes, marker +footnotes, nested `ol`/`ul`/`oi` order and depth, HTML/OOXML rows, Markdown +rows, React table rendering, and unresolved indentation. Full CI remains the +release gate. A persistence regression test proves that an empty same-hash +VISION retry cannot delete previously observed OCR. diff --git a/docs/adr/0105-mathematical-script-semantic-normalization.md b/docs/adr/0105-mathematical-script-semantic-normalization.md deleted file mode 100644 index 839c3c98c..000000000 --- a/docs/adr/0105-mathematical-script-semantic-normalization.md +++ /dev/null @@ -1,29 +0,0 @@ -# ADR 0105: Preserve explicit metric scripts in semantic text - -- Status: Accepted -- Date: 2026-08-21 - -## Context - -Source posts may encode a metric unit such as `m3` or an indexed -quantity such as `m3`. Removing the script element loses searchable -and buyer-visible mathematical meaning, while treating every numeric -superscript as mathematics would break the existing footnote contract. - -## Decision - -1. Preserve the original source body unchanged. -2. In derived semantic text, normalize only an explicit bounded metric base - (`m`, `cm`, `mm`, `km`, or `kg`, optionally preceded by a number) followed - by one-to-three numeric `sup` or `sub` elements into Unicode - superscript/subscript digits. For example, `5m3` becomes `5m³`. -3. Leave ordinary numeric superscripts on prose under the existing footnote - role contract. -4. Apply the same normalization in backend chunks and frontend rendering. - -## Consequences - -Metric exponents remain searchable and readable without inventing formula -semantics. Arbitrary mathematical markup beyond this bounded case remains an -explicit open gap and must be covered by a later ADR and fixture before being -normalized. diff --git a/docs/adr/0103-source-whitespace-is-not-authoritative-structure.md b/docs/adr/0105-source-whitespace-is-not-authoritative-structure.md similarity index 96% rename from docs/adr/0103-source-whitespace-is-not-authoritative-structure.md rename to docs/adr/0105-source-whitespace-is-not-authoritative-structure.md index 4dc6c7ed5..1ced86378 100644 --- a/docs/adr/0103-source-whitespace-is-not-authoritative-structure.md +++ b/docs/adr/0105-source-whitespace-is-not-authoritative-structure.md @@ -1,4 +1,4 @@ -# ADR 0103: Source-only whitespace is not authoritative structure +# ADR 0105: Source-only whitespace is not authoritative structure - Status: Accepted - Date: 2026-08-20 diff --git a/docs/adr/0119-lineage-provider-contract.md b/docs/adr/0119-lineage-provider-contract.md new file mode 100644 index 000000000..cfc02a0a4 --- /dev/null +++ b/docs/adr/0119-lineage-provider-contract.md @@ -0,0 +1,73 @@ +# ADR 0119: Publish a bounded LineageWeave provider contract + +- Status: Accepted on this PR; not main-branch truth until merged +- Date: 2026-08-21 +- Decision owners: LineageWeave maintainers + +## Context + +Naruon owns mailbox/provider access, canonical source identities, workspace +authorization, and provider mutations. LineageWeave owns reconstruction, +channel evidence, cutoff semantics, and inferred lineage. Sharing application +tables or copying LineageWeave internals into Naruon would create a second +authority and bypass the existing ABAC boundary. + +## Decision + +`lineageweave.lineage_contract` is the provider-side versioned boundary for a +bounded, store-agnostic analysis request. It uses immutable Python dataclasses +and canonical JSON (`lineage-analysis/v1`) so a future HTTP or generated SDK +adapter can be added without changing reconstruction. + +- Requests carry an opaque caller-owned `evidence_ref`, an opaque + `authorization_scope_ref`, separate occurred/available clocks, bounded text, + bounded email reference collections, and bounded non-authoritative project + hints. +- A request is valid only when evidence references are unique, clocks are + timezone-aware, payload budgets are bounded, and hints point to submitted + evidence. +- `knowledge_cutoff` excludes evidence whose `available_at` is later. Excluded + evidence is reported as a limitation and cannot appear in any edge. +- The implementation reuses the existing `reconstruct()` pipeline and returns edges only in + terms of submitted opaque evidence references, with channel scores and + explicit `inferred` truth status. +- Missing contextual-orchestrator adjudication is an explicit limitation; it + is not a zero score or a fabricated negative signal. +- Provider exceptions and non-finite or out-of-range provider scores drop the + LLM channel and rerun the bounded reconstruction with renormalized channel + weights. Raw provider response bodies and exception text never cross the + public contract. +- The reconstruction text channel receives the short evidence label only; + bounded body text remains request evidence and is not flattened into every + pairwise comparison. +- The request digest is the idempotency identity. Persistence, retry, tenant + authorization, and provider actions remain the consumer's responsibility. + +LineageWeave does not read Naruon's database, receive Naruon credentials, or +claim authoritative project/task/calendar status. Naruon may adopt a proposed +projection only through its own policy. + +## Consequences + +The provider can be tested with synthetic Naruon-shaped evidence and shipped +independently. A service adapter can later expose the same JSON without +leaking database identifiers. The first version intentionally does not create +project projections or provider mutations; those require a separate reviewed +contract and evidence policy. + +## Research and standards grounding + +- World Wide Web Consortium. (2013). *PROV-O: The PROV ontology*. +- Internet Engineering Task Force. (2008). *RFC 5322: Internet message format*. +- World Wide Web Consortium. (2017). *OWL-Time ontology*. +- National Institute of Standards and Technology. (2020). *Security and + privacy controls for information systems and organizations: NIST SP + 800-53 Rev. 5*. https://doi.org/10.6028/NIST.SP.800-53r5 +- OWASP Foundation. (2024). *Application Security Verification Standard + 5.0.0*. https://github.com/OWASP/ASVS +- MITRE. (n.d.). *CWE-209: Generation of error message containing sensitive + information*. https://cwe.mitre.org/data/definitions/209.html + +These standards support explicit provenance, separate message evidence, +distinct event/availability clocks, and prevention of sensitive information in +error messages; they do not authorize a cross-service database dependency. diff --git a/docs/adr/0124-customer-master-tree-projection.md b/docs/adr/0124-customer-master-tree-projection.md new file mode 100644 index 000000000..1ea83cc20 --- /dev/null +++ b/docs/adr/0124-customer-master-tree-projection.md @@ -0,0 +1,114 @@ +# ADR 0124: Cycle-safe customer master tree projection + +- **Status:** Accepted +- **Date:** 2026-08-21 +- **Owners:** Buyer surface and ontology projection +- **Figma file ID:** `SBpgot7uTvMxEaxUwvoc0S` + +## Context + +`GET /api/customer-master` returns the authorized corporate-entity projection as a flat array with +`parent_entity_id`. The buyer surface previously rebuilt a nested list directly in `App.tsx`. +That recovered ordinary Group → Company → Plant relationships, but it had three product defects: + +1. a self-parent or multi-node cycle had no root and therefore disappeared from the customer master; +2. `aria-expanded` described whether related posts were open, not whether the hierarchy branch was + expanded; and +3. the nested list did not implement the keyboard and focus behavior required by the WAI-ARIA tree + pattern. + +The ontology now correctly separates real organization instances from their classification levels: + +- `CorporateEntity` specializes `org:Organization`; +- `subOrganizationOf` is the semantic projection of `corporate_entity.parent_entity_id` and + specializes `org:subOrganizationOf`; +- `hasSubOrganization` is its inverse; +- Group, Company, and Plant are `CorporateEntityLevel` SKOS concepts selected by `hasEntityLevel`; +- the SHACL profile requires exactly one level and no more than one parent in this projection. + +An entity authorized for the caller must not disappear because an imported hierarchy edge is malformed +or because its parent is outside the caller's visible scope. At the same time, the browser must not +invent a replacement parent or promote an inferred relation to an authoritative ontology fact. + +The W3C Organization Ontology supplies organizational containment. SKOS supplies the separate level +classification. SHACL supplies closed-world cardinality checks. The WAI-ARIA Authoring Practices Tree +View Pattern defines `tree`, `treeitem`, `group`, branch `aria-expanded`, roving focus, and arrow-key +navigation for an interactive hierarchy. + +## Decision + +1. Keep PostgreSQL `corporate_entity.parent_entity_id` as the authority. The ontology and SHACL graphs + are semantic and validation projections, not a second writable hierarchy. +2. Preserve the current ontology boundary: organization containment uses W3C ORG; Group/Company/Plant + level classification uses SKOS. Do not use `skos:broader` between real company instances. +3. Move browser hierarchy assembly into `frontend/src/customerMasterTree.ts`. +4. Preserve API order for roots and siblings and preserve every unique authorized entity. +5. Keep a parent link only when the parent is present in the authorized response. +6. Promote a missing-parent, self-parent, or every member of a detected cycle to a visible root and + mark the relation `unresolved`; do not infer a replacement parent. +7. Render the projection through the reusable `CustomerMasterTree` component. +8. Separate hierarchy disclosure from evidence disclosure: + - Left/Right Arrow collapses, expands, and moves to parent or first child. + - Up/Down Arrow, Home, and End move through visible tree items. + - Enter or Space selects an entity and opens source-backed related posts. + - Related-post evidence is rendered outside the `tree` ownership boundary. +9. Explicitly declare `aria-level`, `aria-posinset`, and `aria-setsize`; use one roving `tabIndex=0` + and `aria-selected` for the entity whose evidence is open. +10. Reject stale related-post responses after the buyer selects another entity. +11. Keep the current flat API as a bounded authorized projection. Legal ownership, operating + structure, sales roll-up, billing structure, and historical hierarchy remain a later normalized, + effective-dated relation model. + +## Consequences + +### Positive + +- Ordinary Group → Company → Plant structures remain visibly hierarchical. +- Real organization instances and SKOS classification concepts remain semantically distinct. +- Malformed or partially visible relations are reviewable instead of silently omitted. +- Keyboard and screen-reader users receive a real tree interaction model. +- Related-post evidence cannot introduce non-tree roles inside the tree ownership boundary. +- The component is independently testable and represented in Storybook. +- Related-post evidence remains source-backed, lazy-loaded, and stale-response safe. + +### Trade-offs + +- An unresolved relation is shown at the root level, which is intentionally less specific than + guessing a parent. +- Client-side defensive projection does not repair the authoritative data. Operators still need a + data-quality workflow for cyclic or invalid source relations. +- The current SHACL profile constrains cardinality and class, but it does not yet prove global + acyclicity or allowed level transitions. +- The referenced Figma file does not contain a dedicated public customer-tree frame. This change uses + the existing design-token and Storybook boundary rather than claiming pixel equivalence to a + nonexistent frame. + +## Verification + +- Existing ontology interoperability tests require ORG organization containment, separate SKOS level + concepts, stable imports/versioning, and SHACL parent/level cardinalities. +- Pure frontend tests cover a three-level hierarchy, missing parent, self-parent, multi-node cycle, + descendant preservation, ordering, and collapsed navigation order. +- Component tests cover ARIA metadata, roving focus, Arrow/Home/End navigation, branch disclosure, + Enter/Space activation, evidence outside the tree, stale request rejection, request failure, + related-post opening, and unresolved relations. +- Storybook includes ordinary and malformed-relation states. +- Focused ontology/frontend tests, frontend lint, complete Vitest suite, production build, and + Storybook build must pass on the exact PR head. + +## References — APA 7th + +World Wide Web Consortium. (2009). *SKOS simple knowledge organization system reference*. +https://www.w3.org/TR/skos-reference/ + +World Wide Web Consortium. (2014). *The Organization Ontology*. +https://www.w3.org/TR/vocab-org/ + +World Wide Web Consortium. (2017). *Shapes Constraint Language (SHACL)*. +https://www.w3.org/TR/shacl/ + +World Wide Web Consortium. (2024). *Web Content Accessibility Guidelines (WCAG) 2.2*. +https://www.w3.org/TR/WCAG22/ + +World Wide Web Consortium, Web Accessibility Initiative. (n.d.). *Tree view pattern*. +https://www.w3.org/WAI/ARIA/apg/patterns/treeview/ diff --git a/docs/adr/0125-customer-master-three-pane-workspace.md b/docs/adr/0125-customer-master-three-pane-workspace.md new file mode 100644 index 000000000..b6ae71a74 --- /dev/null +++ b/docs/adr/0125-customer-master-three-pane-workspace.md @@ -0,0 +1,125 @@ +# ADR 0125: Customer-centered three-pane Customer Master workspace + +- **Status:** Accepted +- **Date:** 2026-08-21 +- **Owners:** Customer Master product surface and evidence navigation +- **Figma file ID:** `SBpgot7uTvMxEaxUwvoc0S` +- **Figma desktop frame:** `313:2` +- **Figma mobile frame:** `314:2` + +## Context + +ADR 0124 established a cycle-safe, authorized WAI-ARIA tree for Group → Company → Plant +containment and deliberately kept related-post evidence outside the tree ownership boundary. That +corrected malformed hierarchy handling and keyboard navigation, but the product composition remained +vertically fragmented: + +1. the hierarchy occupied the first block; +2. selecting an entity caused evidence to appear below the complete tree; +3. relationship-network, unresolved-hint, source-author, and Keyman blocks continued further down; +4. the currently selected customer was not held as the stable visual center of the task. + +Users therefore had to remember which entity they selected while scanning a long page. Parent/child +relationships and source evidence were available, but they were not arranged around the customer that +the user was trying to understand. A free-form graph would add visual complexity and would also risk +presenting inferred edges as if they were authoritative Customer Master facts. + +The uploaded *웹 시스템 UI·UX 표준 가이드 Ver.3.0* requires clear navigation hierarchy and active +state, a 1024 px PC boundary, a 768 px phone boundary, responsive content ordering, system-font +control, and content-page actions that remain discoverable on small screens. ADR 0118 adopted those +breakpoints and design-token rules for LineageWeave. + +## Decision + +1. Compose Customer Master as one customer-centered workspace with three explicit semantic panes: + - **01 Customer hierarchy:** the existing authorized, cycle-safe WAI-ARIA tree; + - **02 Selected customer:** one stable customer summary with the visible parent and direct child + relationships around that customer; + - **03 Linked evidence:** only source-backed related posts, with the existing open-post handoff to + Event Lineage. +2. Keep the selected customer separate from whether its evidence pane is open. Closing evidence must + not lose the customer's centered relationship context. +3. Keep `corporate_entity.parent_entity_id` authoritative. The middle pane may recenter on a visible + parent or direct child, but it must not infer hidden parents, siblings, ownership, or alternative + organizational edges. +4. Preserve all ADR 0124 hierarchy semantics and keyboard behavior. Branch disclosure remains + independent from customer selection. +5. Keep source-backed related posts outside `role="tree"`. A tree item may reference the evidence + region with `aria-controls` only while that region exists. +6. Preserve stale-request rejection and per-entity evidence caching when users move rapidly between + customers. +7. Use existing design tokens for border, focus, color, status, spacing, and dark-mode behavior. Do + not introduce a second Customer Master palette. +8. Use the UI·UX guide's three responsive tiers: + - **PC, greater than 1024 px:** all three panes in one horizontal row; + - **Tablet, up to 1024 px:** hierarchy and selected customer side by side, evidence full width; + - **Phone, up to 768 px:** hierarchy → selected customer → evidence as one vertical task sequence. +9. Maintain complete product copy for all five supported locales: English, Korean, Chinese, + Japanese, and Vietnamese. +10. Represent the desktop, phone, malformed-relation, and unselected states in Storybook. The Figma + frames are the visual design evidence; Storybook remains the executable state inventory. + +## Alternatives considered + +### Keep the vertical tree and accordion evidence + +Rejected because it preserves the long-memory task: the selected customer scrolls away while evidence +and other relationship blocks appear below. + +### Replace the tree with a network graph + +Rejected because graph layout does not provide a predictable hierarchy scan, is harder to operate with +a keyboard, and can blur the boundary between authoritative containment and inferred relationships. +Graphs remain appropriate for Event Lineage, not for the Customer Master authority projection. + +### Put all relationships in one wide table + +Rejected because a table flattens the Group → Company → Plant path and makes recentering around one +customer less direct. Exact-value tables may supplement a graph, but they do not replace the +hierarchical navigation contract here. + +## Consequences + +### Positive + +- The selected customer remains visually and semantically central while users inspect its parent, + children, and evidence. +- The page expresses a stable left-to-right task: choose → understand relations → verify evidence. +- Evidence can close without losing the selected customer or its relationship context. +- Existing WAI-ARIA tree behavior, malformed-relation visibility, authorization scope, and + source-backed evidence boundaries remain intact. +- Responsive layouts preserve the same semantic order instead of hiding relationship context behind a + separate phone-only interaction model. +- Figma and Storybook now describe the same product surface and edge states. + +### Trade-offs + +- A three-pane desktop layout uses more horizontal space than the previous vertical list. +- Tablet users receive a two-row composition rather than all three panes in one row. +- Only the authoritative parent and direct children are shown in the middle pane. Siblings, historical + roles, billing structure, and inferred relationships require separately typed products or later + effective-dated relation models. + +## Verification + +- Existing `CustomerMasterTree` tests continue to cover WAI-ARIA metadata, roving focus, + Arrow/Home/End navigation, independent branch disclosure, stale request rejection, request failure, + evidence caching, and malformed hierarchy members. +- New workspace tests cover the three-pane composition, stable selected-customer state, parent and + direct-child recentering, source evidence outside the tree, evidence close/reopen behavior, + unresolved relation explanation, leaf boundary copy, and five-locale copy completeness. +- Frontend lint, TypeScript, complete Vitest, production build, and Storybook build must pass on the + exact PR head. +- GitHub protected checks and independent review remain the final merge authority. + +## References — APA 7th + +ContextualWisdomLab. (2026). *ADR 0118: UI·UX Standard Guide Ver.3.0 design overhaul*. + +ContextualWisdomLab. (2026). *ADR 0124: Cycle-safe customer master tree projection*. + +World Wide Web Consortium. (2024). *Web Content Accessibility Guidelines (WCAG) 2.2*. +https://www.w3.org/TR/WCAG22/ + +World Wide Web Consortium, Web Accessibility Initiative. (n.d.). *Tree view pattern*. +https://www.w3.org/WAI/ARIA/apg/patterns/treeview/ diff --git a/docs/adr/0130-mathematical-script-semantic-normalization.md b/docs/adr/0130-mathematical-script-semantic-normalization.md new file mode 100644 index 000000000..92daf84c5 --- /dev/null +++ b/docs/adr/0130-mathematical-script-semantic-normalization.md @@ -0,0 +1,50 @@ +# ADR 0130: Preserve explicit metric scripts in semantic text + +**Status:** Accepted on this PR; not protected-main truth +**Date:** 2026-08-21 +**Owners:** LineageWeave ingestion and buyer-surface maintainers + +## Context + +Source posts commonly encode a unit such as `m3`, `m3`, +`m^3`, or `m_3` with HTML or plain-text notation. Dropping the markup changes +the searchable meaning to `m3`, while treating every numeric `sup` element as +mathematics would break the existing numeric-footnote contract. Full MathML +parsing is not yet justified by the current product surface, but the loss of +explicit unit scripts is a buyer-visible defect. + +MathML 4 defines `msup`, `msub`, and `msubsup` as structural script elements; +HTML `sup`/`sub` are a permitted lighter-weight notation when detailed +mathematical markup is not required. This decision therefore adds a bounded +normalization boundary and keeps the source representation unchanged. + +## Decision + +1. Preserve the immutable source body exactly as imported. +2. In derived semantic text only, normalize an explicitly bounded metric base + (`m`, `cm`, `mm`, `km`, or `kg`, optionally preceded by a number) followed + by numeric `sup`/`sub` markup or plain-text `^`/`_` notation into Unicode + superscript/subscript digits. For example, `5m3` and `5m^3` + become `5m³`, while `m3` and `m_3` become `m₃`. +3. Keep ordinary numeric superscripts and caret expressions on prose under the existing footnote + role contract. Do not infer a mathematical formula from an arbitrary word. +4. Apply the same bounded normalization in backend semantic chunks and the + React buyer display so search text and visible text agree. +5. Defer full MathML/LaTeX parsing, expression trees, and ontology term + creation until an authorized fixture demonstrates a need beyond metric + scripts. Any such change requires a new ADR and parser contract. + +## Consequences + +- Search and the buyer popup retain the visible distinction between `m³` and + `m3` without exposing source HTML to the embedding model. +- Existing numeric-footnote tests remain unchanged because the bounded metric + pattern is the only new conversion. +- The current implementation does not claim to understand arbitrary equations; + unsupported script markup remains ordinary source text and must not be + presented as a parsed ontology expression. + +## References (APA 7th) + +World Wide Web Consortium. (2026). *Mathematical Markup Language (MathML) +Version 4.0* (W3C Recommendation). https://www.w3.org/TR/mathml4/ diff --git a/docs/adr/0119-retire-buyer-terminology.md b/docs/adr/0131-retire-buyer-terminology.md similarity index 98% rename from docs/adr/0119-retire-buyer-terminology.md rename to docs/adr/0131-retire-buyer-terminology.md index 4124e1e0a..b5d698526 100644 --- a/docs/adr/0119-retire-buyer-terminology.md +++ b/docs/adr/0131-retire-buyer-terminology.md @@ -1,4 +1,4 @@ -# ADR 0119: Retire "Buyer" as the reader-facing terminology +# ADR 0131: Retire "Buyer" as the reader-facing terminology **Status:** Accepted **Date:** 2026-08-21 diff --git a/docs/adr/0124-operational-controlled-vocabulary-semantic-layer.md b/docs/adr/0132-operational-controlled-vocabulary-semantic-layer.md similarity index 97% rename from docs/adr/0124-operational-controlled-vocabulary-semantic-layer.md rename to docs/adr/0132-operational-controlled-vocabulary-semantic-layer.md index 6853f5308..43431ad85 100644 --- a/docs/adr/0124-operational-controlled-vocabulary-semantic-layer.md +++ b/docs/adr/0132-operational-controlled-vocabulary-semantic-layer.md @@ -1,5 +1,4 @@ -# ADR 0124: Model operational controlled vocabularies as SKOS concepts - +# ADR 0132: Model operational controlled vocabularies as SKOS concepts ## Status Accepted diff --git a/docs/adr/0133-external-email-project-lineage-contract.md b/docs/adr/0133-external-email-project-lineage-contract.md new file mode 100644 index 000000000..324070b1e --- /dev/null +++ b/docs/adr/0133-external-email-project-lineage-contract.md @@ -0,0 +1,48 @@ +# ADR 0133: Publish a bounded external email/project lineage contract + +- Status: Accepted +- Date: 2026-08-21 + +## Context + +Naruon owns customer mail/calendar/file access, canonical message/thread identities, projects, tasks, commitments, provider credentials, authorization, and provider mutations. LineageWeave owns evidence-fused lineage reconstruction and the provenance explaining that reconstruction. Future integration must not give either product direct SQL access to the other's application database, duplicate source authority, or depend on a mutable branch/submodule. + +Email thread facts also have different truth semantics from reconstructed semantic continuation. RFC `Message-ID`, `References`, and `In-Reply-To` evidence may establish a caller-observed reply relation, while LineageWeave text/temporal/project signals produce an inferred relation. Flattening both into one unexplained score would make buyer correction and audit impossible. + +## Decision + +LineageWeave publishes contract version `1.0.0` through: + +- `lineageweave.external_lineage_contract` for strict immutable request/result shapes, canonical serialization, bounds, and deterministic digests; +- `lineageweave.external_lineage_analysis` for adapting caller-authorized evidence to the existing reconstruction kernel. + +The initial implementation is a store-agnostic Python package boundary. It performs no database, mailbox, provider, or network operation. A later service or Naruon plugin adapter must preserve the same JSON Schema and truth boundaries. + +The caller supplies opaque evidence references, bounded text labels, occurrence and availability clocks, an optional secondary key, an optional project reference, and an optional caller-observed parent relation. Explicit observed parent relations replace an inferred parent for the same child and must form an acyclic graph. Reconstructed continuation remains `inferred`. Project groupings remain `proposed`. + +An admitted child with an explicit observed parent is not rescored for an alternative inferred parent and consumes no optional LLM/provider call or inferred-pair budget. The record remains in temporal history and may still be an eligible candidate parent for a later record. This preserves observed authority without weakening downstream lineage reconstruction. + +The caller also supplies `maximum_pair_evaluations` in the bounded policy. The package computes the exact inferred candidate-parent pair count after knowledge-cutoff filtering, excluding children whose parent is already caller-observed, and rejects work above the declared budget before any optional LLM/provider call. Contract v1 caps the declared budget at 5,000 pairs. + +Historical requests include evidence only when: + +```text +available_at <= knowledge_cutoff +``` + +Evidence becoming available after the cutoff is excluded even when it describes an earlier occurrence. + +## Consequences + +- Naruon can eventually consume a released artifact without exposing credentials or application tables. +- RFC reply/thread evidence stays distinguishable from semantic lineage. +- Caller-observed children are never disclosed to an optional model merely to calculate an inferred edge that would be discarded. +- The optional LLM channel is explicit as `not_requested`, `unavailable`, `not_invoked`, or `completed`; missing output is never zero. `not_invoked` means the channel was allowed and available but no candidate pair required a model call. +- Canonical serialization and SHA-256 digesting are deterministic for a given request or result. Repeatability of model-backed scores additionally requires a pinned LineageWeave release, adjudicator implementation, provider/model revision, and model-side determinism policy. +- Explicit parent cycles and analysis work above the caller-approved pair budget fail closed before inference. +- Project evidence can inform Naruon without mutating authoritative project/task/provider state. +- The single generic secondary key reflects the current core kernel. Multiple independent typed secondary-key channels remain a future contract revision rather than being silently flattened. + +## References + +See `docs/doctoring/EXTERNAL_LINEAGE_CONTRACT_REFERENCES.md`. diff --git a/docs/adr/0094-calendar-open-focuses-event-lineage.md b/docs/adr/0134-calendar-open-focuses-event-lineage.md similarity index 94% rename from docs/adr/0094-calendar-open-focuses-event-lineage.md rename to docs/adr/0134-calendar-open-focuses-event-lineage.md index 0b7b1a8b9..ddef9fcab 100644 --- a/docs/adr/0094-calendar-open-focuses-event-lineage.md +++ b/docs/adr/0134-calendar-open-focuses-event-lineage.md @@ -1,4 +1,4 @@ -# ADR 0094: Opening a Calendar commitment focuses Event Lineage +# ADR 0134: Opening a Calendar commitment focuses Event Lineage - Status: Accepted - Date: 2026-08-19 diff --git a/docs/contracts/README.md b/docs/contracts/README.md new file mode 100644 index 000000000..017c05eaf --- /dev/null +++ b/docs/contracts/README.md @@ -0,0 +1,13 @@ +# Integration contracts + +LineageWeave publishes strict, versioned contracts for separately governed consumers. These contracts do not grant source access and do not replace each consumer's authorization, persistence, provider, or audit authority. + +## External lineage analysis v1 + +- JSON Schema: `external-lineage-analysis-v1.schema.json` +- Synthetic request: `external-lineage-analysis-v1.example.json` +- Python parser and immutable types: `lineageweave.external_lineage_contract` +- Store-agnostic execution adapter: `lineageweave.external_lineage_analysis` +- Decision record: `docs/adr/0133-external-email-project-lineage-contract.md` + +A consumer must submit only bounded evidence it is already authorized to disclose. Outputs retain opaque caller references and explicit `observed`, `inferred`, or `proposed` truth boundaries. The contract performs no source-system access or provider mutation. diff --git a/docs/contracts/external-lineage-analysis-v1.authorization.md b/docs/contracts/external-lineage-analysis-v1.authorization.md new file mode 100644 index 000000000..44216d3d2 --- /dev/null +++ b/docs/contracts/external-lineage-analysis-v1.authorization.md @@ -0,0 +1,5 @@ +# External lineage analysis v1 authorization contract + +LineageWeave does not infer authorization from an opaque reference, source kind, group, project, or caller identity. The caller must authorize evidence before projection and must reauthorize any source drill-through after receiving a result. + +The package does not accept provider bearer tokens, browser cookies, mailbox credentials, database DSNs, or caller SQL. A future remote service must use its own audience-scoped service credential and may not forward an end-user token to model providers. diff --git a/docs/contracts/external-lineage-analysis-v1.consumer-checklist.md b/docs/contracts/external-lineage-analysis-v1.consumer-checklist.md new file mode 100644 index 000000000..9c89e17cb --- /dev/null +++ b/docs/contracts/external-lineage-analysis-v1.consumer-checklist.md @@ -0,0 +1,11 @@ +# External lineage analysis v1 consumer checklist + +- Validate the published JSON Schema before sending or accepting payloads. +- Submit only evidence the calling principal is authorized to disclose for the declared purpose. +- Use opaque caller-owned references; never send provider credentials or database locators. +- Bind historical work to a knowledge cutoff and preserve each record's availability time. +- Keep RFC/provider thread observations separate from inferred semantic/project lineage. +- Treat project projections as proposals until the caller's own policy or reviewer accepts them. +- Preserve the returned artifact digest, LineageWeave version, limitations, and channel evidence. +- Fail closed on incompatible contract versions. +- Keep normal caller operation available when LineageWeave is unavailable. diff --git a/docs/contracts/external-lineage-analysis-v1.data-minimization.md b/docs/contracts/external-lineage-analysis-v1.data-minimization.md new file mode 100644 index 000000000..522c55e2a --- /dev/null +++ b/docs/contracts/external-lineage-analysis-v1.data-minimization.md @@ -0,0 +1,12 @@ +# External lineage analysis v1 data minimization + +Consumers should prefer the minimum evidence needed for a declared analysis scope: + +- opaque evidence and grouping references; +- offset-aware occurrence and availability times; +- RFC/provider relation evidence when present; +- bounded subject/title labels or caller-computed text features; +- optional project or secondary-key references; +- optional participant, body, or attachment evidence only when the caller's purpose and policy explicitly permit it. + +The contract does not require a mailbox dump, full thread body, recipient list, provider URL, or attachment bytes. Omitted evidence is unavailable and cannot appear in output. diff --git a/docs/contracts/external-lineage-analysis-v1.example.json b/docs/contracts/external-lineage-analysis-v1.example.json new file mode 100644 index 000000000..b9b90bdce --- /dev/null +++ b/docs/contracts/external-lineage-analysis-v1.example.json @@ -0,0 +1,41 @@ +{ + "contract_version": "1.0.0", + "analysis_id": "analysis:synthetic-email-lineage-001", + "analysis_scope_code": "email_lineage", + "knowledge_cutoff": "2026-08-20T09:30:00Z", + "policy": { + "candidate_window": 50, + "maximum_pair_evaluations": 1000, + "minimum_fused_score": 0.3, + "allow_llm": false + }, + "records": [ + { + "evidence_ref": "email:synthetic-001", + "group_ref": "workspace:synthetic", + "source_kind_code": "email", + "truth_status_code": "observed", + "label": "Synthetic proposal review", + "occurred_at": "2026-08-20T09:00:00Z", + "available_at": "2026-08-20T09:01:00Z", + "secondary_key": "provider-thread:synthetic", + "project_ref": "project:synthetic", + "explicit_parent": null + }, + { + "evidence_ref": "email:synthetic-002", + "group_ref": "workspace:synthetic", + "source_kind_code": "email", + "truth_status_code": "observed", + "label": "Re: Synthetic proposal review", + "occurred_at": "2026-08-20T09:05:00Z", + "available_at": "2026-08-20T09:06:00Z", + "secondary_key": "provider-thread:synthetic", + "project_ref": "project:synthetic", + "explicit_parent": { + "evidence_ref": "email:synthetic-001", + "relation_code": "rfc_reply" + } + } + ] +} diff --git a/docs/contracts/external-lineage-analysis-v1.limitations.md b/docs/contracts/external-lineage-analysis-v1.limitations.md new file mode 100644 index 000000000..fd3864f77 --- /dev/null +++ b/docs/contracts/external-lineage-analysis-v1.limitations.md @@ -0,0 +1,11 @@ +# External lineage analysis v1 limitations + +- The contract does not read IMAP, JMAP, CalDAV, Naruon, or other provider systems. +- It does not authenticate users, authorize tenant access, persist jobs, or retry remote work. +- It does not make semantic lineage equivalent to RFC reply/thread identity. +- It does not turn project groupings, responsibility context, or reconstructed edges into authoritative caller facts. +- It does not infer unavailable evidence as a zero-valued channel. +- It does not guarantee causal relations; reconstructed continuation is an evidence-weighted related-history hypothesis. +- Canonical request/result serialization and digests are deterministic, but an optional remote adjudication channel is not automatically repeatable unless the consumer pins the LineageWeave artifact, adjudicator, provider/model revision, and determinism policy. +- Contract v1 does not carry a remote provider/model receipt inside the result; production wrappers must retain that provenance alongside the result digest before model-backed integration is enabled. +- It does not replace Naruon's canonical email identity, project/task/commitment state, provider mutation, or reconciliation authority. diff --git a/docs/contracts/external-lineage-analysis-v1.operability.md b/docs/contracts/external-lineage-analysis-v1.operability.md new file mode 100644 index 000000000..7e3de54f6 --- /dev/null +++ b/docs/contracts/external-lineage-analysis-v1.operability.md @@ -0,0 +1,5 @@ +# External lineage analysis v1 operability boundary + +The pure package entry point is synchronous and bounded. Remote or model-backed production use must wrap it in a separately reviewed service or plugin lifecycle with durable idempotency, cancellation, timeout, retry classification, rate limiting, resource budgets, artifact retention, OpenTelemetry signals, and user-visible degraded states. + +A consumer must not call optional model-backed pair adjudication directly on an unbounded web request path. LineageWeave #289 tracks the durable asynchronous reconstruction requirement for product persistence, and Naruon #1437 requires an equivalent consumer-side job receipt before integration is enabled. diff --git a/docs/contracts/external-lineage-analysis-v1.schema.json b/docs/contracts/external-lineage-analysis-v1.schema.json new file mode 100644 index 000000000..05d74e6a1 --- /dev/null +++ b/docs/contracts/external-lineage-analysis-v1.schema.json @@ -0,0 +1,255 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://contextualwisdomlab.org/schemas/external-lineage-analysis-v1.schema.json", + "title": "LineageWeave External Lineage Analysis Request v1", + "description": "Bounded caller-authorized evidence for store-agnostic lineage analysis. The response shape is available as $defs.LineageAnalysisResult.", + "type": "object", + "additionalProperties": false, + "required": [ + "contract_version", + "analysis_id", + "analysis_scope_code", + "policy", + "records" + ], + "properties": { + "contract_version": {"const": "1.0.0"}, + "analysis_id": {"$ref": "#/$defs/OpaqueReference"}, + "analysis_scope_code": { + "type": "string", + "enum": ["email_lineage", "project_history", "generic_lineage"] + }, + "knowledge_cutoff": { + "anyOf": [ + {"type": "string", "format": "date-time"}, + {"type": "null"} + ] + }, + "policy": {"$ref": "#/$defs/LineageAnalysisPolicy"}, + "records": { + "type": "array", + "minItems": 1, + "maxItems": 500, + "items": {"$ref": "#/$defs/LineageEvidenceRecord"} + } + }, + "$defs": { + "OpaqueReference": { + "type": "string", + "minLength": 1, + "maxLength": 160, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:@+\\-]*$" + }, + "NullableOpaqueReference": { + "anyOf": [ + {"$ref": "#/$defs/OpaqueReference"}, + {"type": "null"} + ] + }, + "ExplicitParent": { + "type": "object", + "additionalProperties": false, + "required": ["evidence_ref", "relation_code"], + "properties": { + "evidence_ref": {"$ref": "#/$defs/OpaqueReference"}, + "relation_code": { + "type": "string", + "enum": ["rfc_reply", "provider_reply", "manual_parent"] + } + } + }, + "LineageAnalysisPolicy": { + "type": "object", + "additionalProperties": false, + "required": [ + "candidate_window", + "maximum_pair_evaluations", + "minimum_fused_score", + "allow_llm" + ], + "properties": { + "candidate_window": { + "type": "integer", + "minimum": 1, + "maximum": 200 + }, + "maximum_pair_evaluations": { + "type": "integer", + "minimum": 1, + "maximum": 5000 + }, + "minimum_fused_score": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "allow_llm": {"type": "boolean"} + } + }, + "LineageEvidenceRecord": { + "type": "object", + "additionalProperties": false, + "required": [ + "evidence_ref", + "group_ref", + "source_kind_code", + "truth_status_code", + "label", + "occurred_at", + "available_at" + ], + "properties": { + "evidence_ref": {"$ref": "#/$defs/OpaqueReference"}, + "group_ref": {"$ref": "#/$defs/OpaqueReference"}, + "source_kind_code": { + "type": "string", + "enum": ["email", "task", "commitment", "project_event", "generic"] + }, + "truth_status_code": { + "type": "string", + "enum": ["observed", "authoritative_in_caller"] + }, + "label": { + "type": "string", + "minLength": 1, + "maxLength": 2000 + }, + "occurred_at": {"type": "string", "format": "date-time"}, + "available_at": {"type": "string", "format": "date-time"}, + "secondary_key": {"$ref": "#/$defs/NullableOpaqueReference"}, + "project_ref": {"$ref": "#/$defs/NullableOpaqueReference"}, + "explicit_parent": { + "anyOf": [ + {"$ref": "#/$defs/ExplicitParent"}, + {"type": "null"} + ] + } + } + }, + "ChannelEvidence": { + "type": "object", + "additionalProperties": false, + "required": ["channel_code", "score", "weight", "contribution"], + "properties": { + "channel_code": {"type": "string", "minLength": 1, "maxLength": 64}, + "score": {"type": "number", "minimum": 0, "maximum": 1}, + "weight": {"type": "number", "minimum": 0, "maximum": 1}, + "contribution": {"type": "number", "minimum": 0, "maximum": 1} + } + }, + "LineageEdgeResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "parent_evidence_ref", + "child_evidence_ref", + "relation_type_code", + "truth_status_code", + "fused_score", + "channel_evidence" + ], + "properties": { + "parent_evidence_ref": {"$ref": "#/$defs/OpaqueReference"}, + "child_evidence_ref": {"$ref": "#/$defs/OpaqueReference"}, + "relation_type_code": {"type": "string", "minLength": 1, "maxLength": 64}, + "truth_status_code": { + "type": "string", + "enum": ["observed", "inferred"] + }, + "fused_score": {"type": "number", "minimum": 0, "maximum": 1}, + "channel_evidence": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/ChannelEvidence"} + } + } + }, + "ProjectProjection": { + "type": "object", + "additionalProperties": false, + "required": ["group_ref", "project_ref", "evidence_refs", "truth_status_code"], + "properties": { + "group_ref": {"$ref": "#/$defs/OpaqueReference"}, + "project_ref": {"$ref": "#/$defs/OpaqueReference"}, + "evidence_refs": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/OpaqueReference"}, + "uniqueItems": true + }, + "truth_status_code": {"const": "proposed"} + } + }, + "LineageLimitation": { + "type": "object", + "additionalProperties": false, + "required": ["limitation_code", "evidence_ref", "message"], + "properties": { + "limitation_code": {"type": "string", "minLength": 1, "maxLength": 96}, + "evidence_ref": {"$ref": "#/$defs/NullableOpaqueReference"}, + "message": {"type": "string", "minLength": 1, "maxLength": 500} + } + }, + "LineageAnalysisResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "contract_version", + "analysis_id", + "analysis_scope_code", + "knowledge_cutoff", + "included_evidence_refs", + "excluded_evidence_refs", + "llm_status_code", + "edges", + "project_projections", + "limitations", + "result_digest" + ], + "properties": { + "contract_version": {"const": "1.0.0"}, + "analysis_id": {"$ref": "#/$defs/OpaqueReference"}, + "analysis_scope_code": { + "type": "string", + "enum": ["email_lineage", "project_history", "generic_lineage"] + }, + "knowledge_cutoff": { + "anyOf": [ + {"type": "string", "format": "date-time"}, + {"type": "null"} + ] + }, + "included_evidence_refs": { + "type": "array", + "items": {"$ref": "#/$defs/OpaqueReference"}, + "uniqueItems": true + }, + "excluded_evidence_refs": { + "type": "array", + "items": {"$ref": "#/$defs/OpaqueReference"}, + "uniqueItems": true + }, + "llm_status_code": { + "type": "string", + "enum": ["not_requested", "unavailable", "not_invoked", "completed"] + }, + "edges": { + "type": "array", + "items": {"$ref": "#/$defs/LineageEdgeResult"} + }, + "project_projections": { + "type": "array", + "items": {"$ref": "#/$defs/ProjectProjection"} + }, + "limitations": { + "type": "array", + "items": {"$ref": "#/$defs/LineageLimitation"} + }, + "result_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + } + } + } +} diff --git a/docs/contracts/external-lineage-analysis-v1.security.md b/docs/contracts/external-lineage-analysis-v1.security.md new file mode 100644 index 000000000..c87ffbab2 --- /dev/null +++ b/docs/contracts/external-lineage-analysis-v1.security.md @@ -0,0 +1,22 @@ +# External lineage analysis v1 security boundary + +The contract is an analysis interface, not an authorization interface. + +## Caller responsibilities + +- authenticate the caller and authorize every submitted evidence record; +- enforce tenant, workspace, purpose, retention, and export policy; +- minimize text and participant evidence according to data classification; +- retain provider credentials, raw access tokens, browser sessions, and unrelated mailbox content inside the caller boundary; +- pin and record the immutable LineageWeave artifact used for an analysis; +- retain adjudicator and provider/model provenance beside any model-backed result; +- verify the returned contract version and result digest before persistence or display. + +## LineageWeave boundary + +- rejects unsafe opaque references, unknown fields, invalid timestamps, duplicate evidence, and over-budget inferred work; +- returns only references present in the admitted request from the supported analysis adapter; +- distinguishes observed caller relations from inferred reconstruction; +- does not rescore or disclose a caller-observed child to the optional LLM merely to generate an alternative edge that would be discarded; +- never promotes a proposed project projection to caller authority; +- performs no provider mutation and receives no provider credential through this contract. diff --git a/docs/contracts/external-lineage-analysis-v1.versioning.md b/docs/contracts/external-lineage-analysis-v1.versioning.md new file mode 100644 index 000000000..9b0640ee3 --- /dev/null +++ b/docs/contracts/external-lineage-analysis-v1.versioning.md @@ -0,0 +1,10 @@ +# External lineage analysis versioning policy + +- `contract_version` follows semantic versioning independently from the LineageWeave package version. +- Unknown major versions fail closed. +- Additive optional fields require a new minor contract revision and corresponding consumer fixtures. +- Vocabulary changes, field semantic changes, required-field changes, digest changes, or truth-status changes require a new major contract version. +- A released schema, example, parser, serializer, digest algorithm, and consumer fixtures remain immutable for that contract version. +- Consumers must record both the contract version and immutable LineageWeave package/service artifact identity. The contract version alone does not identify the reconstruction implementation. +- Model-backed runs must additionally retain the adjudicator implementation and provider/model revision outside the v1 result payload; canonical digest determinism must not be described as provider repeatability. +- Naruon and other consumers pin an immutable LineageWeave release or service artifact and verify compatibility before enabling the integration. diff --git a/docs/doctoring/CUSTOMER_HIERARCHY_REFERENCES.md b/docs/doctoring/CUSTOMER_HIERARCHY_REFERENCES.md new file mode 100644 index 000000000..f30ffcdfb --- /dev/null +++ b/docs/doctoring/CUSTOMER_HIERARCHY_REFERENCES.md @@ -0,0 +1,39 @@ +# Customer hierarchy standards and research traceability + +**Decision:** ADR 0124 — Cycle-safe customer master tree projection +**Reviewed:** 2026-08-21 +**Figma file ID:** `SBpgot7uTvMxEaxUwvoc0S` + +## Traceability + +| External source | Product decision | Implementation evidence | Current limitation | +|---|---|---|---| +| W3C Organization Ontology | Real corporate entities are `org:Organization` instances; `parent_entity_id` projects through `subOrganizationOf`, a specialization of `org:subOrganizationOf`. | `docs/ontology/lineageweave-kg.ttl`, `tests/test_ontology_interoperability.py` | One parent context only; no effective-dated legal/operating hierarchy contexts yet. | +| W3C SKOS | Group, Company, and Plant are classification concepts, not the real customer organizations. | `CorporateEntityLevel`, `hasEntityLevel`, and level concepts in the ontology | Allowed level-transition rules are not yet encoded. | +| W3C SHACL | Closed-world validation requires exactly one level and at most one parent in the published projection. | `docs/ontology/lineageweave-kg.shacl.ttl`, ontology interoperability tests | Global acyclicity is not yet a SHACL constraint; the buyer projection therefore remains defensive. | +| WAI-ARIA APG Tree View Pattern | Use `tree`, nested `treeitem`/`group`, roving focus, Arrow/Home/End navigation, and branch `aria-expanded`. | `CustomerMasterTree.tsx` and component tests | Type-ahead navigation is not included in this bounded change. | +| WCAG 2.2 | All hierarchy and evidence actions are keyboard operable with a visible focus target and preserve the current selection. | Component keyboard tests and existing focus design tokens | Full assistive-technology browser acceptance remains a release-level check. | + +## Product truth boundary + +The ontology and SHACL graphs describe and validate the relational projection; PostgreSQL remains the +source of record. The browser never writes a repaired parent relation. Missing-parent, self-parent, or +cyclic edges are displayed as `unresolved` roots so an authorized customer cannot disappear and a +replacement parent is not invented. + +## References — APA 7th + +World Wide Web Consortium. (2009). *SKOS simple knowledge organization system reference*. +https://www.w3.org/TR/skos-reference/ + +World Wide Web Consortium. (2014). *The Organization Ontology*. +https://www.w3.org/TR/vocab-org/ + +World Wide Web Consortium. (2017). *Shapes Constraint Language (SHACL)*. +https://www.w3.org/TR/shacl/ + +World Wide Web Consortium. (2024). *Web Content Accessibility Guidelines (WCAG) 2.2*. +https://www.w3.org/TR/WCAG22/ + +World Wide Web Consortium, Web Accessibility Initiative. (n.d.). *Tree view pattern*. +https://www.w3.org/WAI/ARIA/apg/patterns/treeview/ diff --git a/docs/doctoring/EXTERNAL_LINEAGE_CONTRACT_REFERENCES.md b/docs/doctoring/EXTERNAL_LINEAGE_CONTRACT_REFERENCES.md new file mode 100644 index 000000000..0b88b76e3 --- /dev/null +++ b/docs/doctoring/EXTERNAL_LINEAGE_CONTRACT_REFERENCES.md @@ -0,0 +1,23 @@ +# External Lineage Contract References + +## Product traceability + +| Source | Product decision | +|---|---| +| RFC 3339 | Require offset-aware occurrence, availability, and knowledge-cutoff timestamps. | +| RFC 5322 | Preserve Internet-message identity and reply metadata as caller-observed evidence rather than semantic inference. | +| RFC 5256 | Keep standards-based email threading evidence distinct from LineageWeave reconstruction. | +| W3C PROV-O | Return evidence references, truth status, analysis identity, and provenance-friendly result artifacts. | +| W3C OWL-Time | Separate occurrence time from evidence availability and enforce cutoff safety by availability. | + +## References — APA 7th + +Klyne, G., & Newman, C. (2002). *Date and time on the Internet: Timestamps* (RFC 3339). RFC Editor. https://doi.org/10.17487/RFC3339 + +Crispin, M., & Murchison, K. (2008). *Internet Message Access Protocol—SORT and THREAD extensions* (RFC 5256). RFC Editor. https://doi.org/10.17487/RFC5256 + +Resnick, P. W. (2008). *Internet message format* (RFC 5322). RFC Editor. https://doi.org/10.17487/RFC5322 + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology*. https://www.w3.org/TR/prov-o/ + +World Wide Web Consortium. (2017). *Time ontology in OWL*. https://www.w3.org/TR/owl-time/ diff --git a/docs/doctoring/EXTERNAL_LINEAGE_CONTRACT_TRACEABILITY.md b/docs/doctoring/EXTERNAL_LINEAGE_CONTRACT_TRACEABILITY.md new file mode 100644 index 000000000..37587f0d1 --- /dev/null +++ b/docs/doctoring/EXTERNAL_LINEAGE_CONTRACT_TRACEABILITY.md @@ -0,0 +1,12 @@ +# External Lineage Contract Traceability + +| Requirement | Product decision | Implementation | Evidence | +|---|---|---|---| +| Caller authorization remains authoritative | Accept only caller-projected evidence and opaque references | `lineageweave.external_lineage_contract` | strict parser and hostile-input tests | +| Historical answers exclude future evidence | Filter by `available_at <= knowledge_cutoff` | `lineageweave.external_lineage_analysis` | cutoff inclusion/exclusion tests | +| RFC relations remain distinct | Explicit parent relations serialize as observed relation codes | execution adapter | observed-parent precedence tests | +| Semantic lineage remains inferred | Reconstructed edges use `truth_status_code=inferred` | execution adapter | result contract tests | +| Optional LLM absence is honest | Return `not_requested`, `unavailable`, or `not_invoked`; do not fabricate a score | execution adapter | LLM policy and schema-vocabulary tests | +| Work is bounded before provider calls | Enforce record count, candidate window, and maximum pair evaluations | parser and execution adapter | pair-budget tests | +| Project state is not silently mutated | Return only `proposed` project projections | contract/result validator | project truth-status tests | +| Consumer compatibility is machine-checkable | Publish JSON Schema and canonical request/result digests | schema and contract module | schema drift and digest tests | diff --git a/docs/doctoring/PRODUCT_TECHNICAL_GAP_REFERENCES.md b/docs/doctoring/PRODUCT_TECHNICAL_GAP_REFERENCES.md new file mode 100644 index 000000000..73f442d20 --- /dev/null +++ b/docs/doctoring/PRODUCT_TECHNICAL_GAP_REFERENCES.md @@ -0,0 +1,24 @@ +# Product technical-gap references + +These references are the normative basis for the semantic-unit boundary in +[ADR 0103](../adr/0103-semantic-document-evidence-contract.md). Dates and +versions are recorded so a later standards refresh can be reviewed rather +than silently changing parser behavior. + +## APA 7th edition + +CommonMark. (2024). *CommonMark spec (Version 0.31.2)*. https://spec.commonmark.org/0.31.2/ + +WHATWG. (2026). *HTML: Living Standard*. https://html.spec.whatwg.org/multipage/ + +## Applied mapping + +| Source | Boundary used in LineageWeave | +| --- | --- | +| CommonMark (2024) | Recognizable Markdown block/header/separator shape; unrecognized dialects remain source text. | +| WHATWG (2026) | HTML list, table-row, and `sup` semantics; source order and element identity are retained as unit metadata. | + +These standards define syntax and semantics, not an LLM extraction license. +Provider-derived summaries, image descriptions, project boundaries, and +5W1H values still require contextual-orchestrator provenance and explicit +source evidence. diff --git a/docs/ontology/lineageweave-kg.shacl.ttl b/docs/ontology/lineageweave-kg.shacl.ttl new file mode 100644 index 000000000..0e59b5ce3 --- /dev/null +++ b/docs/ontology/lineageweave-kg.shacl.ttl @@ -0,0 +1,40 @@ +@prefix : . +@prefix lw: . +@prefix owl: . +@prefix org: . +@prefix rdfs: . +@prefix sh: . + + + a owl:Ontology ; + owl:versionIRI ; + owl:versionInfo "1.0.0" ; + owl:imports ; + rdfs:label "LineageWeave core ontology SHACL shapes"@en . + +:CorporateEntityShape + a sh:NodeShape ; + sh:targetClass lw:CorporateEntity ; + sh:class org:Organization ; + sh:property [ + sh:path lw:hasEntityLevel ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:class lw:CorporateEntityLevel + ] ; + sh:property [ + sh:path lw:subOrganizationOf ; + sh:maxCount 1 ; + sh:class lw:CorporateEntity + ] . + +:TeamShape + a sh:NodeShape ; + sh:targetClass lw:Team ; + sh:class org:OrganizationalUnit ; + sh:property [ + sh:path lw:teamAffiliatedWith ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:class lw:CorporateEntity + ] . diff --git a/docs/ontology/lineageweave-kg.ttl b/docs/ontology/lineageweave-kg.ttl index c713b2a32..e5365eb34 100644 --- a/docs/ontology/lineageweave-kg.ttl +++ b/docs/ontology/lineageweave-kg.ttl @@ -23,17 +23,22 @@ # this file is the formal semantic layer over it -- PostgreSQL stays # the source of record. See docs/adr/0004-knowledge-graph-ontology.md # for the KG design rationale, docs/adr/0006-role-responsibility-agent-ontology.md -# and docs/adr/0124-operational-controlled-vocabulary-semantic-layer.md +# and docs/adr/0132-operational-controlled-vocabulary-semantic-layer.md # for the R&R actor-type rationale (grounded in W3C PROV-O), and # tests/test_ontology.py for the round-trip check that every code below # actually exists as a common_lookup_value row, and vice versa. # -# Every custom term carries a :lookupCode annotation naming the exact +# Every custom term carrying a :lookupCode annotation names the exact # `common_lookup_value.lookup_code` it corresponds to -- that literal # string, not the IRI fragment, is what the relational schema stores. ################################################################# a owl:Ontology ; + owl:versionIRI ; + owl:versionInfo "1.0.0" ; + owl:imports , + , + ; rdfs:label "LineageWeave Knowledge Graph Ontology" ; rdfs:comment "Formal OWL 2 / RDFS / SKOS vocabulary for LineageWeave's knowledge graph, operational controlled vocabularies, and post-summary actor types." . @@ -66,11 +71,16 @@ :lookupCode "counterparty" . :CorporateEntity a owl:Class ; - rdfs:subClassOf skos:Concept ; + rdfs:subClassOf org:Organization ; rdfs:label "Corporate entity" ; - rdfs:comment "A corporate_entity row. Also a skos:Concept so the self-referencing parent_entity_id hierarchy (e.g. Group -> Company -> Plant) is expressible with skos:broader/skos:narrower on instances." ; + rdfs:comment "A corporate_entity row grounded as a W3C ORG organization. Its authoritative parent_entity_id containment is exposed through :subOrganizationOf; SKOS is reserved for the separate level-classification concepts." ; :lookupCode "node_corporate_entity" . +:CorporateEntityLevel a owl:Class ; + rdfs:subClassOf skos:Concept ; + rdfs:label "Corporate entity level" ; + rdfs:comment "A controlled SKOS classification concept describing whether a corporate entity is a Group, Company, or Plant." . + :Team a owl:Class ; rdfs:subClassOf org:OrganizationalUnit ; rdfs:label "Team" ; @@ -127,10 +137,11 @@ :lookupCode "edge_mention_team" . :teamAffiliatedWith a owl:ObjectProperty ; + rdfs:subPropertyOf org:unitOf ; rdfs:domain :Team ; rdfs:range :CorporateEntity ; rdfs:label "team affiliated with" ; - rdfs:comment "The company a cataloged team belongs to (cataloged_team.affiliated_corporate_entity_id)." ; + rdfs:comment "The company a cataloged team belongs to (cataloged_team.affiliated_corporate_entity_id), specialized from W3C ORG unitOf." ; :lookupCode "edge_team_affiliation" . :mentionsOrganization a owl:ObjectProperty ; @@ -140,6 +151,29 @@ rdfs:comment "A resolved organization is named by a post (post_organization_mention)." ; :lookupCode "edge_mention_organization" . +################################################################# +# Object properties -- authoritative organization containment +################################################################# + +:subOrganizationOf a owl:ObjectProperty ; + rdfs:subPropertyOf org:subOrganizationOf ; + rdfs:domain :CorporateEntity ; + rdfs:range :CorporateEntity ; + rdfs:label "sub-organization of" ; + rdfs:comment "The authoritative direct corporate parent represented by corporate_entity.parent_entity_id, specialized from W3C ORG subOrganizationOf." . + +:hasSubOrganization a owl:ObjectProperty ; + owl:inverseOf :subOrganizationOf ; + rdfs:domain :CorporateEntity ; + rdfs:range :CorporateEntity ; + rdfs:label "has sub-organization" . + +:hasEntityLevel a owl:ObjectProperty ; + rdfs:domain :CorporateEntity ; + rdfs:range :CorporateEntityLevel ; + rdfs:label "has corporate entity level" ; + rdfs:comment "Classifies one corporate entity with exactly one Group, Company, or Plant concept; SHACL owns the closed-world cardinality." . + ################################################################# # Object properties -- entity_relationship_type # (post_counterparty_entity.relationship_type_code) @@ -291,18 +325,18 @@ rdfs:label "Corporate entity level scheme" ; rdfs:comment "The Acme Group -> Acme Electronics Korea -> Acme Electronics Gwangju Plant kind of level, ordered broadest first." . -:GroupLevel a skos:Concept ; +:GroupLevel a :CorporateEntityLevel, skos:Concept ; skos:inScheme :corporateEntityLevelScheme ; skos:prefLabel "Group"@en ; :lookupCode "group" . -:CompanyLevel a skos:Concept ; +:CompanyLevel a :CorporateEntityLevel, skos:Concept ; skos:inScheme :corporateEntityLevelScheme ; skos:broader :GroupLevel ; skos:prefLabel "Company"@en ; :lookupCode "company" . -:PlantLevel a skos:Concept ; +:PlantLevel a :CorporateEntityLevel, skos:Concept ; skos:inScheme :corporateEntityLevelScheme ; skos:broader :CompanyLevel ; skos:prefLabel "Plant"@en ; diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 863dbd055..c71aeee5c 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -11,6 +11,14 @@ Audit anchor: the exact source state carried by this commit at 2026-08-21; record the final PR head with `git rev-parse HEAD` during acceptance. +## 4. Current Stacked PR Product-Surface Gaps +- **Customer Master relationship composition — PR #262**: (Resolved on the current feature branch) The hierarchy, selected customer, and linked evidence were previously stacked vertically, so the selected customer scrolled away while the user inspected relationships and source posts. ADR 0125 and Figma frames `313:2` / `314:2` define a customer-centered three-pane workspace that preserves the WAI-ARIA tree, keeps the selected customer stable, and places source-backed evidence in a separate pane. +- **Responsive Customer Master flow — PR #262**: (Resolved on the current feature branch) PC uses three horizontal panes, tablet uses two columns plus full-width evidence, and phone preserves the semantic order hierarchy → selected customer → evidence at the shared 1024 px / 768 px breakpoints. +- **Effective-dated relationship authority**: (Open) The current Customer Master projection still owns only one `parent_entity_id`. Legal ownership, operating structure, sales roll-up, billing hierarchy, historical roles, and multiple simultaneous relationship types require a normalized, effective-dated relation model before they can be shown as authoritative facts. +- **Unresolved hierarchy repair workflow**: (Open) Cycle, self-parent, and missing-visible-parent members remain safely visible and marked unresolved, but operators still need a source-data quality queue, evidence review, and approved correction workflow. +- **Customer relationship exact-value export**: (Open) The three-pane workspace is accessible and source-backed, but an auditable CSV/JSON export of the selected customer, visible relations, truth status, effective interval, and evidence references remains a later product slice. + +*This document is continuously updated by the hourly automated agent loop.* Current source/test exact head observed before this documentation update: `b897fabe18332081a64a9286e4ab580c79a38f8d`. This documentation update will create the next exact head and therefore requires the protected checks to @@ -191,7 +199,7 @@ adapter, fixture, or HTTP-shaped test double never upgrades a row to | Two-word snake_case database identifiers | ADR 0120, idempotent migration `0104`, live public-schema audit, zero invalid indexes | source + unit + local-integration | | Post list/detail popup, Korean summary, 5W1H, R&R, tickets/calendar | API routes, popup panels, backend/frontend tests | source + unit | | Keyman on both sides, titles, affiliations, related KG nodes | Keyman/affiliate-tree/related-node routes and popup | source + unit; live extraction open | -| Ontology, semantic layer, provenance, W3C PROV-O projection | normalized schema, SKOS operational vocabulary concepts, `ontology_annotations` label fallback, ADR 0124, provenance modules, ADRs, evidence UI | source + unit; corpus verification open | +| Ontology, semantic layer, provenance, W3C PROV-O projection | normalized schema, SKOS operational vocabulary concepts, `ontology_annotations` label fallback, ADR 0132, provenance modules, ADRs, evidence UI | source + unit; corpus verification open | | Branching Event Lineage DAG with evidence trail | `LineageDag.tsx`, Storybook story, Figma frames, accessible node-kind names for screen readers/tooltips, frontend tests; runtime cases include both a rendered DAG and honest empty states, while current corpus coverage remains sparse | source + unit + local-integration partial | | Customer master and hierarchy tree | `/api/customer-master`, affiliate tree, catalog migrations | source + unit; live resolution open | | VOC/VOM/VOP/VOCC/VOCO/VOS role classification | common lookup values and relationship APIs | source + unit; live classification open | diff --git a/docs/superpowers/plans/2026-08-21-external-lineage-integration-contract.md b/docs/superpowers/plans/2026-08-21-external-lineage-integration-contract.md new file mode 100644 index 000000000..ca04be2eb --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-external-lineage-integration-contract.md @@ -0,0 +1,80 @@ +# External Lineage Integration Contract Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish a strict, store-agnostic LineageWeave contract that accepts bounded caller-authorized evidence and returns opaque-reference lineage and project projections for Naruon and other consumers. + +**Architecture:** Keep the existing reconstruction kernel authoritative for candidate scoring and RankWeave fusion. Add a pure contract/parser layer plus a pure execution adapter that performs available-time cutoff filtering, work-budget checks, explicit-vs-inferred relation separation, channel-evidence projection, and canonical digests without database or provider access. Caller-observed children bypass alternative inference and optional model disclosure while remaining available as candidate history for later records. + +**Tech Stack:** Python 3.12+, dataclasses, JSON Schema Draft 2020-12, pytest, coverage.py, Ruff, RankWeave, ThreadWeave. + +**Spec:** `docs/adr/0133-external-email-project-lineage-contract.md` + +## Global Constraints + +- Inputs contain only caller-authorized bounded evidence and opaque references. +- `available_at <= knowledge_cutoff` is the historical-evidence admission rule. +- Missing optional LLM evidence is unavailable, never a fabricated zero. +- Observed RFC/thread relations remain distinct from inferred semantic lineage. +- Children with an explicit observed parent consume no inferred-pair budget and are not sent to the optional LLM for an alternative edge. +- Project projections remain proposed and cannot mutate caller or provider state. +- No direct application-database access, provider credential, persistence, or network call is added to the pure execution adapter. +- Changed production statement and branch coverage must be 100%; public symbols require docstrings. + +--- + +### Task 1: Strict contract and canonical serialization + +**Files:** +- Create: `lineageweave/external_lineage_contract.py` +- Create: `docs/contracts/external-lineage-analysis-v1.schema.json` +- Test: `tests/test_external_lineage_contract.py` + +**Interfaces:** +- Produces: `parse_lineage_analysis_request(payload) -> LineageAnalysisRequest` +- Produces: `serialize_lineage_analysis_request(request) -> dict[str, object]` +- Produces: `serialize_lineage_analysis_result(result) -> dict[str, object]` +- Produces: `request_digest(request) -> str` and `result_digest(result) -> str` + +- [ ] Write failing parser tests for unknown fields, invalid vocabularies, duplicate opaque references, offset-naive timestamps, payload bounds, unsafe references, and policy bounds. +- [ ] Run `uv run --locked --extra dev pytest -q tests/test_external_lineage_contract.py` and confirm the tests fail because the contract does not exist. +- [ ] Implement immutable dataclasses, stable errors, strict parsing, canonical UTC serialization, digest calculation, and result-integrity checks. +- [ ] Add the Draft 2020-12 schema and a drift test comparing its fixed vocabularies and bounds with the parser. +- [ ] Re-run the focused contract tests until they pass. + +### Task 2: Evidence-bounded execution adapter + +**Files:** +- Create: `lineageweave/external_lineage_analysis.py` +- Test: `tests/test_external_lineage_analysis.py` +- Test: `tests/test_external_lineage_explicit_parent_budget.py` + +**Interfaces:** +- Consumes: `LineageAnalysisRequest` and the existing candidate-scoring/fusion kernel. +- Produces: `analyze_external_lineage(request, *, llm=None) -> LineageAnalysisResult`. + +- [ ] Write failing tests for available-time cutoff exclusion, pair-budget rejection before channel execution, explicit observed parent precedence, explicit-parent validation, optional LLM status, project projection, deterministic output, and content-minimized evidence. +- [ ] Add a failing regression proving caller-observed children neither consume inferred-pair budget nor disclose alternative label pairs to an optional LLM. +- [ ] Run the focused execution tests and confirm the missing or defective adapter is the failure cause. +- [ ] Implement request revalidation, explicit-parent acyclicity/group/time checks, cutoff filtering, inference-only pair-budget calculation, core-record adaptation, channel projection, limitations, and result digesting. +- [ ] Preserve an explicit child in candidate history so later unobserved records may still select it as an inferred parent. +- [ ] Re-run the focused execution tests until they pass. + +### Task 3: Public package, decision records, and quality gate + +**Files:** +- Modify: `lineageweave/__init__.py` +- Create: `docs/adr/0133-external-email-project-lineage-contract.md` +- Create: `docs/doctoring/EXTERNAL_LINEAGE_CONTRACT_REFERENCES.md` +- Create: `CHANGELOG.d/external-lineage-contract.md` + +**Interfaces:** +- Produces: a supported package API for consumer contract tests. + +- [ ] Export the contract types, parser/serializer/digest functions, error type, and `analyze_external_lineage` from `lineageweave`. +- [ ] Record the LineageWeave/Naruon authority split, truth statuses, cutoff semantics, model-disclosure minimization, and packaging boundary in the ADR. +- [ ] Record APA 7th sources and one consolidated changelog fragment. +- [ ] Run `uvx ruff check` on changed Python and test files. +- [ ] Run focused statement/branch coverage with `--fail-under=100` for both new production modules. +- [ ] Run documentation hygiene, schema JSON parsing, Python compileall, and `git diff --check`. +- [ ] Open a Draft PR linked to LineageWeave #338; keep Naruon runtime integration out of this slice. diff --git a/docs/superpowers/plans/2026-08-21-naruon-email-project-lineage-contract.md b/docs/superpowers/plans/2026-08-21-naruon-email-project-lineage-contract.md new file mode 100644 index 000000000..63f401f4a --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-naruon-email-project-lineage-contract.md @@ -0,0 +1,68 @@ +# Naruon Email and Project Lineage Contract Implementation Plan + +> **For agentic workers:** Use `superpowers:subagent-driven-development` or `superpowers:executing-plans` task by task. Follow TDD and verify exact-head evidence before publication. + +**Goal:** Publish a strict, store-agnostic LineageWeave package contract that Naruon can later consume for evidence-bounded email lineage and project-history candidates. + +**Architecture:** `external_lineage_contract.py` owns immutable request/result models, strict parsing, canonical serialization, bounded vocabularies, and deterministic SHA-256 digests. `external_lineage_analysis.py` adapts authorized records to the current reconstruction kernel, enforces `available_at <= knowledge_cutoff`, validates caller-observed parent relations, rejects excess pair work before optional provider activity, and projects inferred channel evidence plus proposed project groupings. + +**Tech stack:** Python 3.12+, standard-library dataclasses/JSON/datetime/hashlib, existing LineageWeave reconstruction kernel, pytest, coverage.py, JSON Schema Draft 2020-12. + +## Global constraints + +- Contract version: `1.0.0`. +- Request records: 1–500. +- Candidate window: 1–200. +- Candidate-pair budget: 1–5,000 and enforced before optional LLM calls. +- Identifiers are opaque bounded references; provider credentials and direct database access are forbidden. +- Timestamps are offset-aware RFC 3339 and serialize in UTC with `Z`. +- Caller parent relations remain `observed`, are same-group and acyclic; reconstructed edges remain `inferred`. +- Missing LLM evidence is unavailable, never zero. +- Project projections remain `proposed`. +- New production statement/branch coverage and public docstrings: 100%. + +## Task 1 — Strict contract + +**Files:** +- `lineageweave/external_lineage_contract.py` +- `tests/test_external_lineage_contract.py` + +- [x] Write failing tests for strict object parsing, unknown fields, duplicate references, bounded identifiers/text, offset-aware timestamps, policy bounds, canonical serialization, deterministic digests, result partitions, channel math, and package exports. +- [x] Confirm RED before implementation. +- [x] Implement frozen dataclasses, `LineageContractError`, parser, request/result serializers, and digest functions. +- [x] Confirm focused tests GREEN. + +## Task 2 — Reconstruction adapter + +**Files:** +- `lineageweave/external_lineage_analysis.py` +- `tests/test_external_lineage_analysis.py` + +- [x] Write failing tests for cutoff filtering, explicit RFC parent precedence, cycle/missing/forward-parent rejection, pair-budget pre-call enforcement, LLM status, per-channel evidence, project grouping, and deterministic results. +- [x] Confirm RED before implementation. +- [x] Implement request round-trip validation, available-time partitioning, explicit-parent validation, exact candidate-pair budgeting, core-kernel adaptation, observed/inferred edge projection, limitations, and project projections. +- [x] Confirm focused tests GREEN. + +## Task 3 — Public schema and architecture evidence + +**Files:** +- `docs/contracts/external-lineage-analysis-v1.schema.json` +- `docs/adr/0133-external-email-project-lineage-contract.md` +- `docs/doctoring/EXTERNAL_LINEAGE_CONTRACT_REFERENCES.md` +- `CHANGELOG.d/external-lineage-contract.md` +- `lineageweave/__init__.py` + +- [x] Add JSON Schema Draft 2020-12 mirroring parser names, bounds, and vocabularies. +- [x] Add ADR 0133 and APA 7th references for RFC 3339, RFC 5322, RFC 5256, PROV-O, and OWL-Time. +- [x] Export the contract and adapter from the package root. +- [x] Add changelog evidence. + +## Task 4 — Exact-head verification and Draft PR + +- [x] Focused suite: `57 passed`. +- [x] Isolated new-module coverage: 425/425 statements and 140/140 branches, 100%. +- [x] Public function/class/module docstrings: complete. +- [x] `compileall`, JSON syntax, line-length, and `git diff --check`: passed. +- [ ] Publish the branch from exact protected `main@2feba74b75863810869cde680b19032a93fba413`. +- [ ] Open one Draft PR tracking LineageWeave #338. +- [ ] Keep Draft until exact-head hosted CI/security/documentation gates, review-thread resolution, and independent approval pass. diff --git a/docs/superpowers/specs/2026-08-21-naruon-email-project-lineage-contract-design.md b/docs/superpowers/specs/2026-08-21-naruon-email-project-lineage-contract-design.md new file mode 100644 index 000000000..10a6b3ddb --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-naruon-email-project-lineage-contract-design.md @@ -0,0 +1,108 @@ +# Naruon Email and Project Lineage Contract Design + +## Status + +Accepted for implementation on 2026-08-21 through the user instruction to continue the cross-repository LineageWeave/Naruon integration work. + +## Problem + +Naruon owns customer mail, canonical message/thread identities, projects, tasks, commitments, provider credentials, authorization, and provider mutations. LineageWeave owns lineage reconstruction and the evidence explaining that reconstruction. A future integration needs a released, store-agnostic boundary between those products. Direct database access, copied source, and mutable submodules would collapse their authority boundaries. + +## Decision + +LineageWeave will publish a strict versioned Python contract that accepts bounded caller-authorized evidence and returns only opaque-reference lineage results. The first slice is an in-process, store-agnostic package boundary with no persistence or network access. Naruon can later consume the same schema through a package, service, or reviewed plugin adapter. + +The contract has two layers: + +1. `external_lineage_contract.py` strictly parses and serializes version `1.0.0` requests and results, enforces bounds, normalizes offset-aware timestamps, and computes deterministic content digests. +2. `external_lineage_analysis.py` adapts authorized records to the existing LineageWeave reconstruction kernel, preserves caller-observed parent relations ahead of inference, exposes per-channel evidence, enforces knowledge cutoffs using `available_at`, and emits project evidence groupings without promoting them to Naruon project truth. + +## Authority and truth model + +- Caller-supplied records are `observed` or `authoritative_in_caller` evidence. +- Caller-supplied explicit parent relations remain `observed`; they are not reclassified as semantic inference. +- LineageWeave reconstructed continuation edges are `inferred`. +- Project projections are `proposed` groupings from caller-supplied project references; they never claim authoritative Naruon project state. +- Missing LLM evidence is `unavailable`, never a numeric zero. +- Provider credentials, access tokens, mailbox access, and unrelated tenant data are outside the contract. + +## Request contract + +A request carries: + +- immutable `analysis_id`; +- `analysis_scope_code` in `email_lineage`, `project_history`, or `generic_lineage`; +- optional offset-aware `knowledge_cutoff`; +- bounded reconstruction policy, including a maximum of 5,000 declared candidate-pair evaluations; +- one to 500 evidence records. + +Each evidence record carries: + +- opaque `evidence_ref` and `group_ref`; +- source kind and caller truth status; +- bounded label text; +- offset-aware `occurred_at` and `available_at`; +- optional single `secondary_key` used by the current reconstruction kernel; +- optional `project_ref` used only for proposed project grouping; +- optional explicit parent relation with a controlled relation code. + +## Result contract + +A result carries: + +- deterministic `result_digest`; +- included and cutoff-excluded evidence references; +- LLM channel status; +- stable-sorted edges; +- per-channel score, normalized active weight, and contribution; +- proposed project groupings; +- explicit limitations. + +## Temporal safety + +Historical analysis is governed by: + +```text +available_at <= knowledge_cutoff +``` + +`occurred_at` describes the event/message time; `available_at` describes when the caller could use the evidence. Evidence first available after the cutoff is excluded even if it describes an earlier event. + +## Email safety + +RFC reply/thread evidence and semantic lineage are separate: + +- `rfc_reply`, `provider_reply`, and `manual_parent` are caller-observed explicit relations. +- `reconstructed_continuation` is a LineageWeave inference. +- Provider thread IDs or caller project keys can be supplied only as opaque secondary keys. +- The package never parses a mailbox, fetches a provider, or mutates mail state. + +## Project safety + +The result may group evidence under caller-supplied opaque `project_ref` values. This is a proposed evidence projection only. Naruon must apply its own deterministic or human approval policy before updating authoritative project/task/commitment state. + +## Error handling + +Unknown fields, duplicate references, unsafe or empty identifiers, naive timestamps, non-finite scores, invalid policy bounds, missing explicit parents, forward-inconsistent explicit parents, and unsupported vocabularies fail closed with `LineageContractError` and stable reason codes. + +## Testing + +The implementation uses TDD and must prove: + +- strict parsing and canonical timestamp normalization; +- bounded payloads, duplicate rejection, and pre-provider candidate-pair budget enforcement; +- knowledge-cutoff exclusion by available time; +- observed explicit relations override inferred parent choices; +- RFC reply evidence remains distinct from semantic/project inference; +- absent LLM evidence is explicit; +- deterministic request/result digests; +- no omitted evidence reference can appear in output; +- proposed project groupings never claim caller authority; +- statement and branch coverage for the new production modules are 100%. + +## Standards + +- RFC 3339 for offset-aware timestamps. +- RFC 5322 and RFC 5256 for preserving email identity/thread evidence distinctions. +- W3C PROV-O for provenance and evidence authority. +- W3C OWL-Time for temporal interpretation boundaries. diff --git a/frontend/package.json b/frontend/package.json index c6a4389de..96c2b72ba 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "2.14.0", + "version": "2.16.0", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 5c2c568de..926d23009 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -100,10 +100,12 @@ describe("App, authenticated", () => { pluralAffiliations?: boolean; deferMe?: boolean; deferPosts?: boolean; + deferSecondAsk?: boolean; meFailed?: boolean; postBody?: string; manyCustomerHints?: number; customerEntityHierarchy?: boolean; + deferCustomerRelated?: boolean; boardPosts?: { post_id: string; post_title: string; @@ -134,7 +136,13 @@ describe("App, authenticated", () => { }[]; staleSummary?: boolean; contentAfterSummary?: boolean; - }): ReturnType & { releaseMe: () => void; releasePosts: () => void } { + }): ReturnType & { + releaseMe: () => void; + releaseSecondAsk: () => void; + releaseGroupRelated: () => void; + releaseDemoRelated: () => void; + releasePosts: () => void; + } { const statusLabel: Record = { open: "Open", in_progress: "In progress", @@ -167,6 +175,25 @@ describe("App, authenticated", () => { releaseMe = resolve; }) : Promise.resolve(); + let releaseSecondAsk = () => {}; + const secondAskReady = options?.deferSecondAsk + ? new Promise((resolve) => { + releaseSecondAsk = resolve; + }) + : Promise.resolve(); + let askRequestCount = 0; + let releaseGroupRelated = () => {}; + let releaseDemoRelated = () => {}; + const groupRelatedReady = options?.deferCustomerRelated + ? new Promise((resolve) => { + releaseGroupRelated = resolve; + }) + : Promise.resolve(); + const demoRelatedReady = options?.deferCustomerRelated + ? new Promise((resolve) => { + releaseDemoRelated = resolve; + }) + : Promise.resolve(); const postsReady = options?.deferPosts ? new Promise((resolve) => { releasePosts = resolve; @@ -262,6 +289,9 @@ describe("App, authenticated", () => { if (url.endsWith("/api/posts/post-1/activity") && method === "GET") { return Promise.resolve(jsonResponse({ events })); } + if (url.endsWith("/api/posts/post-2/activity") && method === "GET") { + return Promise.resolve(jsonResponse({ events: [] })); + } if (url.endsWith("/api/posts/post-1/derive-commitment") && method === "POST") { if (options?.chatUnavailable) { return Promise.resolve( @@ -1292,6 +1322,17 @@ describe("App, authenticated", () => { }), ); } + if (url.endsWith("/api/posts/post-2/summary")) { + return Promise.resolve( + jsonResponse({ + post_id: "post-2", + korean_summary: "연결된 글입니다.", + key_events: [], + roles_and_responsibilities: [], + project_mentions: [], + }), + ); + } if (url.endsWith("/api/posts/post-1/keymen")) { return Promise.resolve( jsonResponse({ @@ -1456,6 +1497,35 @@ describe("App, authenticated", () => { }), ); } + if (url.endsWith("/api/corporate-entities/corp-group/related")) { + return groupRelatedReady.then(() => + jsonResponse({ + corporate_entity_id: "corp-group", + entity_name: "Demo Group", + related: [], + }), + ); + } + if (url.endsWith("/api/corporate-entities/corp-demo/related")) { + return demoRelatedReady.then(() => + jsonResponse({ + corporate_entity_id: "corp-demo", + entity_name: "Demo Corp", + related: [ + { + node_id: "post-1", + node_type_code: "node_post", + ontology_iri: "https://contextualwisdomlab.github.io/lineageweave/ontology#Post", + ontology_label: "Post", + label: "Public post", + relevance: 0.8, + post_body_excerpt: "The full body text.", + post_body_truncated: false, + }, + ], + }), + ); + } if (url.endsWith("/api/posts/post-1/affiliate-tree")) { return Promise.resolve( jsonResponse({ @@ -1573,6 +1643,15 @@ describe("App, authenticated", () => { }), ); } + if (url.endsWith("/api/posts/post-2/lineage")) { + return Promise.resolve( + jsonResponse({ + post_id: "post-2", + direct: [{ post_id: "post-1", post_title: "Public post" }], + indirect: [], + }), + ); + } if (url.endsWith("/api/posts/post-1/chat") && method === "GET") { return Promise.resolve( jsonResponse({ @@ -1623,7 +1702,13 @@ describe("App, authenticated", () => { ); } if (url.endsWith("/api/ask") && method === "POST") { - return Promise.resolve( + askRequestCount += 1; + const ready = + options?.deferSecondAsk && askRequestCount === 2 + ? secondAskReady + : Promise.resolve(); + return ready.then(() => + Promise.resolve( jsonResponse({ answer_text: "The cited project is supported by the stored semantic evidence.", cited_post_ids: ["post-2"], @@ -1639,6 +1724,7 @@ describe("App, authenticated", () => { ], source_post_ids: ["post-1", "post-2"], }), + ), ); } if (url.endsWith("/api/customer-master") && method === "GET") { @@ -1734,7 +1820,13 @@ describe("App, authenticated", () => { return Promise.reject(new Error(`unexpected fetch: ${method} ${url}`)); }); vi.stubGlobal("fetch", fetchMock); - return Object.assign(fetchMock, { releaseMe, releasePosts }); + return Object.assign(fetchMock, { + releaseMe, + releaseSecondAsk, + releaseGroupRelated, + releaseDemoRelated, + releasePosts, + }); } it("renders safe Ask Agent evidence under each cited post", async () => { @@ -1767,6 +1859,40 @@ describe("App, authenticated", () => { expect(input).toHaveValue(""); }); + it("keeps prior Ask evidence visible while a new conversation turn is pending", async () => { + const fetchMock = stubBackend({ deferSecondAsk: true }); + render(); + + await userEvent.click(await screen.findByRole("button", { name: "Ask Agent" })); + const ask = await screen.findByRole("region", { name: "Ask Agent" }); + const question = within(ask).getByRole("textbox", { name: "Ask a question" }); + await userEvent.type(question, "Which project?"); + await userEvent.click(within(ask).getByRole("button", { name: "Ask" })); + expect( + await within(ask).findByText( + "The cited project is supported by the stored semantic evidence.", + ), + ).toBeInTheDocument(); + + await userEvent.clear(question); + await userEvent.type(question, "Which person?"); + await userEvent.click(within(ask).getByRole("button", { name: "Ask" })); + + expect(within(ask).getByText("Thinking...")).toBeInTheDocument(); + expect( + within(ask).queryByText( + "The cited project is supported by the stored semantic evidence.", + ), + ).toBeInTheDocument(); + + fetchMock.releaseSecondAsk(); + expect( + await within(ask).findByText( + "The cited project is supported by the stored semantic evidence.", + ), + ).toBeInTheDocument(); + }); + it("labels the Customer Master entity level and Keymen side, never the raw lookup code", async () => { // Live UI finding (2026-08-19): read_customer_master() skipped the // common_lookup_value join both endpoints elsewhere already use, @@ -2083,6 +2209,87 @@ describe("App, authenticated", () => { expect(screen.queryByRole("status", { name: "Event Lineage next action" })).not.toBeInTheDocument(); }); + it("opening a Customer master related post focuses Event Lineage; a home list open does not", async () => { + stubBackend(); + render(); + + await userEvent.click(await screen.findByRole("button", { name: "Customer master" })); + const customers = await screen.findByRole("region", { name: "Customer master" }); + expect(within(customers).getByLabelText("Next action")).toHaveTextContent( + "Authorized customer entities are current. Open a related post to read Event Lineage.", + ); + await userEvent.click(within(customers).getByRole("treeitem", { name: /Demo Corp/ })); + await userEvent.click( + await within(customers).findByRole("button", { name: "Open related post: Public post" }), + ); + + await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); + expect(document.getElementById("post-event-lineage")).toHaveFocus(); + expect(screen.getByRole("status", { name: "Event Lineage next action" })).toHaveTextContent( + "Public post is current in Event Lineage. Read Keyman and evaluation next.", + ); + + await userEvent.click(screen.getByRole("button", { name: "Close" })); + const boardAfterCustomer = screen.getByRole("region", { name: "Board" }); + await userEvent.click( + within(boardAfterCustomer).getByRole("button", { name: "View post: Public post" }), + ); + await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); + expect(document.getElementById("post-event-lineage")).not.toHaveFocus(); + expect(screen.queryByRole("status", { name: "Event Lineage next action" })).not.toBeInTheDocument(); + }); + + it("keeps the current Customer master loading state when an older request finishes", async () => { + const fetchMock = stubBackend({ + customerEntityHierarchy: true, + deferCustomerRelated: true, + }); + render(); + + await userEvent.click(await screen.findByRole("button", { name: "Customer master" })); + const customers = await screen.findByRole("region", { name: "Customer master" }); + await userEvent.click(within(customers).getByRole("treeitem", { name: /Demo Group/ })); + await userEvent.click(within(customers).getByRole("treeitem", { name: /Demo Corp/ })); + + fetchMock.releaseGroupRelated(); + await waitFor(() => expect(within(customers).getByText("Loading related posts...")).toBeInTheDocument()); + expect(within(customers).queryByText("No linked posts yet.")).not.toBeInTheDocument(); + + fetchMock.releaseDemoRelated(); + expect( + await within(customers).findByRole("button", { name: "Open related post: Public post" }), + ).toBeInTheDocument(); + }); + + it("opening an Ask Agent cited post focuses Event Lineage; a home list open does not", async () => { + stubBackend(); + render(); + + await userEvent.click(await screen.findByRole("button", { name: "Ask Agent" })); + const ask = await screen.findByRole("region", { name: "Ask Agent" }); + await userEvent.type(within(ask).getByRole("textbox", { name: "Ask a question" }), "Which project?"); + await userEvent.click(within(ask).getByRole("button", { name: "Ask" })); + expect(await within(ask).findByLabelText("Next action")).toHaveTextContent( + "Authorized cited posts are current. Open a cited post to read Event Lineage.", + ); + await userEvent.click(within(ask).getByRole("button", { name: "Open cited post: Linked post" })); + + await waitFor(() => + expect(screen.getByText("The evidence panel should show exactly this text.")).toBeInTheDocument(), + ); + expect(document.getElementById("post-event-lineage")).toHaveFocus(); + expect(screen.getByRole("status", { name: "Event Lineage next action" })).toHaveTextContent( + "Linked post is current in Event Lineage. Read Keyman and evaluation next.", + ); + + await userEvent.click(screen.getByRole("button", { name: "Close" })); + const boardAfterAsk = screen.getByRole("region", { name: "Board" }); + await userEvent.click(within(boardAfterAsk).getByRole("button", { name: "View post: Public post" })); + await waitFor(() => expect(screen.getByText("The full body text.")).toBeInTheDocument()); + expect(document.getElementById("post-event-lineage")).not.toHaveFocus(); + expect(screen.queryByRole("status", { name: "Event Lineage next action" })).not.toBeInTheDocument(); + }); + it("renders the A-100 fork as a git-style DAG, not a flat edge list", async () => { stubBackend(); render(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 7bbf21eeb..c5d8e85f9 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -56,7 +56,6 @@ import { type ChatAnswer, type ChatExchange, type CorporateEntityRef, - type CustomerMasterEntity, type CustomerMasterResponse, type Counterparty, type EvaluationResponse, @@ -93,6 +92,7 @@ import { LineageDag } from "./LineageDag"; import { PostBody } from "./PostBody"; import { decodeHtmlEntities } from "./postBodyDisplay"; import { FiveW1H } from "./components/FiveW1H"; +import { CustomerMasterTree, CustomerRelatedPostCard } from "./components/CustomerMasterTree"; import { subgraphForPost } from "./lineageLayout"; import { isSupportedLocale, @@ -2590,10 +2590,12 @@ type SelectPostOptions = { fromReportMember?: boolean; fromWeeklyVoc?: boolean; fromCalendar?: boolean; + fromCustomerMaster?: boolean; /** Set when re-entering a post from a popstate (browser back/forward) so * the handler doesn't push a duplicate history entry for a navigation * the browser already performed. */ fromPopState?: boolean; + fromAskAgent?: boolean; }; /** @@ -3644,6 +3646,8 @@ function PostList({ showLabPanels = false, postIdToOpen = null, postOpenFromCalendar = false, + postOpenFromCustomerMaster = false, + postOpenFromAskAgent = false, onPostOpened, focusSearchRequest = 0, onSearchFocusHandled, @@ -3652,6 +3656,8 @@ function PostList({ showLabPanels?: boolean; postIdToOpen?: string | null; postOpenFromCalendar?: boolean; + postOpenFromCustomerMaster?: boolean; + postOpenFromAskAgent?: boolean; onPostOpened?: () => void; focusSearchRequest?: number; onSearchFocusHandled?: () => void; @@ -3674,6 +3680,8 @@ function PostList({ const [openedFromReportMember, setOpenedFromReportMember] = useState(false); const [openedFromWeeklyVoc, setOpenedFromWeeklyVoc] = useState(false); const [openedFromCalendar, setOpenedFromCalendar] = useState(false); + const [openedFromCustomerMaster, setOpenedFromCustomerMaster] = useState(false); + const [openedFromAskAgent, setOpenedFromAskAgent] = useState(false); const [corporateEntities, setCorporateEntities] = useState(null); const [entitiesLoadError, setEntitiesLoadError] = useState(null); const [totalPosts, setTotalPosts] = useState(0); @@ -3748,6 +3756,7 @@ function PostList({ setOpenedFromReportMember(Boolean(options?.fromReportMember)); setOpenedFromWeeklyVoc(Boolean(options?.fromWeeklyVoc)); setOpenedFromCalendar(Boolean(options?.fromCalendar)); + setOpenedFromCustomerMaster(Boolean(options?.fromCustomerMaster)); if (!options?.fromPopState) { const url = new URL(window.location.href); if (url.searchParams.get("post") !== postId) { @@ -3755,13 +3764,24 @@ function PostList({ window.history.pushState({}, "", `${url.pathname}${url.search}${url.hash}`); } } + setOpenedFromAskAgent(Boolean(options?.fromAskAgent)); } useEffect(() => { if (!postIdToOpen) return; - selectPost(postIdToOpen, postOpenFromCalendar ? { fromCalendar: true } : undefined); + selectPost(postIdToOpen, { + fromCalendar: postOpenFromCalendar, + fromCustomerMaster: postOpenFromCustomerMaster, + fromAskAgent: postOpenFromAskAgent, + }); onPostOpened?.(); - }, [onPostOpened, postIdToOpen, postOpenFromCalendar]); + }, [ + onPostOpened, + postIdToOpen, + postOpenFromCalendar, + postOpenFromCustomerMaster, + postOpenFromAskAgent, + ]); useEffect(() => { function handlePopState() { @@ -3786,6 +3806,8 @@ function PostList({ setOpenedFromReportMember(false); setOpenedFromWeeklyVoc(false); setOpenedFromCalendar(false); + setOpenedFromCustomerMaster(false); + setOpenedFromAskAgent(false); const url = new URL(window.location.href); if (url.searchParams.has("post")) { url.searchParams.delete("post"); @@ -4274,7 +4296,13 @@ function PostList({ openedAfterCutoff ? analysisRunOpenedBodyWarning(openedCutoffIso) : null } knowledgeCutoff={openedAfterCutoff ? openedCutoffIso : null} - focusEventLineage={openedFromReportMember || openedFromWeeklyVoc || openedFromCalendar} + focusEventLineage={ + openedFromReportMember || + openedFromWeeklyVoc || + openedFromCalendar || + openedFromCustomerMaster || + openedFromAskAgent + } focusAskOnLand={openedFromReportMember} onClose={closeSelectedPost} onSelectPost={selectPost} @@ -4285,148 +4313,6 @@ function PostList({ ); } -interface CustomerEntityTreeNode { - entity: CustomerMasterEntity; - children: CustomerEntityTreeNode[]; -} - -// Live bug (2026-08-19): Customer Master's own entity list rendered every -// corporate_entity as an independent top-level row, even though the API -// already carries parent_entity_id and the codebase already knows how to -// build a real forest from it (lineageweave/affiliate_tree.py, used for -// the post-detail popup's Affiliate tree) -- a group holding company and -// its subsidiaries showed up as an unrelated flat list with no visual -// hierarchy at all. A parent not present in this account's own visible -// entity list (a real possibility -- ABAC can authorize a child entity -// without its parent) is not dropped; that entity becomes a root here -// instead of disappearing. -function buildCustomerEntityTree(entities: CustomerMasterEntity[]): CustomerEntityTreeNode[] { - const byId = new Map(entities.map((entity) => [entity.corporate_entity_id, entity])); - const childrenByParent = new Map(); - const roots: CustomerMasterEntity[] = []; - for (const entity of entities) { - if (entity.parent_entity_id && byId.has(entity.parent_entity_id)) { - const siblings = childrenByParent.get(entity.parent_entity_id) ?? []; - siblings.push(entity); - childrenByParent.set(entity.parent_entity_id, siblings); - } else { - roots.push(entity); - } - } - const toNode = (entity: CustomerMasterEntity): CustomerEntityTreeNode => ({ - entity, - children: (childrenByParent.get(entity.corporate_entity_id) ?? []).map(toNode), - }); - return roots.map(toNode); -} - -function CustomerEntityTreeRow({ - node, - depth, - expandedEntityId, - relatedByEntity, - relatedLoading, - onToggle, - onOpenPost, -}: { - node: CustomerEntityTreeNode; - depth: number; - expandedEntityId: string | null; - relatedByEntity: Record; - relatedLoading: string | null; - onToggle: (entityId: string) => void; - onOpenPost: (postId: string) => void; -}) { - const { entity, children } = node; - const relatedPosts = (relatedByEntity[entity.corporate_entity_id] ?? []).filter( - (related) => related.node_type_code === NODE_POST, - ); - return ( -
  • - - {expandedEntityId === entity.corporate_entity_id ? ( -
    - {relatedLoading === entity.corporate_entity_id ?

    {t("Loading related posts...")}

    : null} - {relatedLoading !== entity.corporate_entity_id && relatedPosts.length === 0 ? ( -

    {t("No linked posts yet.")}

    - ) : null} - {relatedPosts.length > 0 ? ( -
      - {relatedPosts.map((related) => ( -
    • - -
      • - ))} -
    - ) : null} -
    - ) : null} - {children.length > 0 ? ( -
      - {children.map((child) => ( - - ))} -
    - ) : null} -
    • - ); -} - -function CustomerRelatedPostCard({ - postId, - postTitle, - postBodyExcerpt, - postBodyTruncated, - onOpenPost, -}: { - postId: string; - postTitle: string; - postBodyExcerpt?: string | null; - postBodyTruncated?: boolean; - onOpenPost: (postId: string) => void; -}) { - return ( - - ); -} - function CustomerMasterPanel({ accessToken, onOpenPost, @@ -4436,9 +4322,6 @@ function CustomerMasterPanel({ }) { const [master, setMaster] = useState(null); const [error, setError] = useState(null); - const [expandedEntityId, setExpandedEntityId] = useState(null); - const [relatedByEntity, setRelatedByEntity] = useState>({}); - const [relatedLoading, setRelatedLoading] = useState(null); const [resolvingHint, setResolvingHint] = useState(null); const [resolveError, setResolveError] = useState(null); // Fetched independently, same pattern as PostList's own canRebuild -- @@ -4485,49 +4368,32 @@ function CustomerMasterPanel({ } } - async function toggleEntity(entityId: string) { - if (expandedEntityId === entityId) { - setExpandedEntityId(null); - return; - } - setExpandedEntityId(entityId); - if (relatedByEntity[entityId]) return; - setRelatedLoading(entityId); - try { - const response = await fetchRelatedEntity(accessToken, entityId); - setRelatedByEntity((previous) => ({ ...previous, [entityId]: response.related })); - } catch { - setRelatedByEntity((previous) => ({ ...previous, [entityId]: [] })); - } finally { - setRelatedLoading(null); - } - } + const loadRelated = useCallback( + async (entityId: string) => (await fetchRelatedEntity(accessToken, entityId)).related, + [accessToken], + ); return (

    {t("Authorized customer scope")}

    {t("Customer master")}

    {t("Customer entities available to this account.")}

    + {master && master.corporate_entities.length > 0 ? ( +

    + {t("Authorized customer entities are current. Open a related post to read Event Lineage.")} +

    + ) : null} {error ?

    {error}

    : null} {master === null && !error ?

    {t("Loading customer master...")}

    : null} {master?.corporate_entities.length === 0 ? (

    {t("No customer entities are connected to this account.")}

    ) : null} {master && master.corporate_entities.length > 0 ? ( -
      - {buildCustomerEntityTree(master.corporate_entities).map((node) => ( - - ))} -
    + ) : null} {master && (master.relationship_network ?? []).length > 0 ? (
    @@ -4721,7 +4587,12 @@ function AskAgentPanel({ } return ( -
    +

    {t("Evidence-grounded questions")}

    {t("Ask Agent")}

    @@ -4753,14 +4624,26 @@ function AskAgentPanel({ {exchange.status === "pending" ?

    {t("Thinking...")}

    : null} {exchange.status === "error" ?

    {exchange.error}

    : null} {response?.answer_text ?

    {response.answer_text}

    : null} - {response?.next_action ?

    {t(response.next_action)}

    : null} + {response?.next_action ? ( +

    + {t(response.next_action)} +

    + ) : response?.cited_posts?.length ? ( +

    + {t("Authorized cited posts are current. Open a cited post to read Event Lineage.")} +

    + ) : null} {response?.cited_posts && response.cited_posts.length > 0 ? (

    {t("Cited posts")}

      {response.cited_posts.map((post) => (
    • - @@ -4835,6 +4718,8 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean return new URLSearchParams(window.location.search).get("post"); }); const [postOpenFromCalendar, setPostOpenFromCalendar] = useState(false); + const [postOpenFromCustomerMaster, setPostOpenFromCustomerMaster] = useState(false); + const [postOpenFromAskAgent, setPostOpenFromAskAgent] = useState(false); // Test-only compatibility for legacy analysis-panel coverage; this prop // never forces the panels open outside Vitest. In a real build the // advanced-review section (ADR 0037) is gated on PostList's own @@ -5027,9 +4912,13 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean showLabPanels={testOnlyLabPanels} postIdToOpen={postToOpen} postOpenFromCalendar={postOpenFromCalendar} + postOpenFromCustomerMaster={postOpenFromCustomerMaster} + postOpenFromAskAgent={postOpenFromAskAgent} onPostOpened={() => { setPostToOpen(null); setPostOpenFromCalendar(false); + setPostOpenFromCustomerMaster(false); + setPostOpenFromAskAgent(false); }} focusSearchRequest={searchFocusRequest} onSearchFocusHandled={() => setSearchFocusRequest(0)} @@ -5040,6 +4929,9 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean accessToken={accessToken} onOpenPost={(postId) => { setPostToOpen(postId); + setPostOpenFromCalendar(false); + setPostOpenFromCustomerMaster(true); + setPostOpenFromAskAgent(false); changeDestination("board"); }} /> @@ -5052,6 +4944,8 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean onSelectPost={(postId) => { setPostToOpen(postId); setPostOpenFromCalendar(true); + setPostOpenFromCustomerMaster(false); + setPostOpenFromAskAgent(false); changeDestination("board"); }} /> @@ -5061,6 +4955,9 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean accessToken={accessToken} onOpenPost={(postId) => { setPostToOpen(postId); + setPostOpenFromCalendar(false); + setPostOpenFromCustomerMaster(false); + setPostOpenFromAskAgent(true); changeDestination("board"); }} /> diff --git a/frontend/src/PostBody.stories.tsx b/frontend/src/PostBody.stories.tsx new file mode 100644 index 000000000..358758cb0 --- /dev/null +++ b/frontend/src/PostBody.stories.tsx @@ -0,0 +1,70 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { PostBody } from "./PostBody"; + +const meta = { + title: "Evidence/PostBody", + component: PostBody, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +const TINY_PNG = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + +export const MarkdownTableEvidence: Story = { + args: { + body: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |", + structureUnits: [ + { + unit_index: 0, + unit_kind_code: "plain_text", + unit_label: "markdown_tr", + unit_text: "Workstream | State", + indent_level: 0, + indent_source_code: "unresolved", + indent_confidence: 0, + indent_evidence: "Markdown table row", + }, + { + unit_index: 1, + unit_kind_code: "plain_text", + unit_label: "markdown_tr", + unit_text: "Alpha | Ready", + indent_level: 0, + indent_source_code: "unresolved", + indent_confidence: 0, + indent_evidence: "Markdown table row", + }, + ], + }, +}; + +export const MarkdownTableFallback: Story = { + args: { + body: "Intro.\n\n| Workstream | State |\n| --- | --- |\n| Alpha | Ready |\n\nNext action.", + }, +}; + +export const ImageOcrTableEvidence: Story = { + args: { + body: ``, + imageContent: [ + { + unit_index: 0, + mime_type: "image/png", + status_code: "completed", + extracted_text: "| Workstream | State |\n| --- | --- |\n| Alpha | Ready |", + caption: "A synthetic workstream status table.", + tags: ["table"], + }, + ], + }, +}; + +export const NumericFootnote: Story = { + args: { + body: "

      Evidence remains attached to the source.

      1 Source note.

      ", + }, +}; diff --git a/frontend/src/PostBody.test.tsx b/frontend/src/PostBody.test.tsx index e1322bb96..db4f65f32 100644 --- a/frontend/src/PostBody.test.tsx +++ b/frontend/src/PostBody.test.tsx @@ -133,6 +133,39 @@ describe("PostBody", () => { expect(screen.queryAllByText("No.")).toHaveLength(1); }); + it("renders persisted Markdown table rows as a table", () => { + render( + , + ); + + expect(screen.getByRole("table")).toBeInTheDocument(); + expect(screen.getAllByRole("row")).toHaveLength(2); + }); + it("keeps source indentation after a persisted table unit", () => { render( { expect(screen.getAllByRole("row")).toHaveLength(4); }); + it("renders a raw Markdown table when persisted structure is unavailable", () => { + render( + , + ); + + expect(screen.getByRole("table")).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Project" })).toBeInTheDocument(); + expect(screen.getByText("Intro.")).toBeInTheDocument(); + expect(screen.getByText("Next action.")).toBeInTheDocument(); + }); + + it("keeps a persisted Markdown table row searchable", () => { + render( + , + ); + + expect(screen.getAllByRole("row")).toHaveLength(2); + }); + + it("renders table-shaped image OCR as accessible evidence", () => { + render( + '} + imageContent={[ + { + unit_index: 0, + mime_type: "image/png", + status_code: "completed", + extracted_text: "| Project | Status |\n| --- | --- |\n| Alpha | Ready |", + caption: "A synthetic project status table.", + tags: ["table"], + regions: [ + { + region_index: 0, + x_ratio: 0, + y_ratio: 0, + width_ratio: 1, + height_ratio: 1, + status_code: "completed", + extracted_text: "| Owner | Action |\n| --- | --- |\n| Team A | Review |", + caption: "The table region assigns an action to a team.", + tags: ["assignment"], + }, + ], + }, + ]} + />, + ); + + expect(screen.getAllByRole("table")).toHaveLength(2); + expect(screen.getByRole("columnheader", { name: "Project" })).toBeInTheDocument(); + expect(screen.getByText("Ready")).toBeInTheDocument(); + expect(screen.getByText("The table region assigns an action to a team.")).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Owner" })).toBeInTheDocument(); + }); + + it("keeps an embedded source image when prose also contains a Markdown table", () => { + render( + Before the table

      ' + + "\n\n| Project | Status |\n| --- | --- |\n| Alpha | Ready |" + } + imageContent={[ + { + unit_index: 1, + mime_type: "image/png", + status_code: "described", + extracted_text: "", + caption: "Source image beside the table", + tags: [], + }, + ]} + />, + ); + + expect(screen.getByAltText("Source image beside the table")).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Project" })).toBeInTheDocument(); + expect(screen.getByText("Ready")).toBeInTheDocument(); + }); + it("marks persisted footnotes as footnote evidence", () => { render( { + if (block.kind === "prose") { + return ( + + {splitPostBody(block.text).map((segment, segmentIndex) => + renderSegment( + segment, + blockIndex * 1000 + segmentIndex, + segment.kind === "image" ? imageContent[imageOrdinal++] : undefined, + ), + )} + + ); + } + const [header, ...rows] = block.rows; + return ( + + + + {header.map((cell, cellIndex) => ( + + ))} + + + + {rows.map((row, rowIndex) => ( + + {row.map((cell, cellIndex) => ( + + ))} + + ))} + +
      + {cell} +
      + {cell} +
      + ); + }); +} + export function PostBody({ body, imageContent = [], @@ -299,6 +352,10 @@ export function PostBody({ if (hasPersistedStructuralUnits) { return
      {renderStructuredUnits(body, structureUnits, imageContent)}
      ; } + const markdownBlocks = !hasPersistedStructuralUnits ? splitMarkdownTableBody(body) : null; + if (markdownBlocks) { + return
      {renderMarkdownBlocks(markdownBlocks, imageContent)}
      ; + } return (
      {splitPostBody(body).map((segment, index) => { diff --git a/frontend/src/components/CustomerMasterRelationshipWorkspace.test.tsx b/frontend/src/components/CustomerMasterRelationshipWorkspace.test.tsx new file mode 100644 index 000000000..16fa26c78 --- /dev/null +++ b/frontend/src/components/CustomerMasterRelationshipWorkspace.test.tsx @@ -0,0 +1,227 @@ +import { render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import type { CustomerMasterEntity, RelatedNode } from "../api"; +import { getCustomerMasterWorkspaceCopy } from "../customerMasterWorkspace"; +import { SUPPORTED_LOCALES } from "../i18n"; +import { CustomerMasterTree } from "./CustomerMasterTree"; + +function entity( + id: string, + name: string, + parentEntityId: string | null = null, + level: "Group" | "Company" | "Plant" = "Company", +): CustomerMasterEntity { + return { + corporate_entity_id: id, + corporate_entity_code: id.toUpperCase(), + entity_name: name, + entity_level_code: level.toLowerCase(), + entity_level_label: level, + parent_entity_id: parentEntityId, + }; +} + +const hierarchy = [ + entity("group", "Demo Group", null, "Group"), + entity("company", "Demo Company", "group", "Company"), + entity("plant", "Demo Plant", "company", "Plant"), +]; + +function post(entityId: string): RelatedNode { + return { + node_id: `post-${entityId}`, + node_type_code: "node_post", + relevance: 1, + label: `${entityId} evidence`, + post_body_excerpt: `${entityId} source-backed excerpt`, + post_body_truncated: false, + }; +} + +describe("Customer Master three-pane workspace", () => { + it("centers the selected customer between hierarchy and source-backed evidence", async () => { + const loadRelated = vi.fn(async (entityId: string) => [post(entityId)]); + render( + undefined} + />, + ); + + const workspace = screen.getByRole("heading", { name: "Choose a customer in scope" }) + .closest(".customer-master-relationship-workspace"); + expect(workspace).not.toBeNull(); + expect(workspace?.querySelectorAll(".customer-master-workspace-pane")).toHaveLength(3); + + await userEvent.click(screen.getByRole("treeitem", { name: /Demo Company/ })); + + const focusPane = screen.getByRole("heading", { name: "Customer relationship focus" }) + .closest("section"); + expect(focusPane).not.toBeNull(); + expect(within(focusPane as HTMLElement).getByRole("heading", { name: "Demo Company" })) + .toBeInTheDocument(); + expect( + within(focusPane as HTMLElement).getByRole("button", { + name: "Center this customer: Demo Group", + }), + ).toBeInTheDocument(); + expect( + within(focusPane as HTMLElement).getByRole("button", { + name: "Center this customer: Demo Plant", + }), + ).toBeInTheDocument(); + + const evidence = await screen.findByRole("region", { + name: "Related posts: Demo Company", + }); + expect(screen.getByRole("tree")).not.toContainElement(evidence); + expect(within(evidence).getByText("company source-backed excerpt")).toBeInTheDocument(); + expect(loadRelated).toHaveBeenCalledWith("company"); + }); + + it("recenters through relationship cards without inventing hidden relations", async () => { + render( + [post(entityId)]} + onOpenPost={() => undefined} + />, + ); + + await userEvent.click(screen.getByRole("treeitem", { name: /Demo Company/ })); + await userEvent.click( + screen.getByRole("button", { name: "Center this customer: Demo Group" }), + ); + + const focusPane = screen.getByRole("heading", { name: "Customer relationship focus" }) + .closest("section"); + expect( + within(focusPane as HTMLElement).getByRole("heading", { name: "Demo Group" }), + ).toBeInTheDocument(); + expect(within(focusPane as HTMLElement).getByText( + "No parent organization is visible in the authorized scope.", + )).toBeInTheDocument(); + expect( + within(focusPane as HTMLElement).getByRole("button", { + name: "Center this customer: Demo Company", + }), + ).toBeInTheDocument(); + expect(await screen.findByRole("region", { name: "Related posts: Demo Group" })) + .toBeInTheDocument(); + }); + + it("keeps the customer selected while the evidence pane is closed and reopened from cache", async () => { + const loadRelated = vi.fn(async (entityId: string) => [post(entityId)]); + render( + undefined} + />, + ); + + const company = screen.getByRole("treeitem", { name: /Demo Company/ }); + await userEvent.click(company); + expect(await screen.findByRole("region", { name: "Related posts: Demo Company" })) + .toBeInTheDocument(); + + await userEvent.click(screen.getByRole("button", { name: "Close linked evidence" })); + expect(screen.queryByRole("region", { name: "Related posts: Demo Company" })) + .not.toBeInTheDocument(); + expect(company).toHaveAttribute("aria-selected", "true"); + expect(screen.getByRole("heading", { name: "Demo Company" })).toBeInTheDocument(); + + await userEvent.click(screen.getAllByRole("button", { name: "Open linked evidence" })[0]); + expect(await screen.findByRole("region", { name: "Related posts: Demo Company" })) + .toBeInTheDocument(); + expect(loadRelated).toHaveBeenCalledTimes(1); + }); + + it("supports deterministic selected and unselected Storybook states without fabricating evidence", () => { + const loadRelated = vi.fn(async () => []); + const { rerender } = render( + undefined} + />, + ); + + expect(screen.getByRole("heading", { name: "Demo Company" })).toBeInTheDocument(); + expect(screen.queryByRole("region", { name: "Related posts: Demo Company" })) + .not.toBeInTheDocument(); + expect(loadRelated).not.toHaveBeenCalled(); + + rerender( + undefined} + />, + ); + expect(screen.queryByRole("heading", { name: "Demo Company" })) + .not.toBeInTheDocument(); + expect(screen.getByText( + "Select a customer from the hierarchy to center its relationships.", + )).toBeInTheDocument(); + }); + + it("explains leaf and unresolved relationship boundaries in the center pane", async () => { + render( + []} + onOpenPost={() => undefined} + />, + ); + + await userEvent.click(screen.getByRole("treeitem", { name: /Unresolved Plant.*unresolved/i })); + const focusPane = screen.getByRole("heading", { name: "Customer relationship focus" }) + .closest("section"); + expect(within(focusPane as HTMLElement).getByText( + "This hierarchy relation is unresolved. Review the source data before treating it as authoritative.", + )).toBeInTheDocument(); + expect(within(focusPane as HTMLElement).getByText( + "No parent organization is visible in the authorized scope.", + )).toBeInTheDocument(); + expect(within(focusPane as HTMLElement).getByText( + "No direct child organization is visible in the authorized scope.", + )).toBeInTheDocument(); + await waitFor(() => expect(screen.getByText("No linked posts yet.")).toBeInTheDocument()); + }); + + it("does not show a malformed self-parent as an authoritative parent", async () => { + render( + []} + onOpenPost={() => undefined} + />, + ); + + await userEvent.click(screen.getByRole("treeitem", { name: /Self Parent.*unresolved/i })); + const focusPane = screen.getByRole("heading", { name: "Customer relationship focus" }) + .closest("section"); + expect(within(focusPane as HTMLElement).getByText( + "No parent organization is visible in the authorized scope.", + )).toBeInTheDocument(); + expect(within(focusPane as HTMLElement).queryByRole("button", { + name: "Center this customer: Self Parent", + })).not.toBeInTheDocument(); + }); +}); + +describe("Customer Master workspace localization", () => { + it("provides the same complete copy contract for all product locales", () => { + const englishKeys = Object.keys(getCustomerMasterWorkspaceCopy("en")).sort(); + for (const locale of SUPPORTED_LOCALES) { + const localized = getCustomerMasterWorkspaceCopy(locale); + expect(Object.keys(localized).sort(), locale).toEqual(englishKeys); + expect(Object.values(localized).every((value) => value.trim().length > 0), locale).toBe(true); + } + }); +}); diff --git a/frontend/src/components/CustomerMasterTree.css b/frontend/src/components/CustomerMasterTree.css new file mode 100644 index 000000000..b89c9bae5 --- /dev/null +++ b/frontend/src/components/CustomerMasterTree.css @@ -0,0 +1,386 @@ +.customer-master-relationship-workspace { + display: grid; + grid-template-columns: minmax(16rem, 0.9fr) minmax(18rem, 1fr) minmax(18rem, 1.1fr); + gap: 1rem; + align-items: start; +} + +.customer-master-workspace-pane { + min-width: 0; + padding: 1rem; + border: 1px solid var(--color-border); + border-radius: var(--radius-panel); + background: var(--color-background); +} + +.customer-master-focus-pane { + border: 2px solid var(--color-accent-border); + box-shadow: 0 0 0 1px var(--color-accent-background); +} + +.customer-master-pane-header { + display: grid; + gap: 0.35rem; + margin-bottom: 1rem; +} + +.customer-master-pane-header p, +.customer-master-pane-header h3, +.customer-master-pane-header span { + margin: 0; +} + +.customer-master-pane-header p { + color: var(--color-accent); + font-size: var(--font-size-badge); + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.customer-master-pane-header h3 { + color: var(--color-text-heading); + font-size: 1.05rem; + line-height: 1.35; +} + +.customer-master-pane-header span { + color: var(--color-text); + font-size: 0.83rem; + line-height: 1.5; +} + +.customer-master-tree-widget { + display: grid; + max-height: 34rem; + gap: 0.65rem; + margin: 0; + padding: 0; + overflow: auto; + list-style: none; + scrollbar-gutter: stable; +} + +.customer-master-tree-widget .customer-tree-node { + margin: 0; + padding: 0; + border: 0; + background: transparent; +} + +.customer-master-tree-widget .customer-tree-node:focus-visible { + outline: 3px solid var(--color-focus-border); + outline-offset: 3px; +} + +.customer-master-tree-widget .customer-entity-button { + display: grid; + grid-template-columns: var(--size-control-min) minmax(0, 1fr); + align-items: start; + gap: var(--space-control-gap); + padding: 0.75rem; + border: 1px solid var(--color-border); + border-radius: var(--radius-control); + background: var(--color-background); + cursor: pointer; + transition: border-color 120ms ease, background-color 120ms ease, box-shadow 120ms ease; +} + +.customer-master-tree-widget .customer-entity-button:hover { + border-color: var(--color-accent-border); + background: var(--color-accent-background); +} + +.customer-master-tree-widget .customer-tree-node[aria-selected="true"] > .customer-entity-button { + border-color: var(--color-accent-border); + background: var(--color-accent-background); + box-shadow: inset 3px 0 0 var(--color-accent); +} + +.customer-master-tree-widget .customer-tree-node[data-hierarchy-issue] > .customer-entity-button { + border-color: var(--badge-status-danger-text); +} + +.customer-tree-branch-indicator, +.customer-tree-branch-spacer { + display: inline-grid; + place-items: center; + inline-size: var(--size-control-min); + block-size: var(--size-control-min); + border-radius: var(--radius-control); +} + +.customer-tree-branch-indicator { + border: 1px solid var(--color-border); + background: var(--color-background); + color: var(--color-text-heading); +} + +.customer-entity-button:hover .customer-tree-branch-indicator, +.customer-tree-node:focus-visible > .customer-entity-button .customer-tree-branch-indicator { + border-color: var(--color-accent-border); + background: var(--color-accent-background); +} + +.customer-tree-label { + display: flex; + min-width: 0; + flex-direction: column; + gap: 0.25rem; + align-items: flex-start; +} + +.customer-tree-label strong { + color: var(--color-text-heading); + overflow-wrap: anywhere; +} + +.customer-tree-label > span:not(.customer-tree-unresolved) { + color: var(--color-text); + font-size: 0.8rem; + overflow-wrap: anywhere; +} + +.customer-master-tree-group { + display: grid; + gap: 0.65rem; + margin: 0.65rem 0 0; + padding: 0 0 0 1.1rem; + list-style: none; + border-inline-start: 1px solid var(--color-border-subtle); +} + +.customer-tree-unresolved { + inline-size: fit-content; + padding: var(--space-chip-block) var(--space-chip-inline); + border-radius: var(--radius-chip); + background: var(--badge-status-danger-bg); + color: var(--badge-status-danger-text) !important; + font-size: var(--font-size-badge) !important; + font-weight: 700; +} + +.customer-master-focus-content, +.customer-master-relationship-section, +.customer-master-relationship-list, +.customer-tree-evidence { + display: grid; + gap: 0.75rem; +} + +.customer-master-selected-summary { + display: grid; + gap: 0.35rem; + padding: 1rem; + border-radius: var(--radius-panel); + background: var(--color-accent-background); +} + +.customer-master-selected-summary h4, +.customer-master-selected-summary p { + margin: 0; +} + +.customer-master-selected-summary h4 { + color: var(--color-text-heading); + font-size: 1.15rem; +} + +.customer-master-selected-summary p { + color: var(--color-text); + font-size: 0.85rem; +} + +.customer-master-selected-label, +.customer-master-relationship-kind { + color: var(--color-accent); + font-size: var(--font-size-badge); + font-weight: 700; + letter-spacing: 0.035em; + text-transform: uppercase; +} + +.customer-master-status-badge { + width: fit-content; + padding: var(--space-chip-block) var(--space-chip-inline); + border-radius: var(--radius-chip); + font-size: var(--font-size-badge); + font-weight: 700; +} + +.customer-master-status-verified { + background: var(--badge-status-success-bg); + color: var(--badge-status-success-text); +} + +.customer-master-status-unresolved { + background: var(--badge-status-danger-bg); + color: var(--badge-status-danger-text); +} + +.customer-master-unresolved-callout { + margin: 0; + padding: 0.75rem; + border: 1px solid var(--badge-status-danger-text); + border-radius: var(--radius-control); + background: var(--badge-status-danger-bg); + color: var(--badge-status-danger-text); + font-size: 0.82rem; + line-height: 1.5; +} + +.customer-master-relationship-section h4 { + margin: 0; + color: var(--color-text-heading); + font-size: 0.85rem; +} + +.customer-master-relationship-card { + display: grid; + width: 100%; + gap: 0.25rem; + padding: 0.8rem; + border: 1px solid var(--color-border); + border-radius: var(--radius-control); + background: var(--color-background); + color: var(--color-text); + text-align: left; + cursor: pointer; +} + +.customer-master-relationship-card strong { + color: var(--color-text-heading); +} + +.customer-master-relationship-card:hover { + border-color: var(--color-accent-border); + background: var(--color-accent-background); +} + +.customer-master-relationship-card:focus-visible, +.customer-master-evidence-action:focus-visible, +.customer-master-evidence-close:focus-visible { + outline: 3px solid var(--color-focus-ring); + outline-offset: 2px; +} + +.customer-master-empty-relation, +.customer-master-workspace-placeholder { + margin: 0; + padding: 1rem; + border: 1px dashed var(--color-border); + border-radius: var(--radius-control); + color: var(--color-text); + font-size: 0.85rem; + line-height: 1.5; +} + +.customer-master-evidence-action { + width: 100%; + min-height: 2.75rem; + padding: 0.7rem 1rem; + border: 1px solid var(--color-btn-primary-bg); + border-radius: var(--radius-control); + background: var(--color-btn-primary-bg); + color: var(--color-btn-primary-text); + font-weight: 700; + cursor: pointer; +} + +.customer-master-evidence-action:hover { + background: var(--color-btn-primary-hover); +} + +.customer-master-evidence-action-secondary { + border-color: var(--color-btn-secondary-border); + background: var(--color-btn-secondary-bg); + color: var(--color-btn-secondary-text); +} + +.customer-master-evidence-action-secondary:hover { + background: var(--color-btn-secondary-hover); +} + +.customer-master-evidence-context { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.75rem; + padding: 0.75rem; + border-radius: var(--radius-control); + background: var(--color-accent-background); +} + +.customer-master-evidence-context > div { + display: grid; + gap: 0.25rem; +} + +.customer-master-evidence-context strong { + color: var(--color-text-heading); +} + +.customer-master-evidence-context span { + color: var(--color-text); + font-size: 0.8rem; + line-height: 1.45; +} + +.customer-master-evidence-close { + display: inline-grid; + flex: 0 0 auto; + place-items: center; + min-width: var(--size-control-min); + min-height: var(--size-control-min); + border: 1px solid var(--color-border); + border-radius: var(--radius-control); + background: var(--color-background); + color: var(--color-text-heading); + font-size: 1.1rem; + cursor: pointer; +} + +.customer-tree-evidence { + margin-top: 0; +} + +.customer-tree-evidence > ul { + display: grid; + gap: 0.65rem; + margin: 0; + padding: 0; + list-style: none; +} + +.customer-master-evidence-pane .related-post-card { + width: 100%; + text-align: left; +} + +@media (max-width: 1024px) { + .customer-master-relationship-workspace { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .customer-master-evidence-pane { + grid-column: 1 / -1; + } +} + +@media (max-width: 768px) { + .customer-master-relationship-workspace { + grid-template-columns: minmax(0, 1fr); + } + + .customer-master-evidence-pane { + grid-column: auto; + } + + .customer-master-tree-widget { + max-height: none; + } + + .customer-master-tree-group { + padding-inline-start: 0.65rem; + } +} diff --git a/frontend/src/components/CustomerMasterTree.stories.tsx b/frontend/src/components/CustomerMasterTree.stories.tsx new file mode 100644 index 000000000..6cb3553c8 --- /dev/null +++ b/frontend/src/components/CustomerMasterTree.stories.tsx @@ -0,0 +1,108 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import type { CustomerMasterEntity } from "../api"; +import { CustomerMasterTree } from "./CustomerMasterTree"; + +const hierarchy: CustomerMasterEntity[] = [ + { + corporate_entity_id: "group-demo", + corporate_entity_code: "DEMO-GROUP", + entity_name: "Demo Group", + entity_level_code: "group", + entity_level_label: "Group", + parent_entity_id: null, + }, + { + corporate_entity_id: "company-grid", + corporate_entity_code: "GRID-01", + entity_name: "Grid Systems", + entity_level_code: "company", + entity_level_label: "Company", + parent_entity_id: "group-demo", + }, + { + corporate_entity_id: "plant-east", + corporate_entity_code: "PLANT-EAST", + entity_name: "East Plant", + entity_level_code: "plant", + entity_level_label: "Plant", + parent_entity_id: "company-grid", + }, + { + corporate_entity_id: "company-energy", + corporate_entity_code: "ENERGY-01", + entity_name: "Energy Services", + entity_level_code: "company", + entity_level_label: "Company", + parent_entity_id: "group-demo", + }, +]; + +const meta = { + title: "Product/Customer Master/Three Pane Workspace", + component: CustomerMasterTree, + parameters: { + layout: "fullscreen", + }, + decorators: [ + (Story) => ( +
      + +
      + ), + ], + args: { + entities: hierarchy, + initialSelectedEntityId: "company-grid", + loadRelated: async (entityId: string) => [ + { + node_id: `post-${entityId}`, + node_type_code: "node_post", + relevance: 1, + label: "Open the latest customer evidence", + ontology_label: "Post", + post_body_excerpt: "A source-backed post attached to this customer entity.", + post_body_truncated: false, + }, + ], + onOpenPost: () => undefined, + }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const DesktopThreePane: Story = {}; + +export const PhoneStackedSteps: Story = { + parameters: { + viewport: { + defaultViewport: "mobile1", + }, + }, +}; + +export const MalformedRelationsRemainVisible: Story = { + args: { + initialSelectedEntityId: "orphan", + entities: [ + { + ...hierarchy[1], + corporate_entity_id: "cycle-a", + entity_name: "Cycle A", + parent_entity_id: "cycle-b", + }, + { + ...hierarchy[1], + corporate_entity_id: "cycle-b", + entity_name: "Cycle B", + parent_entity_id: "cycle-a", + }, + { + ...hierarchy[1], + corporate_entity_id: "orphan", + entity_name: "Missing visible parent", + parent_entity_id: "outside-scope", + }, + ], + }, +}; diff --git a/frontend/src/components/CustomerMasterTree.test.tsx b/frontend/src/components/CustomerMasterTree.test.tsx new file mode 100644 index 000000000..987c30435 --- /dev/null +++ b/frontend/src/components/CustomerMasterTree.test.tsx @@ -0,0 +1,225 @@ +import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import type { CustomerMasterEntity, RelatedNode } from "../api"; +import { CustomerMasterTree } from "./CustomerMasterTree"; + +function entity( + id: string, + name: string, + parentEntityId: string | null = null, + level: "Group" | "Company" | "Plant" = "Company", +): CustomerMasterEntity { + return { + corporate_entity_id: id, + corporate_entity_code: id.toUpperCase(), + entity_name: name, + entity_level_code: level.toLowerCase(), + entity_level_label: level, + parent_entity_id: parentEntityId, + }; +} + +const hierarchy = [ + entity("group", "Demo Group", null, "Group"), + entity("company", "Demo Company", "group", "Company"), + entity("plant", "Demo Plant", "company", "Plant"), +]; + +describe("CustomerMasterTree", () => { + it("exposes hierarchy metadata and the complete WAI-ARIA navigation contract", async () => { + render( + []} + onOpenPost={() => undefined} + />, + ); + + expect(screen.getByRole("tree")).toBeInTheDocument(); + const group = screen.getByRole("treeitem", { name: /Demo Group/ }); + const company = screen.getByRole("treeitem", { name: /Demo Company/ }); + const plant = screen.getByRole("treeitem", { name: /Demo Plant/ }); + expect(group).toHaveAttribute("aria-level", "1"); + expect(group).toHaveAttribute("aria-posinset", "1"); + expect(group).toHaveAttribute("aria-setsize", "1"); + expect(company).toHaveAttribute("aria-level", "2"); + expect(plant).toHaveAttribute("aria-level", "3"); + + group.focus(); + fireEvent.keyDown(group, { key: "ArrowDown" }); + expect(company).toHaveFocus(); + fireEvent.keyDown(company, { key: "ArrowDown" }); + expect(plant).toHaveFocus(); + fireEvent.keyDown(plant, { key: "ArrowUp" }); + expect(company).toHaveFocus(); + fireEvent.keyDown(company, { key: "Home" }); + expect(group).toHaveFocus(); + fireEvent.keyDown(group, { key: "End" }); + expect(plant).toHaveFocus(); + fireEvent.keyDown(plant, { key: "ArrowLeft" }); + expect(company).toHaveFocus(); + + fireEvent.keyDown(company, { key: "ArrowLeft" }); + await waitFor(() => + expect(screen.queryByRole("treeitem", { name: /Demo Plant/ })).not.toBeInTheDocument(), + ); + expect(company).toHaveAttribute("aria-expanded", "false"); + fireEvent.keyDown(company, { key: "ArrowRight" }); + expect(await screen.findByRole("treeitem", { name: /Demo Plant/ })).toBeInTheDocument(); + expect(company).toHaveAttribute("aria-expanded", "true"); + fireEvent.keyDown(company, { key: "ArrowRight" }); + expect(screen.getByRole("treeitem", { name: /Demo Plant/ })).toHaveFocus(); + }); + + it("uses branch disclosure independently from evidence selection", async () => { + render( + []} + onOpenPost={() => undefined} + />, + ); + + const group = screen.getByRole("treeitem", { name: /Demo Group/ }); + const branchToggle = group.querySelector("[data-customer-branch-toggle]"); + expect(branchToggle).not.toBeNull(); + await userEvent.click(branchToggle as HTMLElement); + expect(group).toHaveAttribute("aria-expanded", "false"); + expect(group).toHaveAttribute("aria-selected", "false"); + expect(screen.queryByRole("treeitem", { name: /Demo Company/ })).not.toBeInTheDocument(); + await userEvent.click(branchToggle as HTMLElement); + expect(await screen.findByRole("treeitem", { name: /Demo Company/ })).toBeInTheDocument(); + }); + + it("opens source-backed posts outside the tree with keyboard activation", async () => { + const onOpenPost = vi.fn(); + const related: RelatedNode[] = [ + { + node_id: "post-1", + node_type_code: "node_post", + relevance: 1, + label: "Customer escalation", + ontology_label: "Post", + post_body_excerpt: "Escalation evidence", + post_body_truncated: false, + }, + ]; + const loadRelated = vi.fn(async () => related); + render( + , + ); + + const tree = screen.getByRole("tree"); + const company = screen.getByRole("treeitem", { name: /Demo Company/ }); + company.focus(); + fireEvent.keyDown(company, { key: "Enter" }); + expect(loadRelated).toHaveBeenCalledWith("company"); + expect(company).toHaveAttribute("aria-selected", "true"); + const evidence = await screen.findByRole("region", { + name: "Related posts: Demo Company", + }); + expect(tree).not.toContainElement(evidence); + expect(company).toHaveAttribute("aria-controls", evidence.id); + expect( + within(evidence).getByRole("button", { + name: "Open related post: Customer escalation", + }), + ).toHaveTextContent("Escalation evidence"); + expect(screen.getByRole("treeitem", { name: /Demo Plant/ })).toBeInTheDocument(); + + fireEvent.keyDown(company, { key: " " }); + expect( + screen.queryByRole("region", { name: "Related posts: Demo Company" }), + ).not.toBeInTheDocument(); + fireEvent.keyDown(company, { key: "Enter" }); + const reopenedEvidence = await screen.findByRole("region", { + name: "Related posts: Demo Company", + }); + expect(loadRelated).toHaveBeenCalledTimes(1); + + await userEvent.click( + within(reopenedEvidence).getByRole("button", { + name: "Open related post: Customer escalation", + }), + ); + expect(onOpenPost).toHaveBeenCalledWith("post-1"); + }); + + it("does not let a stale entity request overwrite newly selected evidence", async () => { + let resolveGroup: ((value: RelatedNode[]) => void) | undefined; + const groupRequest = new Promise((resolve) => { + resolveGroup = resolve; + }); + const loadRelated = vi.fn((entityId: string) => + entityId === "group" ? groupRequest : Promise.resolve([]), + ); + render( + undefined} + />, + ); + + await userEvent.click(screen.getByRole("treeitem", { name: /Demo Group/ })); + await userEvent.click(screen.getByRole("treeitem", { name: /Demo Company/ })); + resolveGroup?.([ + { + node_id: "stale-post", + node_type_code: "node_post", + relevance: 1, + label: "Stale group post", + ontology_label: "Post", + }, + ]); + + await waitFor(() => + expect( + screen.getByRole("region", { name: "Related posts: Demo Company" }), + ).toBeInTheDocument(), + ); + expect(screen.queryByText("Stale group post")).not.toBeInTheDocument(); + }); + + it("fails a related-post request closed without hiding the customer", async () => { + render( + { + throw new Error("network unavailable"); + }} + onOpenPost={() => undefined} + />, + ); + + await userEvent.click(screen.getByRole("treeitem", { name: /Demo Plant/ })); + expect(await screen.findByText("No linked posts yet.")).toBeInTheDocument(); + expect(screen.getByRole("treeitem", { name: /Demo Plant/ })).toBeInTheDocument(); + }); + + it("keeps malformed hierarchy members visible and marks the relation unresolved", () => { + render( + []} + onOpenPost={() => undefined} + />, + ); + + expect(screen.getAllByRole("treeitem")).toHaveLength(4); + expect(screen.getByRole("treeitem", { name: /Cycle A.*unresolved/i })).toBeInTheDocument(); + expect(screen.getByRole("treeitem", { name: /Cycle B.*unresolved/i })).toBeInTheDocument(); + expect(screen.getByRole("treeitem", { name: /Self Parent.*unresolved/i })).toBeInTheDocument(); + expect(screen.getByRole("treeitem", { name: /Missing Parent.*unresolved/i })).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/components/CustomerMasterTree.tsx b/frontend/src/components/CustomerMasterTree.tsx new file mode 100644 index 000000000..9395b28b0 --- /dev/null +++ b/frontend/src/components/CustomerMasterTree.tsx @@ -0,0 +1,546 @@ +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + type FocusEvent, + type KeyboardEvent, + type MouseEvent, + type ReactNode, +} from "react"; +import type { CustomerMasterEntity, RelatedNode } from "../api"; +import { getCustomerMasterWorkspaceCopy } from "../customerMasterWorkspace"; +import { t, tf, useLocale } from "../i18n"; +import { + buildCustomerEntityForest, + flattenVisibleCustomerTree, + type CustomerEntityTreeNode, + type VisibleCustomerTreeItem, +} from "../customerMasterTree"; +import "./CustomerMasterTree.css"; + +const NODE_POST = "node_post"; + +/** Inputs for the authorized Customer Master relationship workspace. */ +export interface CustomerMasterTreeProps { + entities: readonly CustomerMasterEntity[]; + loadRelated: (entityId: string) => Promise; + onOpenPost: (postId: string) => void; + initialSelectedEntityId?: string | null; +} + +/** A source-backed related-post card shared by customer hierarchy and hint lists. */ +export function CustomerRelatedPostCard({ + postId, + postTitle, + postBodyExcerpt, + postBodyTruncated, + onOpenPost, +}: { + postId: string; + postTitle: string; + postBodyExcerpt?: string | null; + postBodyTruncated?: boolean; + onOpenPost: (postId: string) => void; +}) { + return ( + + ); +} + +function branchEntityIds(roots: readonly CustomerEntityTreeNode[]): Set { + const result = new Set(); + const stack = [...roots]; + while (stack.length > 0) { + const node = stack.pop()!; + if (node.children.length > 0) result.add(node.entity.corporate_entity_id); + stack.push(...node.children); + } + return result; +} + +function indexCustomerTree( + roots: readonly CustomerEntityTreeNode[], +): Map { + const index = new Map(); + const stack = [...roots]; + while (stack.length > 0) { + const node = stack.pop()!; + index.set(node.entity.corporate_entity_id, node); + stack.push(...node.children); + } + return index; +} + +/** + * Render the authorized Customer Master as a customer-centered three-pane workspace. + * + * The left pane preserves the cycle-safe WAI-ARIA hierarchy, the middle pane keeps one customer and + * its visible parent/child relations in focus, and the right pane owns only source-backed evidence. + */ +export function CustomerMasterTree({ + entities, + loadRelated, + onOpenPost, + initialSelectedEntityId = null, +}: CustomerMasterTreeProps) { + const locale = useLocale(); + const copy = getCustomerMasterWorkspaceCopy(locale); + const forest = useMemo(() => buildCustomerEntityForest(entities), [entities]); + const allBranchIds = useMemo(() => branchEntityIds(forest), [forest]); + const treeNodeById = useMemo(() => indexCustomerTree(forest), [forest]); + const entityById = useMemo( + () => new Map(entities.map((entity) => [entity.corporate_entity_id, entity])), + [entities], + ); + const resolvedInitialSelection = + initialSelectedEntityId && entityById.has(initialSelectedEntityId) + ? initialSelectedEntityId + : null; + const [expandedEntityIds, setExpandedEntityIds] = useState>( + () => new Set(allBranchIds), + ); + const [focusedEntityId, setFocusedEntityId] = useState( + () => forest[0]?.entity.corporate_entity_id ?? null, + ); + const [selectedEntityId, setSelectedEntityId] = useState( + resolvedInitialSelection, + ); + const [evidenceEntityId, setEvidenceEntityId] = useState(null); + const [relatedByEntity, setRelatedByEntity] = useState>({}); + const [relatedLoadingId, setRelatedLoadingId] = useState(null); + const treeItemRefs = useRef(new Map()); + const relatedRequestSerial = useRef(0); + + useEffect(() => { + setExpandedEntityIds(new Set(allBranchIds)); + setFocusedEntityId(forest[0]?.entity.corporate_entity_id ?? null); + setSelectedEntityId(resolvedInitialSelection); + setEvidenceEntityId(null); + setRelatedByEntity({}); + setRelatedLoadingId(null); + relatedRequestSerial.current += 1; + }, [allBranchIds, forest, resolvedInitialSelection]); + + useEffect( + () => () => { + relatedRequestSerial.current += 1; + }, + [], + ); + + const visibleItems = useMemo( + () => flattenVisibleCustomerTree(forest, expandedEntityIds), + [expandedEntityIds, forest], + ); + const visibleById = useMemo( + () => new Map(visibleItems.map((item) => [item.entityId, item])), + [visibleItems], + ); + + useEffect(() => { + if (focusedEntityId && visibleById.has(focusedEntityId)) return; + setFocusedEntityId(visibleItems[0]?.entityId ?? null); + }, [focusedEntityId, visibleById, visibleItems]); + + const focusTreeItem = useCallback((entityId: string) => { + setFocusedEntityId(entityId); + treeItemRefs.current.get(entityId)?.focus(); + }, []); + + const setBranchExpanded = useCallback((entityId: string, expanded: boolean) => { + setExpandedEntityIds((current) => { + const next = new Set(current); + if (expanded) next.add(entityId); + else next.delete(entityId); + return next; + }); + }, []); + + const activateEntity = useCallback( + async (entityId: string) => { + setSelectedEntityId(entityId); + setFocusedEntityId(entityId); + if (evidenceEntityId === entityId) { + relatedRequestSerial.current += 1; + setEvidenceEntityId(null); + setRelatedLoadingId(null); + return; + } + + const requestSerial = ++relatedRequestSerial.current; + setEvidenceEntityId(entityId); + if (relatedByEntity[entityId]) { + setRelatedLoadingId(null); + return; + } + + setRelatedLoadingId(entityId); + try { + const related = await loadRelated(entityId); + if (requestSerial !== relatedRequestSerial.current) return; + setRelatedByEntity((current) => ({ ...current, [entityId]: related })); + } catch { + if (requestSerial !== relatedRequestSerial.current) return; + setRelatedByEntity((current) => ({ ...current, [entityId]: [] })); + } finally { + if (requestSerial === relatedRequestSerial.current) setRelatedLoadingId(null); + } + }, + [evidenceEntityId, loadRelated, relatedByEntity], + ); + + function handleTreeKeyDown( + event: KeyboardEvent, + item: VisibleCustomerTreeItem, + ) { + event.stopPropagation(); + const visibleIndex = visibleItems.findIndex((candidate) => candidate.entityId === item.entityId); + switch (event.key) { + case "ArrowDown": { + event.preventDefault(); + const next = visibleItems[Math.min(visibleIndex + 1, visibleItems.length - 1)]; + if (next) focusTreeItem(next.entityId); + return; + } + case "ArrowUp": { + event.preventDefault(); + const previous = visibleItems[Math.max(visibleIndex - 1, 0)]; + if (previous) focusTreeItem(previous.entityId); + return; + } + case "Home": { + event.preventDefault(); + if (visibleItems[0]) focusTreeItem(visibleItems[0].entityId); + return; + } + case "End": { + event.preventDefault(); + const last = visibleItems[visibleItems.length - 1]; + if (last) focusTreeItem(last.entityId); + return; + } + case "ArrowRight": { + if (item.node.children.length === 0) return; + event.preventDefault(); + if (!expandedEntityIds.has(item.entityId)) { + setBranchExpanded(item.entityId, true); + } else { + focusTreeItem(item.node.children[0].entity.corporate_entity_id); + } + return; + } + case "ArrowLeft": { + event.preventDefault(); + if (item.node.children.length > 0 && expandedEntityIds.has(item.entityId)) { + setBranchExpanded(item.entityId, false); + } else if (item.parentEntityId) { + focusTreeItem(item.parentEntityId); + } + return; + } + case "Enter": + case " ": { + event.preventDefault(); + void activateEntity(item.entityId); + return; + } + default: + return; + } + } + + function renderNodes(nodes: readonly CustomerEntityTreeNode[]): ReactNode { + return nodes.map((node) => { + const entity = node.entity; + const entityId = entity.corporate_entity_id; + const item = visibleById.get(entityId); + if (!item) return null; + const isBranch = node.children.length > 0; + const isExpanded = expandedEntityIds.has(entityId); + const isSelected = selectedEntityId === entityId; + const isEvidenceOpen = evidenceEntityId === entityId; + const accessibleLabel = [ + entity.entity_name, + `${entity.corporate_entity_code} · ${entity.entity_level_label}`, + node.hierarchyIssue ? t("unresolved") : null, + ] + .filter(Boolean) + .join(", "); + return ( +
    • { + if (element) treeItemRefs.current.set(entityId, element); + else treeItemRefs.current.delete(entityId); + }} + role="treeitem" + className="customer-tree-node" + tabIndex={focusedEntityId === entityId ? 0 : -1} + aria-label={accessibleLabel} + aria-level={item.level} + aria-posinset={item.positionInSet} + aria-setsize={item.setSize} + aria-expanded={isBranch ? isExpanded : undefined} + aria-selected={isSelected} + aria-controls={isEvidenceOpen ? `customer-evidence-${entityId}` : undefined} + data-hierarchy-issue={node.hierarchyIssue ?? undefined} + onFocus={(event: FocusEvent) => { + if (event.currentTarget === event.target) setFocusedEntityId(entityId); + }} + onKeyDown={(event: KeyboardEvent) => handleTreeKeyDown(event, item)} + onClick={(event: MouseEvent) => { + event.stopPropagation(); + const target = event.target as HTMLElement; + if (target !== event.currentTarget) { + const row = target.closest("[data-customer-tree-row]"); + if (!row || row.parentElement !== event.currentTarget) return; + } + if (isBranch && target.closest("[data-customer-branch-toggle]")) { + setBranchExpanded(entityId, !isExpanded); + return; + } + void activateEntity(entityId); + }} + > +
      + + + {entity.entity_name} + + {entity.corporate_entity_code} · {entity.entity_level_label} + + {node.hierarchyIssue ? ( + {t("unresolved")} + ) : null} + +
      + {isBranch && isExpanded ? ( +
        + {renderNodes(node.children)} +
      + ) : null} +
      • + ); + }); + } + + const selectedEntity = selectedEntityId ? entityById.get(selectedEntityId) : undefined; + const selectedNode = selectedEntityId ? treeNodeById.get(selectedEntityId) : undefined; + const selectedParent = selectedNode && !selectedNode.hierarchyIssue && selectedEntity?.parent_entity_id + ? entityById.get(selectedEntity.parent_entity_id) + : undefined; + const selectedChildren = selectedNode?.children ?? []; + const evidenceEntity = evidenceEntityId ? entityById.get(evidenceEntityId) : undefined; + const relatedPosts = evidenceEntityId + ? (relatedByEntity[evidenceEntityId] ?? []).filter( + (related) => related.node_type_code === NODE_POST, + ) + : []; + + function renderRelationshipButton( + entity: CustomerMasterEntity, + relationshipLabel: string, + ): ReactNode { + return ( + + ); + } + + return ( +
      +
      +
      +

      {copy.hierarchyKicker}

      +

      {copy.hierarchyTitle}

      + {copy.hierarchyHelp} +
      +
        + {renderNodes(forest)} +
      +
      + +
      +
      +

      {copy.focusKicker}

      +

      {copy.focusTitle}

      + {copy.focusHelp} +
      + {selectedEntity && selectedNode ? ( +
      +
      + {copy.selectedCustomer} +

      {selectedEntity.entity_name}

      +

      + {selectedEntity.corporate_entity_code} · {selectedEntity.entity_level_label} +

      + + {selectedNode.hierarchyIssue ? t("unresolved") : copy.verifiedMaster} + +
      + + {selectedNode.hierarchyIssue ? ( +

      {copy.unresolvedRelation}

      + ) : null} + +
      +

      {copy.parentRelationship}

      + {selectedParent + ? renderRelationshipButton(selectedParent, copy.parentRelationship) + :

      {copy.noParent}

      } +
      + +
      +

      {copy.childRelationships}

      + {selectedChildren.length > 0 ? ( +
      + {selectedChildren.map((child) => ( +
      + {renderRelationshipButton(child.entity, copy.childRelationships)} +
      + ))} +
      + ) : ( +

      {copy.noChildren}

      + )} +
      + + +
      + ) : ( +

      {copy.selectCustomerPrompt}

      + )} +
      + + +
      + ); +} diff --git a/frontend/src/customerMasterTree.test.ts b/frontend/src/customerMasterTree.test.ts new file mode 100644 index 000000000..fd9159f1d --- /dev/null +++ b/frontend/src/customerMasterTree.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; +import type { CustomerMasterEntity } from "./api"; +import { buildCustomerEntityForest, flattenVisibleCustomerTree } from "./customerMasterTree"; + +function entity( + id: string, + parentEntityId: string | null = null, + level: "Group" | "Company" | "Plant" = "Company", +): CustomerMasterEntity { + return { + corporate_entity_id: id, + corporate_entity_code: id.toUpperCase(), + entity_name: id, + entity_level_code: level.toLowerCase(), + entity_level_label: level, + parent_entity_id: parentEntityId, + }; +} + +describe("buildCustomerEntityForest", () => { + it("builds a stable group-company-plant hierarchy", () => { + const forest = buildCustomerEntityForest([ + entity("group", null, "Group"), + entity("company", "group", "Company"), + entity("plant", "company", "Plant"), + ]); + + expect(forest).toHaveLength(1); + expect(forest[0].entity.corporate_entity_id).toBe("group"); + expect(forest[0].children[0].entity.corporate_entity_id).toBe("company"); + expect(forest[0].children[0].children[0].entity.corporate_entity_id).toBe("plant"); + expect(forest[0].hierarchyIssue).toBeNull(); + }); + + it("promotes a missing-parent and self-parent entity instead of hiding it", () => { + const forest = buildCustomerEntityForest([ + entity("self", "self"), + entity("orphan", "not-authorized"), + ]); + + expect(forest.map((node) => node.entity.corporate_entity_id)).toEqual(["self", "orphan"]); + expect(forest.map((node) => node.hierarchyIssue)).toEqual([ + "self_parent", + "missing_parent", + ]); + }); + + it("breaks every member of a cycle into a reviewable root and preserves descendants", () => { + const forest = buildCustomerEntityForest([ + entity("a", "b"), + entity("b", "a"), + entity("child", "a"), + ]); + + expect(forest.map((node) => node.entity.corporate_entity_id)).toEqual(["a", "b"]); + expect(forest.map((node) => node.hierarchyIssue)).toEqual(["cycle", "cycle"]); + expect(forest[0].children.map((node) => node.entity.corporate_entity_id)).toEqual(["child"]); + }); +}); + +describe("flattenVisibleCustomerTree", () => { + it("returns WAI-ARIA navigation order and hides collapsed descendants", () => { + const forest = buildCustomerEntityForest([ + entity("group", null, "Group"), + entity("company", "group", "Company"), + entity("plant", "company", "Plant"), + entity("other", null, "Group"), + ]); + + expect( + flattenVisibleCustomerTree(forest, new Set(["group", "company"])).map((item) => ({ + id: item.entityId, + level: item.level, + parent: item.parentEntityId, + position: item.positionInSet, + setSize: item.setSize, + })), + ).toEqual([ + { id: "group", level: 1, parent: null, position: 1, setSize: 2 }, + { id: "company", level: 2, parent: "group", position: 1, setSize: 1 }, + { id: "plant", level: 3, parent: "company", position: 1, setSize: 1 }, + { id: "other", level: 1, parent: null, position: 2, setSize: 2 }, + ]); + + expect(flattenVisibleCustomerTree(forest, new Set()).map((item) => item.entityId)).toEqual([ + "group", + "other", + ]); + }); +}); diff --git a/frontend/src/customerMasterTree.ts b/frontend/src/customerMasterTree.ts new file mode 100644 index 000000000..ae0071f2d --- /dev/null +++ b/frontend/src/customerMasterTree.ts @@ -0,0 +1,140 @@ +import type { CustomerMasterEntity } from "./api"; + +/** A malformed source relation that cannot safely participate in the visible hierarchy. */ +export type CustomerHierarchyIssue = "missing_parent" | "self_parent" | "cycle"; + +/** A cycle-safe customer master node projected from the authorized flat API response. */ +export interface CustomerEntityTreeNode { + entity: CustomerMasterEntity; + children: CustomerEntityTreeNode[]; + hierarchyIssue: CustomerHierarchyIssue | null; +} + +/** Metadata for one tree item in keyboard-navigation order. */ +export interface VisibleCustomerTreeItem { + node: CustomerEntityTreeNode; + entityId: string; + level: number; + parentEntityId: string | null; + positionInSet: number; + setSize: number; +} + +/** + * Build an ordered forest without dropping authorized entities when a parent is missing or cyclic. + * + * Valid parent links are preserved. A missing parent, self-parent, or every member of a detected + * cycle is promoted to a root and marked for buyer-visible review instead of disappearing. + */ +export function buildCustomerEntityForest( + entities: readonly CustomerMasterEntity[], +): CustomerEntityTreeNode[] { + const orderedEntities: CustomerMasterEntity[] = []; + const byId = new Map(); + for (const entity of entities) { + if (byId.has(entity.corporate_entity_id)) { + continue; + } + byId.set(entity.corporate_entity_id, entity); + orderedEntities.push(entity); + } + + const parentById = new Map(); + const issueById = new Map(); + for (const entity of orderedEntities) { + const entityId = entity.corporate_entity_id; + const parentId = entity.parent_entity_id; + if (!parentId) { + parentById.set(entityId, null); + } else if (parentId === entityId) { + parentById.set(entityId, null); + issueById.set(entityId, "self_parent"); + } else if (!byId.has(parentId)) { + parentById.set(entityId, null); + issueById.set(entityId, "missing_parent"); + } else { + parentById.set(entityId, parentId); + } + } + + const completed = new Set(); + for (const startId of byId.keys()) { + if (completed.has(startId)) continue; + const path: string[] = []; + const pathIndex = new Map(); + let currentId: string | null = startId; + while (currentId !== null && !completed.has(currentId)) { + const repeatedAt = pathIndex.get(currentId); + if (repeatedAt !== undefined) { + for (const cycleId of path.slice(repeatedAt)) { + parentById.set(cycleId, null); + issueById.set(cycleId, "cycle"); + } + break; + } + pathIndex.set(currentId, path.length); + path.push(currentId); + currentId = parentById.get(currentId) ?? null; + } + for (const entityId of path) completed.add(entityId); + } + + const nodeById = new Map(); + for (const entity of orderedEntities) { + nodeById.set(entity.corporate_entity_id, { + entity, + children: [], + hierarchyIssue: issueById.get(entity.corporate_entity_id) ?? null, + }); + } + + const roots: CustomerEntityTreeNode[] = []; + for (const entity of orderedEntities) { + const node = nodeById.get(entity.corporate_entity_id)!; + const parentId = parentById.get(entity.corporate_entity_id) ?? null; + if (parentId) { + nodeById.get(parentId)!.children.push(node); + } else { + roots.push(node); + } + } + return roots; +} + +/** Return visible nodes in WAI-ARIA tree keyboard order for the current expansion state. */ +export function flattenVisibleCustomerTree( + roots: readonly CustomerEntityTreeNode[], + expandedEntityIds: ReadonlySet, +): VisibleCustomerTreeItem[] { + const visible: VisibleCustomerTreeItem[] = []; + const stack: VisibleCustomerTreeItem[] = []; + for (let index = roots.length - 1; index >= 0; index -= 1) { + const node = roots[index]; + stack.push({ + node, + entityId: node.entity.corporate_entity_id, + level: 1, + parentEntityId: null, + positionInSet: index + 1, + setSize: roots.length, + }); + } + while (stack.length > 0) { + const item = stack.pop()!; + visible.push(item); + if (!expandedEntityIds.has(item.entityId)) continue; + const children = item.node.children; + for (let index = children.length - 1; index >= 0; index -= 1) { + const child = children[index]; + stack.push({ + node: child, + entityId: child.entity.corporate_entity_id, + level: item.level + 1, + parentEntityId: item.entityId, + positionInSet: index + 1, + setSize: children.length, + }); + } + } + return visible; +} diff --git a/frontend/src/customerMasterWorkspace.ts b/frontend/src/customerMasterWorkspace.ts new file mode 100644 index 000000000..d4dab70cb --- /dev/null +++ b/frontend/src/customerMasterWorkspace.ts @@ -0,0 +1,156 @@ +import type { Locale } from "./i18n"; + +/** Localized copy used by the three-pane Customer Master workspace. */ +export interface CustomerMasterWorkspaceCopy { + hierarchyKicker: string; + hierarchyTitle: string; + hierarchyHelp: string; + focusKicker: string; + focusTitle: string; + focusHelp: string; + evidenceKicker: string; + evidenceTitle: string; + evidenceHelp: string; + selectCustomerPrompt: string; + selectEvidencePrompt: string; + parentRelationship: string; + childRelationships: string; + noParent: string; + noChildren: string; + openEvidence: string; + closeEvidence: string; + verifiedMaster: string; + selectedCustomer: string; + unresolvedRelation: string; + relationshipAction: string; + evidenceNextAction: string; +} + +const COPY_BY_LOCALE: Record = { + en: { + hierarchyKicker: "01 · Customer hierarchy", + hierarchyTitle: "Choose a customer in scope", + hierarchyHelp: "Use the authorized hierarchy for navigation, then keep one customer centered.", + focusKicker: "02 · Selected customer", + focusTitle: "Customer relationship focus", + focusHelp: "Review the visible parent and direct children around the selected customer.", + evidenceKicker: "03 · Linked evidence", + evidenceTitle: "Source-backed customer evidence", + evidenceHelp: "Open a source post to continue into Event Lineage.", + selectCustomerPrompt: "Select a customer from the hierarchy to center its relationships.", + selectEvidencePrompt: "Select a customer to inspect its linked source evidence.", + parentRelationship: "Parent relationship", + childRelationships: "Direct child relationships", + noParent: "No parent organization is visible in the authorized scope.", + noChildren: "No direct child organization is visible in the authorized scope.", + openEvidence: "Open linked evidence", + closeEvidence: "Close linked evidence", + verifiedMaster: "Verified customer master", + selectedCustomer: "Selected customer", + unresolvedRelation: + "This hierarchy relation is unresolved. Review the source data before treating it as authoritative.", + relationshipAction: "Center this customer", + evidenceNextAction: "Open the source post to continue in Event Lineage.", + }, + ko: { + hierarchyKicker: "01 · 고객 계층", + hierarchyTitle: "권한 범위에서 고객사 선택", + hierarchyHelp: "권한이 확인된 계층에서 탐색한 뒤 한 고객사를 가운데에 고정합니다.", + focusKicker: "02 · 선택 고객사", + focusTitle: "고객사 관계 중심", + focusHelp: "선택 고객사를 기준으로 현재 보이는 상위 관계와 직접 하위 관계를 확인합니다.", + evidenceKicker: "03 · 연결 근거", + evidenceTitle: "원문으로 확인된 고객 근거", + evidenceHelp: "원문 글을 열어 이벤트 계보로 이어서 확인합니다.", + selectCustomerPrompt: "왼쪽 고객 계층에서 고객사를 선택해 관계의 중심으로 두세요.", + selectEvidencePrompt: "고객사를 선택하면 연결된 원문 근거를 확인할 수 있습니다.", + parentRelationship: "상위 관계", + childRelationships: "직접 하위 관계", + noParent: "권한 범위에서 확인되는 상위 조직이 없습니다.", + noChildren: "권한 범위에서 확인되는 직접 하위 조직이 없습니다.", + openEvidence: "연결 근거 열기", + closeEvidence: "연결 근거 닫기", + verifiedMaster: "확인된 고객 마스터", + selectedCustomer: "선택 고객사", + unresolvedRelation: "이 계층 관계는 미해결 상태입니다. 권위 있는 사실로 사용하기 전에 원천 데이터를 검토하세요.", + relationshipAction: "이 고객사를 가운데에 고정", + evidenceNextAction: "원문 글을 열어 이벤트 계보에서 이어서 확인하세요.", + }, + zh: { + hierarchyKicker: "01 · 客户层级", + hierarchyTitle: "在授权范围内选择客户", + hierarchyHelp: "从已授权层级中导航，并将一个客户固定在中央。", + focusKicker: "02 · 已选客户", + focusTitle: "以客户为中心的关系", + focusHelp: "围绕已选客户查看可见的上级关系和直接下级关系。", + evidenceKicker: "03 · 关联证据", + evidenceTitle: "有原文依据的客户证据", + evidenceHelp: "打开来源文章，继续查看事件谱系。", + selectCustomerPrompt: "从左侧客户层级中选择客户，将其作为关系中心。", + selectEvidencePrompt: "选择客户后可查看其关联的来源证据。", + parentRelationship: "上级关系", + childRelationships: "直接下级关系", + noParent: "授权范围内没有可见的上级组织。", + noChildren: "授权范围内没有可见的直接下级组织。", + openEvidence: "打开关联证据", + closeEvidence: "关闭关联证据", + verifiedMaster: "已验证客户主数据", + selectedCustomer: "已选客户", + unresolvedRelation: "此层级关系尚未解决。在作为权威事实使用前，请检查源数据。", + relationshipAction: "将此客户置于中央", + evidenceNextAction: "打开来源文章，继续查看事件谱系。", + }, + ja: { + hierarchyKicker: "01 · 顧客階層", + hierarchyTitle: "権限範囲から顧客を選択", + hierarchyHelp: "認可済みの階層をたどり、1社を中央に固定します。", + focusKicker: "02 · 選択中の顧客", + focusTitle: "顧客を中心とした関係", + focusHelp: "選択した顧客を基準に、表示可能な上位関係と直属の下位関係を確認します。", + evidenceKicker: "03 · 関連証拠", + evidenceTitle: "原文に裏付けられた顧客証拠", + evidenceHelp: "元投稿を開き、イベント系譜で続けて確認します。", + selectCustomerPrompt: "左の顧客階層から顧客を選び、関係の中心に置いてください。", + selectEvidencePrompt: "顧客を選択すると、関連する原文証拠を確認できます。", + parentRelationship: "上位関係", + childRelationships: "直属の下位関係", + noParent: "権限範囲内に表示できる上位組織はありません。", + noChildren: "権限範囲内に表示できる直属の下位組織はありません。", + openEvidence: "関連証拠を開く", + closeEvidence: "関連証拠を閉じる", + verifiedMaster: "確認済み顧客マスター", + selectedCustomer: "選択中の顧客", + unresolvedRelation: "この階層関係は未解決です。権威ある事実として扱う前に元データを確認してください。", + relationshipAction: "この顧客を中央に固定", + evidenceNextAction: "元投稿を開き、イベント系譜で続けて確認してください。", + }, + vi: { + hierarchyKicker: "01 · Cây phân cấp khách hàng", + hierarchyTitle: "Chọn khách hàng trong phạm vi được cấp quyền", + hierarchyHelp: "Đi theo cây phân cấp đã được cấp quyền rồi giữ một khách hàng ở vị trí trung tâm.", + focusKicker: "02 · Khách hàng đã chọn", + focusTitle: "Quan hệ lấy khách hàng làm trung tâm", + focusHelp: "Xem quan hệ cấp trên và các quan hệ cấp dưới trực tiếp quanh khách hàng đã chọn.", + evidenceKicker: "03 · Bằng chứng liên kết", + evidenceTitle: "Bằng chứng khách hàng có nguồn gốc", + evidenceHelp: "Mở bài viết nguồn để tiếp tục trong Dòng sự kiện.", + selectCustomerPrompt: "Chọn một khách hàng từ cây phân cấp bên trái để đặt làm trung tâm quan hệ.", + selectEvidencePrompt: "Chọn khách hàng để xem bằng chứng nguồn được liên kết.", + parentRelationship: "Quan hệ cấp trên", + childRelationships: "Quan hệ cấp dưới trực tiếp", + noParent: "Không có tổ chức cấp trên nào hiển thị trong phạm vi được cấp quyền.", + noChildren: "Không có tổ chức cấp dưới trực tiếp nào hiển thị trong phạm vi được cấp quyền.", + openEvidence: "Mở bằng chứng liên kết", + closeEvidence: "Đóng bằng chứng liên kết", + verifiedMaster: "Danh mục khách hàng đã xác minh", + selectedCustomer: "Khách hàng đã chọn", + unresolvedRelation: "Quan hệ phân cấp này chưa được giải quyết. Hãy kiểm tra dữ liệu nguồn trước khi coi là sự thật có thẩm quyền.", + relationshipAction: "Đặt khách hàng này ở trung tâm", + evidenceNextAction: "Mở bài viết nguồn để tiếp tục trong Dòng sự kiện.", + }, +}; + +/** Return the complete Customer Master workspace copy for a supported locale. */ +export function getCustomerMasterWorkspaceCopy(locale: Locale): CustomerMasterWorkspaceCopy { + return COPY_BY_LOCALE[locale]; +} diff --git a/frontend/src/i18n.test.ts b/frontend/src/i18n.test.ts index 621c9ef2a..58902f39d 100644 --- a/frontend/src/i18n.test.ts +++ b/frontend/src/i18n.test.ts @@ -48,6 +48,8 @@ describe("i18n", () => { "Filter by ISO week", "All weeks", "Authorized commitments are current. Open a commitment to read Event Lineage.", + "Authorized customer entities are current. Open a related post to read Event Lineage.", + "Authorized cited posts are current. Open a cited post to read Event Lineage.", ] as const; const eventLineageLabels = [ "Authorized scope", diff --git a/frontend/src/i18n.ts b/frontend/src/i18n.ts index 27c5ab329..51d844cec 100644 --- a/frontend/src/i18n.ts +++ b/frontend/src/i18n.ts @@ -126,6 +126,11 @@ const TRANSLATIONS: Partial>> = { "{week} Voice of Customer 글이 현재 표시되어 있습니다. 이벤트 계보를 읽으려면 글을 여세요.", "Authorized commitments are current. Open a commitment to read Event Lineage.": "권한이 있는 일정이 현재 표시되어 있습니다. 이벤트 계보를 읽으려면 일정을 여세요.", + "Authorized customer entities are current. Open a related post to read Event Lineage.": + "권한이 있는 고객 엔터티가 현재 표시되어 있습니다. 이벤트 계보를 읽으려면 관련 글을 여세요.", + "Authorized cited posts are current. Open a cited post to read Event Lineage.": + "권한이 있는 인용 글이 현재 표시되어 있습니다. 이벤트 계보를 읽으려면 인용 글을 여세요.", + "Open cited post:": "인용 글 열기:", "Filter by visibility": "공개 여부로 필터", "Sort posts": "글 정렬", "All VOC types": "모든 VOC 유형", @@ -493,6 +498,11 @@ const TRANSLATIONS: Partial>> = { "{week} 的 Voice of Customer 文章为当前内容。打开一篇文章阅读事件谱系。", "Authorized commitments are current. Open a commitment to read Event Lineage.": "已授权承诺为当前内容。打开一项承诺阅读事件谱系。", + "Authorized customer entities are current. Open a related post to read Event Lineage.": + "已授权客户实体为当前内容。打开一篇相关文章阅读事件谱系。", + "Authorized cited posts are current. Open a cited post to read Event Lineage.": + "已授权引用文章为当前内容。打开一篇引用文章阅读事件谱系。", + "Open cited post:": "打开引用文章：", "Filter by visibility": "按公开状态筛选", "Sort posts": "排序文章", "All VOC types": "所有 VOC 类型", @@ -883,6 +893,11 @@ const TRANSLATIONS: Partial>> = { "{week}のVoice of Customer投稿が現在表示されています。イベント系譜を読むには投稿を開いてください。", "Authorized commitments are current. Open a commitment to read Event Lineage.": "権限のある約束が現在表示されています。イベント系譜を読むには約束を開いてください。", + "Authorized customer entities are current. Open a related post to read Event Lineage.": + "権限のある顧客エンティティが現在表示されています。イベント系譜を読むには関連投稿を開いてください。", + "Authorized cited posts are current. Open a cited post to read Event Lineage.": + "権限のある引用投稿が現在表示されています。イベント系譜を読むには引用投稿を開いてください。", + "Open cited post:": "引用投稿を開く:", "Filter by visibility": "公開状態で絞り込み", "Sort posts": "投稿を並べ替え", "All VOC types": "すべての VOC 種類", @@ -1249,6 +1264,11 @@ const TRANSLATIONS: Partial>> = { "Các bài Voice of Customer của {week} đang hiện tại. Hãy mở một bài để đọc Dòng sự kiện.", "Authorized commitments are current. Open a commitment to read Event Lineage.": "Các cam kết được phép đang hiện tại. Hãy mở một cam kết để đọc Dòng sự kiện.", + "Authorized customer entities are current. Open a related post to read Event Lineage.": + "Các thực thể khách hàng được cấp quyền đang hiện tại. Hãy mở một bài liên quan để đọc Dòng sự kiện.", + "Authorized cited posts are current. Open a cited post to read Event Lineage.": + "Các bài được trích dẫn được cấp quyền đang hiện tại. Hãy mở một bài trích dẫn để đọc Dòng sự kiện.", + "Open cited post:": "Mở bài trích dẫn:", "Filter by visibility": "Lọc theo trạng thái hiển thị", "Sort posts": "Sắp xếp bài viết", "All VOC types": "Tất cả loại VOC", diff --git a/frontend/src/postBodyDisplay.test.ts b/frontend/src/postBodyDisplay.test.ts index ff51c8562..cfa010798 100644 --- a/frontend/src/postBodyDisplay.test.ts +++ b/frontend/src/postBodyDisplay.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { splitPostBody } from "./postBodyDisplay"; +import { splitMarkdownTableBody, splitPostBody } from "./postBodyDisplay"; /** 1x1 transparent PNG — the same synthetic fixture the Python vision tests use. */ const TINY_PNG_B64 = @@ -61,6 +61,100 @@ describe("splitPostBody", () => { ]); }); + it("recognizes numeric superscript-style footnotes", () => { + expect(splitPostBody("

      1 Source note

      ")).toEqual([ + { kind: "text", text: "1 Source note", role: "footnote" }, + ]); + }); + + it("preserves explicit metric superscripts and subscripts", () => { + expect(splitPostBody("

      Volume: 5m3, index m3.

      ")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃." }, + ]); + }); + + it("normalizes plain-text metric superscripts and subscripts", () => { + expect(splitPostBody("

      Volume: 5m^3, index m_3, braced m^{2}.

      ")).toEqual([ + { kind: "text", text: "Volume: 5m³, index m₃, braced m²." }, + ]); + }); + + it("preserves HTML and Word footnote blocks as footnote paragraphs", () => { + expect( + splitPostBody( + "

      Body evidence.

      " + + "HTML note." + + "Word note." + + "

      Next action.

      ", + ), + ).toEqual([ + { kind: "text", text: "Body evidence." }, + { kind: "text", text: "HTML note.", role: "footnote" }, + { kind: "text", text: "Word note.", role: "footnote" }, + { kind: "text", text: "Next action." }, + ]); + }); + + it("limits numeric superscript footnote roles to their source paragraph", () => { + expect( + splitPostBody( + "

      Evidence remains attached to the source.

      " + + "

      1 Source note.

      " + + "

      Continue with the next source action.

      ", + ), + ).toEqual([ + { kind: "text", text: "Evidence remains attached to the source." }, + { kind: "text", text: "1 Source note.", role: "footnote" }, + { kind: "text", text: "Continue with the next source action." }, + ]); + }); + + it("keeps exporter list containers in the visible nesting hierarchy", () => { + expect(splitPostBody("
    • Parent
      • Child
      • ")).toEqual([ + { kind: "text", text: "Parent", indentLevel: 1 }, + { kind: "text", text: "Child", indentLevel: 2 }, + ]); + }); + + it("resets indentation between separate top-level lists", () => { + expect(splitPostBody("
      • First
      • Second
      ")).toEqual([ + { kind: "text", text: "First", indentLevel: 1 }, + { kind: "text", text: "Second", indentLevel: 1 }, + ]); + }); + + it("keeps list items separate when optional closing tags are omitted", () => { + expect(splitPostBody("
      • First
      • Second
      ")).toEqual([ + { kind: "text", text: "First", indentLevel: 1 }, + { kind: "text", text: "Second", indentLevel: 1 }, + ]); + }); + + it("preserves prose around the supported Markdown table shape", () => { + expect( + splitMarkdownTableBody("Intro.\n\n| Project | Status |\n| --- | --- |\n| Alpha | Ready |\n\nNext action."), + ).toEqual([ + { kind: "prose", text: "Intro." }, + { kind: "table", rows: [["Project", "Status"], ["Alpha", "Ready"]] }, + { kind: "prose", text: "Next action." }, + ]); + }); + + it("normalizes metric scripts inside Markdown table cells", () => { + expect( + splitMarkdownTableBody("| Metric | Index |\n| --- | --- |\n| 5m^3 | m3 |"), + ).toEqual([{ kind: "table", rows: [["Metric", "Index"], ["5m³", "m₃"]] }]); + }); + + it("unescapes pipe characters inside Markdown cells without accepting a short delimiter", () => { + expect( + splitMarkdownTableBody( + "| Project | Notes |\n| :--- | ---: |\n| Alpha | Ready \\| review |", + ), + ).toEqual([{ kind: "table", rows: [["Project", "Notes"], ["Alpha", "Ready | review"]] }]); + expect(splitMarkdownTableBody("| Project | Status |\n| -- | -- |\n| Alpha | Ready |")).toBeNull(); + }); + it("leaves a plain-text post unchanged so existing popups keep their wording", () => { expect(splitPostBody("The full body text.")).toEqual([ { kind: "text", text: "The full body text." }, diff --git a/frontend/src/postBodyDisplay.ts b/frontend/src/postBodyDisplay.ts index 6fc12303c..3ea379213 100644 --- a/frontend/src/postBodyDisplay.ts +++ b/frontend/src/postBodyDisplay.ts @@ -11,32 +11,47 @@ export type PostBodySegment = | { kind: "text"; text: string; indentLevel?: number; role?: "footnote" } | { kind: "image"; src: string; mimeType: string; position: number }; +export type MarkdownBodyBlock = + | { kind: "prose"; text: string } + | { kind: "table"; rows: string[][] }; + const DATA_URI_IMG = /]*\bsrc\s*=\s*["']data:(image\/[a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+)["'][^>]*>/gi; const HTML_TAG = /<\/?[a-zA-Z][^>]*>/g; const BREAK_TAG = /]*>/gi; const BLOCK_TAG = - /<\/?(?:article|blockquote|div|h[1-6]|li|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi; + /<\/?(?:article|blockquote|div|endnote|footnote|h[1-6]|li|oi|ol|p|section|table|tbody|td|tfoot|th|thead|tr|ul|w:endnote|w:footnote|w:p|w:tbl|w:tr|w:tc)\b[^>]*>/gi; const WORD_INDENT_TAG = /]*\/?\s*>/gi; const LIST_ITEM_START = /^\s*(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)/; -const FOOTNOTE_START = /^\s*[*†‡](?=\S)/; +const FOOTNOTE_START = /^\s*[*†‡]+(?=\S)/; const INDENT_MARKER = "\u0001lw-indent:"; const INDENT_MARKER_END = "\u0002"; const INDENT_MARKER_PATTERN = /lw-indent:(\d+)/g; +const NUMERIC_FOOTNOTE_MARKER = "\u0003lw-numeric-footnote\u0004"; +const FOOTNOTE_BLOCK_MARKER = "\u0005lw-footnote-block\u0006"; +const FOOTNOTE_BLOCK_OPEN = /<\s*(?:footnote|endnote|w:footnote|w:endnote)\b[^>]*>/gi; +const NUMERIC_SUPERSCRIPT = /]*>\s*(\d{1,3})\s*<\/sup>/gi; const SUPERSCRIPT_DIGITS = "⁰¹²³⁴⁵⁶⁷⁸⁹"; const SUBSCRIPT_DIGITS = "₀₁₂₃₄₅₆₇₈₉"; const METRIC_MARKUP = /((?]*>\s*(\d{1,3})\s*<\/\2>/gi; +const METRIC_PLAIN_SCRIPT = + /((? { + return raw + .replace(METRIC_MARKUP, (_match, base: string, kind: string, digits: string) => { const table = kind.toLowerCase() === "sup" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS; return `${base}${[...digits].map((digit) => table[Number(digit)]).join("")}`; - }, - ); + }) + .replace( + METRIC_PLAIN_SCRIPT, + (_match, base: string, kind: string, bracedDigits: string, digits: string) => { + const table = kind === "^" ? SUPERSCRIPT_DIGITS : SUBSCRIPT_DIGITS; + return `${base}${[...(bracedDigits || digits)].map((digit) => table[Number(digit)]).join("")}`; + }, + ); } function stripIndentMarkers(value: string): string { @@ -79,7 +94,7 @@ function lengthToIndentUnits(value: string): number { function declaredIndentWidth(tag: string): number { const name = tag.match(/^<\/?\s*([a-z0-9:]+)/i)?.[1]?.toLowerCase() ?? ""; - let width = name === "blockquote" || name === "ul" || name === "ol" ? 4 : 0; + let width = name === "blockquote" || name === "ul" || name === "ol" || name === "oi" ? 4 : 0; const style = tag.match(/\bstyle\s*=\s*(["'])(.*?)\1/i)?.[2] ?? ""; for (const match of style.matchAll( /(?:^|;)\s*(?:margin-left|padding-left|padding-inline-start|text-indent)\s*:\s*([^;]+)/gi, @@ -104,17 +119,34 @@ function indentMarker(width: number): string { } function stripHtmlTags(text: string): string { - text = normalizeMetricMarkup(text).replace(/]*>(.*?)<\/sup>/gi, "^$1"); + text = text.replace(/]*>(.*?)<\/sup>/gi, "^$1"); + let listDepth = 0; const withBoundaries = text .replace(BREAK_TAG, "\n") .replace(BLOCK_TAG, (tag) => { - if (/^<\//.test(tag)) return "\n\n"; + const closing = /^<\//.test(tag); + const listContainer = /^<\s*\/?\s*(?:ul|ol|oi)\b/i.test(tag); + if (closing) { + if (listContainer) listDepth = Math.max(0, listDepth - 1); + return "\n\n"; + } + if (listContainer) { + listDepth += 1; + return "\n\n"; + } + if (/^<\s*\/?\s*(?:footnote|endnote|w:footnote|w:endnote)\b/i.test(tag)) { + return closing ? "\n\n" : `\n\n${FOOTNOTE_BLOCK_MARKER}`; + } + if (/^<\s*li\b/i.test(tag)) { + return `\n\n${indentMarker(Math.max(listDepth * 4, declaredIndentWidth(tag)))}`; + } return `\n\n${indentMarker(declaredIndentWidth(tag))}`; }) .replace(WORD_INDENT_TAG, (tag) => indentMarker(declaredIndentWidth(tag))); - const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => - /^<\/?w:/i.test(tag) ? "" : " ", - ); + const withoutTags = withBoundaries.replace(HTML_TAG, (tag) => { + if (/^<\/?w:/i.test(tag) || /^<\/?sup\b/i.test(tag)) return ""; + return " "; + }); const decoded = decodeHtmlEntities(withoutTags); return decoded .split("\n") @@ -212,10 +244,20 @@ function isDecodableBase64(raw: string): boolean { } function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): void { - const text = stripHtmlTags(raw); + const text = stripHtmlTags( + normalizeMetricMarkup(raw) + .replace(FOOTNOTE_BLOCK_OPEN, FOOTNOTE_BLOCK_MARKER) + .replace(NUMERIC_SUPERSCRIPT, `${NUMERIC_FOOTNOTE_MARKER}$1`), + ); + let pendingFootnoteBlock = false; for (const paragraph of splitSemanticParagraphs(text)) { + const hasNumericSuperscriptMarker = paragraph.includes(NUMERIC_FOOTNOTE_MARKER); + const hasFootnoteBlockMarker = paragraph.includes(FOOTNOTE_BLOCK_MARKER); + pendingFootnoteBlock ||= hasFootnoteBlockMarker; const indentLevel = indentationLevel(paragraph, indentUnit); - const normalized = stripIndentMarkers(paragraph) + const normalized = stripIndentMarkers( + paragraph.replaceAll(NUMERIC_FOOTNOTE_MARKER, "").replaceAll(FOOTNOTE_BLOCK_MARKER, ""), + ) .replace(/^[ \t]+/, "") .replace(/[ \t]+$/gm, ""); if (normalized.trim()) { @@ -223,10 +265,67 @@ function pushText(segments: PostBodySegment[], raw: string, indentUnit: number): kind: "text", text: normalized, ...(indentLevel > 0 ? { indentLevel } : {}), - ...(FOOTNOTE_START.test(normalized) ? { role: "footnote" as const } : {}), + ...(hasNumericSuperscriptMarker || pendingFootnoteBlock || FOOTNOTE_START.test(normalized) + ? { role: "footnote" as const } + : {}), }); + pendingFootnoteBlock = false; + } + } +} + +function markdownCells(line: string): string[] | null { + const value = line.trim().replace(/^\|/, "").replace(/(? cell.trim().replace(/\\\|/g, "|")); + return cells.length >= 2 && cells.every(Boolean) ? cells.map(normalizeMetricMarkup) : null; +} + +function isMarkdownSeparatorRow(cells: string[] | null): boolean { + return Boolean(cells?.every((cell) => /^:?-{3,}:?$/.test(cell))); +} + +/** + * Split the narrow Markdown-table shape supported by the ingestion boundary. + * The return value preserves prose and skips only the delimiter row. + */ +export function splitMarkdownTableBody(body: string): MarkdownBodyBlock[] | null { + const lines = body.replace(/\r\n?/g, "\n").split("\n"); + const blocks: MarkdownBodyBlock[] = []; + let prose: string[] = []; + let foundTable = false; + + const flushProse = () => { + const text = prose.join("\n").trim(); + if (text) blocks.push({ kind: "prose", text }); + prose = []; + }; + + let index = 0; + while (index < lines.length) { + const header = markdownCells(lines[index]); + const separator = markdownCells(lines[index + 1] ?? ""); + if (!header || !isMarkdownSeparatorRow(separator)) { + prose.push(lines[index]); + index += 1; + continue; } + + foundTable = true; + flushProse(); + const rows = [header]; + index += 2; + while (index < lines.length && lines[index].trim()) { + const row = markdownCells(lines[index]); + if (!row) break; + rows.push(row); + index += 1; + } + blocks.push({ kind: "table", rows }); } + + flushProse(); + return foundTable ? blocks : null; } export function splitPostBody(body: string): PostBodySegment[] { diff --git a/lineageweave/__init__.py b/lineageweave/__init__.py index 95330cb50..2ac589eb7 100644 --- a/lineageweave/__init__.py +++ b/lineageweave/__init__.py @@ -10,6 +10,25 @@ from .affiliate_tree import build_affiliate_forest from .corporate_hierarchy_resolution import resolve_corporate_entity from .entity_relationship_classification import OrganizationRelationship +from .external_lineage_analysis import analyze_external_lineage +from .external_lineage_contract import ( + CONTRACT_VERSION, + ChannelEvidence, + ExplicitParent, + LineageAnalysisPolicy, + LineageAnalysisRequest, + LineageAnalysisResult, + LineageContractError, + LineageEdgeResult, + LineageEvidenceRecord, + LineageLimitation, + ProjectProjection, + parse_lineage_analysis_request, + request_digest, + result_digest, + serialize_lineage_analysis_request, + serialize_lineage_analysis_result, +) from .knowledge_graph import random_walk_with_restart, select_related_nodes from .lineage_persistence import lineage_edge_specs from .models import Edge, Record, Tree @@ -30,8 +49,18 @@ from .voc_evidence import sentence_excerpts __all__ = [ + "CONTRACT_VERSION", + "ChannelEvidence", "ChatAnswer", "Edge", + "ExplicitParent", + "LineageAnalysisPolicy", + "LineageAnalysisRequest", + "LineageAnalysisResult", + "LineageContractError", + "LineageEdgeResult", + "LineageEvidenceRecord", + "LineageLimitation", "OrganizationRelationship", "PROV", "PROV_CLASSES", @@ -39,20 +68,27 @@ "PROV_RELATIONS", "PROV_RECOMMENDED_INVERSES", "PostSummary", + "ProjectProjection", "ProvAssertion", "ProvGraph", "ProvLiteral", "ProvValidationError", "Record", "Tree", + "analyze_external_lineage", "build_affiliate_forest", "cited_post_summaries", "lineage_edge_specs", + "parse_lineage_analysis_request", "random_walk_with_restart", "reconstruct", + "request_digest", "resolve_corporate_entity", + "result_digest", "select_related_nodes", "sentence_excerpts", + "serialize_lineage_analysis_request", + "serialize_lineage_analysis_result", ] -__version__ = "2.12.6" +__version__ = "2.16.0" diff --git a/lineageweave/adjudication_client.py b/lineageweave/adjudication_client.py index 37e4fee7f..566947833 100644 --- a/lineageweave/adjudication_client.py +++ b/lineageweave/adjudication_client.py @@ -28,6 +28,10 @@ def judge(self, candidate_label: str, record_label: str) -> float: raise NotImplementedError +class AdjudicationClientError(RuntimeError): + """The provider returned an unusable adjudication response.""" + + class NullAdjudicationClient: """No LLM orchestrator configured -- the llm channel is skipped.""" @@ -38,7 +42,20 @@ def judge(self, candidate_label: str, record_label: str) -> float: # pragma: no raise RuntimeError("NullAdjudicationClient has no llm channel; check .available first") -_CONFIDENCE_PATTERN = re.compile(r"([01](?:\.\d+)?)") +_CONFIDENCE_PATTERN = re.compile(r"(?:0(?:\.\d+)?|1(?:\.0+)?)") + + +def parse_confidence_response(content: object) -> float: + """Parse the provider's number-only confidence response strictly.""" + + if not isinstance(content, str): + raise AdjudicationClientError("provider confidence response was not text") + normalized = content.strip() + if _CONFIDENCE_PATTERN.fullmatch(normalized) is None: + raise AdjudicationClientError( + "provider confidence response was not a number in 0..1" + ) + return float(normalized) class ContextualOrchestratorAdjudicationClient: @@ -76,8 +93,10 @@ def judge(self, candidate_label: str, record_label: str) -> float: headers={"authorization": f"Bearer {self._api_key}"}, timeout=self._timeout, ) - content = chat_completion_content(body) - match = _CONFIDENCE_PATTERN.search(content) - if match is None: - return 0.0 - return max(0.0, min(1.0, float(match.group(1)))) + try: + content = chat_completion_content(body) + except (TypeError, ValueError) as exc: + raise AdjudicationClientError( + "provider response did not contain one chat message" + ) from exc + return parse_confidence_response(content) diff --git a/lineageweave/chunking.py b/lineageweave/chunking.py index 61b5f28cd..64b8aed53 100644 --- a/lineageweave/chunking.py +++ b/lineageweave/chunking.py @@ -65,9 +65,10 @@ "footer", "div", "p", - "ol", - "ul", "li", + "ul", + "ol", + "oi", "footnote", "endnote", "w:footnote", @@ -97,11 +98,12 @@ # readable and attributable as one unit. _TABLE_ROW_TAGS = frozenset({"tr", "w:tr"}) _TABLE_CELL_TAGS = frozenset({"td", "th", "w:tc"}) +_LIST_CONTAINER_TAGS = frozenset({"ul", "ol", "oi"}) _LIST_ITEM_START = re.compile( r"^(?:[-*•·]\s+|[*†‡](?=\S)|(?:\d{1,3}|[A-Za-z가-힣])[.)]\s+|[①-⑳]\s+)" ) -_FOOTNOTE_START = re.compile(r"^[*†‡](?=\S)") +_FOOTNOTE_START = re.compile(r"^[*†‡]+(?=\S)") def _is_footnote_block(tag: str, attrs: list[tuple[str, str | None]]) -> bool: @@ -132,6 +134,7 @@ def _is_footnote_reference(attrs: list[tuple[str, str | None]]) -> bool: def normalize_semantic_text(text: str) -> str: """Remove visual hanging-indent breaks without changing source content.""" + text = _normalize_metric_markup(_normalize_plain_metric_scripts(text)) lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") normalized: list[str] = [] for line in lines: @@ -199,7 +202,7 @@ def _shorthand_left_value(raw: str) -> str: def _declared_indent_width(tag: str, attrs: list[tuple[str, str | None]]) -> int: """Read HTML CSS and WordprocessingML paragraph indentation declarations.""" - width = 4 if tag in {"blockquote", "ul", "ol"} else 0 + width = 4 if tag in {"blockquote", "ul", "ol", "oi"} else 0 style = next((value or "" for name, value in attrs if name == "style"), "") for match in re.finditer( r"(?:^|;)\s*(?:margin-left|padding-left|padding-inline-start|text-indent)\s*:\s*([^;]+)", @@ -301,6 +304,21 @@ def chunk_by_paragraph(text: str) -> list[Chunk]: r"<(?Psup|sub)\b[^>]*>\s*(?P\d{1,3})\s*", re.IGNORECASE, ) +_METRIC_PLAIN_SCRIPT = re.compile( + r"(?P(?\^|_)\s*(?:\{(?P\d{1,3})\}|(?P\d{1,3}))", + re.IGNORECASE, +) + + +def _normalize_plain_metric_scripts(text: str) -> str: + """Normalize bounded plain-text metric exponents and indices.""" + def replace(match: re.Match[str]) -> str: + table = _SUPERSCRIPT_DIGITS if match.group("kind") == "^" else _SUBSCRIPT_DIGITS + digits = match.group("braced_digits") or match.group("digits") or "" + return f"{match.group('base')}{digits.translate(table)}" + + return _METRIC_PLAIN_SCRIPT.sub(replace, text) def _normalize_metric_markup(html: str) -> str: @@ -361,15 +379,27 @@ class _BlockTextExtractor(HTMLParser): """ def __init__(self) -> None: + """Initialize parser buffers for ordered text, image, and footnote units.""" super().__init__() self._stack: list[tuple[str, list[str], str | None, int, bool]] = [] self._unscoped_buffer: list[str] = [] + self._active_superscripts: list[tuple[int, list[str]]] = [] + self._numeric_superscript_buffers: set[int] = set() # Each entry is ("text", str, tag_name, style) or # ("image", (mime_type, bytes), "", None) -- a single sequence in # true document order, so an image's index among its siblings # reflects where it actually sat. self._finished: list[tuple[str, object, str, str | None, int, int]] = [] + def _declared_stack_width(self) -> int: + """Combine list depth with explicit width without double counting.""" + list_depth = sum(entry[0] in _LIST_CONTAINER_TAGS for entry in self._stack) + explicit_width = sum( + max(0, entry[3] - 4) if entry[0] in _LIST_CONTAINER_TAGS else entry[3] + for entry in self._stack + ) + return max(explicit_width, list_depth * 4) + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: """Collect relevant text state when an HTML start tag is encountered.""" if tag == "img": @@ -379,6 +409,10 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None if decoded is not None: self._finished.append(("image", decoded, "", None, 0, 0)) return + if tag == "sup": + if self._stack: + self._active_superscripts.append((id(self._stack[-1][1]), [])) + return if tag in {"br", "w:br"} and self._stack: self._stack[-1][1].append("\n") return @@ -400,15 +434,39 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None if self._stack and self._stack[-1][0] in _TABLE_ROW_TAGS and self._stack[-1][1]: self._stack[-1][1].append(" | ") return - # A rich-text editor commonly wraps a table cell in a nested

      or - #

      . Keep that content in the open row; otherwise the nested block - # closes first and destroys the row/column boundary. + # Nested blocks belong to the open table row. Keep list-item text + # readable without letting a nested list become a separate chunk. if any(entry[0] in _TABLE_ROW_TAGS for entry in self._stack): + if tag == "li" and self._stack[-1][1]: + self._stack[-1][1].append(" ") + return + if tag in _LIST_CONTAINER_TAGS: + # Emit a parent list item before entering its nested list. Closing + # tags otherwise make the child appear before the parent in the + # finished list, which destroys the source order buyers use to + # read a hierarchy. + if self._stack and self._stack[-1][0] == "li" and self._stack[-1][1]: + tag_name, buffer, style, indent_width, is_footnote = self._stack[-1] + self._stack[-1] = (tag_name, [], style, indent_width, is_footnote) + self._finish_block( + tag_name, + buffer, + style, + self._declared_stack_width(), + is_footnote, + ) + style = next((value for name, value in attrs if name == "style" and value), None) + is_footnote = _is_footnote_block(tag, attrs) or any( + entry[4] for entry in self._stack + ) + self._stack.append( + (tag, [], style, _declared_indent_width(tag, attrs), is_footnote) + ) return if tag in _DOM_BLOCK_TAGS: if self._stack and self._stack[-1][1]: tag_name, buffer, style, _, is_footnote = self._stack[-1] - declared_width = sum(entry[3] for entry in self._stack) + declared_width = self._declared_stack_width() self._finish_block(tag_name, buffer, style, declared_width, is_footnote) buffer.clear() style = next((value for name, value in attrs if name == "style" and value), None) @@ -427,8 +485,14 @@ def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> N def handle_endtag(self, tag: str) -> None: """Close the relevant text state when an HTML end tag is encountered.""" + if tag == "sup": + if self._active_superscripts: + buffer_id, content = self._active_superscripts.pop() + if re.fullmatch(r"\s*\d{1,3}\s*", "".join(content)): + self._numeric_superscript_buffers.add(buffer_id) + return if tag in _DOM_BLOCK_TAGS and self._stack and self._stack[-1][0] == tag: - declared_width = sum(entry[3] for entry in self._stack) + declared_width = self._declared_stack_width() tag_name, buffer, style, _, is_footnote = self._stack.pop() self._finish_block(tag_name, buffer, style, declared_width, is_footnote) @@ -442,11 +506,16 @@ def _finish_block( ) -> None: """Emit one block buffer, including a block closed only at EOF.""" raw_text = "".join(buffer) + superscript_marker = id(buffer) in self._numeric_superscript_buffers for raw_unit, source_indent in _split_dom_units(raw_text): text = normalize_semantic_text(raw_unit) if text: indent_width = declared_width + source_indent - label = "footnote" if is_footnote or _FOOTNOTE_START.match(text) else tag_name + label = ( + "footnote" + if is_footnote or superscript_marker or _FOOTNOTE_START.match(text) + else tag_name + ) self._finished.append( ( "text", @@ -457,6 +526,7 @@ def _finish_block( declared_width, ) ) + self._numeric_superscript_buffers.discard(id(buffer)) def handle_data(self, data: str) -> None: """Collect character data from the current HTML text region.""" @@ -466,6 +536,8 @@ def handle_data(self, data: str) -> None: if decoded == text: break text = decoded + if self._active_superscripts: + self._active_superscripts[-1][1].append(text) had_nbsp = "\xa0" in text text = text.replace("\xa0", " ") if self._stack and (text.strip() or had_nbsp): @@ -476,7 +548,7 @@ def handle_data(self, data: str) -> None: def finished(self) -> list[tuple[str, object, str, str | None, int, int]]: """Return the normalized records collected from the HTML fragment.""" while self._stack: - declared_width = sum(entry[3] for entry in self._stack) + declared_width = self._declared_stack_width() tag_name, buffer, style, _, is_footnote = self._stack.pop() self._finish_block(tag_name, buffer, style, declared_width, is_footnote) if not self._finished: @@ -494,10 +566,10 @@ def _split_dom_units(raw_text: str) -> list[tuple[str, int]]: current: list[str] = [] def flush() -> None: + """Move the current non-empty DOM unit into the result list.""" if current: raw_unit = "\n".join(current) - if raw_unit.strip(): - units.append((raw_unit, _source_indent_width(raw_unit))) + units.append((raw_unit, _source_indent_width(raw_unit))) current.clear() for line in raw_text.replace("\r\n", "\n").replace("\r", "\n").split("\n"): @@ -511,6 +583,62 @@ def flush() -> None: return units +_MARKDOWN_SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$") + + +def _markdown_cells(line: str) -> list[str] | None: + """Return Markdown table cells, or ``None`` for a non-table line.""" + if "|" not in line: + return None + value = line.strip() + value = value.removeprefix("|") + if value.endswith("|") and not value.endswith("\\|"): + value = value[:-1] + cells = [cell.strip().replace(r"\|", "|") for cell in re.split(r"(?= 2 and all(cells) else None + + +def _markdown_table_entries(text: str) -> list[tuple[str, str]]: + """Extract table rows while retaining non-table prose around the table.""" + lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + entries: list[tuple[str, str]] = [] + pending: list[str] = [] + found_table = False + + def flush_pending() -> None: + """Emit prose accumulated outside a recognized Markdown table.""" + if pending: + value = normalize_semantic_text("\n".join(pending)) + if value: + entries.append(("", value)) + pending.clear() + + index = 0 + while index < len(lines): + header = _markdown_cells(lines[index]) + separator = _markdown_cells(lines[index + 1]) if index + 1 < len(lines) else None + if header is None or separator is None or not all( + _MARKDOWN_SEPARATOR_CELL.fullmatch(cell) for cell in separator + ): + pending.append(lines[index]) + index += 1 + continue + + found_table = True + flush_pending() + entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in header))) + index += 2 + while index < len(lines) and lines[index].strip(): + cells = _markdown_cells(lines[index]) + if cells is None: + break + entries.append(("markdown_tr", " | ".join(normalize_semantic_text(cell) for cell in cells))) + index += 1 + + flush_pending() + return entries if found_table else [] + + _MARKDOWN_TABLE_SEPARATOR = re.compile( r"^\s*\|?\s*:?-{3,}:?\s*(?:\|\s*:?-{3,}:?\s*)+\|?\s*$" ) @@ -524,7 +652,10 @@ def _is_markdown_table_row(line: str) -> bool: def _render_markdown_table_row(line: str) -> str: """Keep Markdown table columns as searchable row evidence.""" - return " | ".join(cell.strip() for cell in line.strip().strip("|").split("|")) + return " | ".join( + normalize_semantic_text(cell.strip()) + for cell in line.strip().strip("|").split("|") + ) def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]: @@ -534,11 +665,11 @@ def _split_plain_text_units(text: str) -> list[tuple[str, int, str]]: current: list[str] = [] def flush() -> None: + """Emit the current authored plain-text semantic unit.""" if current: raw_unit = "\n".join(current) normalized = normalize_semantic_text(raw_unit) - if normalized: - units.append((normalized, _source_indent_width(raw_unit), "")) + units.append((normalized, _source_indent_width(raw_unit), "")) current.clear() index = 0 @@ -589,6 +720,19 @@ def chunk_by_dom(html: str) -> list[Chunk]: is what lets the image be placed back where it actually was relative to the surrounding text chunks. """ + if "<" not in html: + markdown_entries = _markdown_table_entries(html) + if markdown_entries: + return [ + Chunk( + text=text, + unit_type="plain_text", + index=index, + label=label, + ) + for index, (label, text) in enumerate(markdown_entries) + ] + parser = _BlockTextExtractor() parser.feed(_normalize_metric_markup(html)) entries = parser.finished() diff --git a/lineageweave/external_lineage.py b/lineageweave/external_lineage.py new file mode 100644 index 000000000..6a875ff17 --- /dev/null +++ b/lineageweave/external_lineage.py @@ -0,0 +1,41 @@ +"""Stable public package surface for external lineage consumers.""" + +from .external_lineage_analysis import analyze_external_lineage +from .external_lineage_contract import ( + CONTRACT_VERSION, + ChannelEvidence, + ExplicitParent, + LineageAnalysisPolicy, + LineageAnalysisRequest, + LineageAnalysisResult, + LineageContractError, + LineageEdgeResult, + LineageEvidenceRecord, + LineageLimitation, + ProjectProjection, + parse_lineage_analysis_request, + request_digest, + result_digest, + serialize_lineage_analysis_request, + serialize_lineage_analysis_result, +) + +__all__ = [ + "CONTRACT_VERSION", + "ChannelEvidence", + "ExplicitParent", + "LineageAnalysisPolicy", + "LineageAnalysisRequest", + "LineageAnalysisResult", + "LineageContractError", + "LineageEdgeResult", + "LineageEvidenceRecord", + "LineageLimitation", + "ProjectProjection", + "analyze_external_lineage", + "parse_lineage_analysis_request", + "request_digest", + "result_digest", + "serialize_lineage_analysis_request", + "serialize_lineage_analysis_result", +] diff --git a/lineageweave/external_lineage_analysis.py b/lineageweave/external_lineage_analysis.py new file mode 100644 index 000000000..984f6d2b5 --- /dev/null +++ b/lineageweave/external_lineage_analysis.py @@ -0,0 +1,484 @@ +"""Execute the external lineage contract through the core reconstruction kernel. + +This adapter is deliberately store-agnostic. It accepts an already parsed, +caller-authorized request, applies available-time cutoff rules, invokes the +existing deterministic/optional-LLM reconstruction kernel, and returns only +opaque caller references plus evidence-bounded result metadata. +""" + +from __future__ import annotations + +import math +from collections import defaultdict +from dataclasses import replace + +from .adjudication_client import ( + AdjudicationClient, + NullAdjudicationClient, +) +from .external_lineage_contract import ( + CONTRACT_VERSION, + ChannelEvidence, + LineageAnalysisRequest, + LineageAnalysisResult, + LineageContractError, + LineageEdgeResult, + LineageEvidenceRecord, + LineageLimitation, + ProjectProjection, + parse_lineage_analysis_request, + result_digest, + serialize_lineage_analysis_request, +) +from .models import Record +from .reconstruct import _best_parent, active_weights + + +def _contract_error(code: str, message: str, field: str | None = None) -> None: + """Raise a stable execution-time contract error.""" + + raise LineageContractError(code, message, field=field) + + +class _BoundedAdjudicationClient: + """Keep provider channel scores inside the fusion contract boundary.""" + + available = True + + def __init__(self, client: AdjudicationClient) -> None: + """Wrap one available client without changing its provider behavior.""" + + self._client = client + + def judge(self, candidate_label: str, record_label: str) -> float: + """Return one finite unit-interval score or fail with a stable code.""" + + try: + score = self._client.judge(candidate_label, record_label) + except Exception as exc: + raise LineageContractError( + "llm_channel_error", + "LLM channel returned an unusable provider response", + field="llm", + ) from exc + if isinstance(score, bool) or not isinstance(score, (int, float)): + _contract_error( + "channel_score_out_of_bounds", + "LLM channel score must be finite and within 0..1", + "llm", + ) + number = float(score) + if not math.isfinite(number) or not 0.0 <= number <= 1.0: + _contract_error( + "channel_score_out_of_bounds", + "LLM channel score must be finite and within 0..1", + "llm", + ) + return number + + +def _validated_request(request: LineageAnalysisRequest) -> LineageAnalysisRequest: + """Round-trip a dataclass through the public parser before execution.""" + + return parse_lineage_analysis_request( + serialize_lineage_analysis_request(request) + ) + + +def _validate_explicit_parent_relations( + records: tuple[LineageEvidenceRecord, ...], +) -> None: + """Validate caller-observed parent relations before cutoff filtering.""" + + by_ref = {record.evidence_ref: record for record in records} + for child in records: + explicit = child.explicit_parent + if explicit is None: + continue + if explicit.evidence_ref == child.evidence_ref: + _contract_error( + "explicit_parent_self_reference", + "an evidence record cannot be its own parent", + child.evidence_ref, + ) + parent = by_ref.get(explicit.evidence_ref) + if parent is None: + _contract_error( + "explicit_parent_missing", + "explicit parent is absent from the request", + child.evidence_ref, + ) + if parent.group_ref != child.group_ref: + _contract_error( + "explicit_parent_group_mismatch", + "explicit parent and child must share one group", + child.evidence_ref, + ) + if parent.occurred_at > child.occurred_at: + _contract_error( + "explicit_parent_after_child", + "explicit parent occurs after the child", + child.evidence_ref, + ) + + parent_by_child = { + child.evidence_ref: child.explicit_parent.evidence_ref + for child in records + if child.explicit_parent is not None + } + for start_ref in parent_by_child: + current_ref = start_ref + visited: set[str] = set() + while current_ref in parent_by_child: + if current_ref in visited: + _contract_error( + "explicit_parent_cycle", + "explicit parent relations must form an acyclic graph", + start_ref, + ) + visited.add(current_ref) + current_ref = parent_by_child[current_ref] + + +def _selected_llm( + request: LineageAnalysisRequest, + llm: AdjudicationClient | None, +) -> tuple[AdjudicationClient, str]: + """Apply the explicit LLM admission policy and return its result status.""" + + if not request.policy.allow_llm: + return NullAdjudicationClient(), "not_requested" + if llm is None or not getattr(llm, "available", False): + return NullAdjudicationClient(), "unavailable" + return _BoundedAdjudicationClient(llm), "completed" + + +def _included_records( + request: LineageAnalysisRequest, +) -> tuple[ + tuple[LineageEvidenceRecord, ...], + tuple[LineageEvidenceRecord, ...], +]: + """Partition evidence by available time, not occurrence time.""" + + if request.knowledge_cutoff is None: + return request.records, () + included = tuple( + record + for record in request.records + if record.available_at <= request.knowledge_cutoff + ) + excluded = tuple( + record + for record in request.records + if record.available_at > request.knowledge_cutoff + ) + return included, excluded + + +def _ordered_contract_groups( + records: tuple[LineageEvidenceRecord, ...], +) -> tuple[tuple[LineageEvidenceRecord, ...], ...]: + """Return deterministic groups ordered by time and opaque reference.""" + + grouped: dict[str, list[LineageEvidenceRecord]] = defaultdict(list) + for record in records: + grouped[record.group_ref].append(record) + return tuple( + tuple( + sorted( + grouped[group_ref], + key=lambda item: (item.occurred_at, item.evidence_ref), + ) + ) + for group_ref in sorted(grouped) + ) + + +def _pair_evaluation_count( + records: tuple[LineageEvidenceRecord, ...], + candidate_window: int, +) -> int: + """Count only candidate pairs that require inferred parent selection.""" + + return sum( + min(index, candidate_window) + for group_records in _ordered_contract_groups(records) + for index, record in enumerate(group_records) + if record.explicit_parent is None + ) + + +def _enforce_pair_budget( + records: tuple[LineageEvidenceRecord, ...], + request: LineageAnalysisRequest, +) -> int: + """Reject excess pair work before optional LLM/provider activity.""" + + pair_count = _pair_evaluation_count( + records, + request.policy.candidate_window, + ) + if pair_count > request.policy.maximum_pair_evaluations: + _contract_error( + "pair_evaluation_budget_exceeded", + "candidate-pair work exceeds the declared maximum", + "policy.maximum_pair_evaluations", + ) + return pair_count + + +def _core_record(record: LineageEvidenceRecord) -> Record: + """Convert one contract record to the core reconstruction shape.""" + + return Record( + record_id=record.evidence_ref, + group_key=record.group_ref, + label=record.label, + occurred_at=record.occurred_at, + secondary_key=record.secondary_key or "", + ) + + +def _channel_evidence( + channel_scores: dict[str, float], + weights: dict[str, float], +) -> tuple[ChannelEvidence, ...]: + """Project finite active scores with their normalized contributions.""" + + projected: list[ChannelEvidence] = [] + for channel_code in sorted(channel_scores): + score = float(channel_scores[channel_code]) + weight = float(weights[channel_code]) + contribution = score * weight + values = (score, weight, contribution) + if not all( + math.isfinite(value) and 0.0 <= value <= 1.0 + for value in values + ): + _contract_error( + "channel_score_out_of_bounds", + "channel values must be finite within 0..1", + channel_code, + ) + projected.append( + ChannelEvidence( + channel_code, + score, + weight, + contribution, + ) + ) + return tuple(projected) + + +def _inferred_edges( + records: tuple[LineageEvidenceRecord, ...], + llm: AdjudicationClient, + request: LineageAnalysisRequest, +) -> list[LineageEdgeResult]: + """Select inferred parents without rescoring explicit observed children.""" + + if not records: + return [] + weights = active_weights(llm) + edges: list[LineageEdgeResult] = [] + for group_records in _ordered_contract_groups(records): + core_records = [_core_record(record) for record in group_records] + for index, source_record in enumerate(group_records): + if source_record.explicit_parent is not None: + continue + candidates = core_records[ + max(0, index - request.policy.candidate_window) : index + ] + parent_choice = _best_parent( + core_records[index], + candidates, + llm, + weights, + request.policy.minimum_fused_score, + ) + if parent_choice is None: + continue + parent, fused_score, channel_scores = parent_choice + edges.append( + LineageEdgeResult( + parent_evidence_ref=parent.record_id, + child_evidence_ref=source_record.evidence_ref, + relation_type_code="reconstructed_continuation", + truth_status_code="inferred", + fused_score=float(fused_score), + channel_evidence=_channel_evidence( + channel_scores, + weights, + ), + ) + ) + return edges + + +def _explicit_edges( + included: tuple[LineageEvidenceRecord, ...], +) -> tuple[ + list[LineageEdgeResult], + set[str], + list[LineageLimitation], +]: + """Project included caller-observed parent relations ahead of inference.""" + + included_refs = {record.evidence_ref for record in included} + edges: list[LineageEdgeResult] = [] + explicit_children: set[str] = set() + limitations: list[LineageLimitation] = [] + for child in included: + explicit = child.explicit_parent + if explicit is None: + continue + explicit_children.add(child.evidence_ref) + if explicit.evidence_ref not in included_refs: + limitations.append( + LineageLimitation( + "explicit_parent_after_cutoff", + child.evidence_ref, + ( + "The caller-observed parent was unavailable at " + "the requested cutoff." + ), + ) + ) + continue + edges.append( + LineageEdgeResult( + parent_evidence_ref=explicit.evidence_ref, + child_evidence_ref=child.evidence_ref, + relation_type_code=explicit.relation_code, + truth_status_code="observed", + fused_score=1.0, + channel_evidence=( + ChannelEvidence( + explicit.relation_code, + 1.0, + 1.0, + 1.0, + ), + ), + ) + ) + return edges, explicit_children, limitations + + +def _project_groups( + records: tuple[LineageEvidenceRecord, ...], +) -> tuple[ProjectProjection, ...]: + """Group included project evidence without crossing caller groups.""" + + grouped: dict[tuple[str, str], list[str]] = defaultdict(list) + for record in records: + if record.project_ref is not None: + grouped[(record.group_ref, record.project_ref)].append( + record.evidence_ref + ) + return tuple( + ProjectProjection( + group_ref, + project_ref, + tuple(sorted(evidence_refs)), + "proposed", + ) + for (group_ref, project_ref), evidence_refs in sorted( + grouped.items() + ) + ) + + +def analyze_external_lineage( + request: LineageAnalysisRequest, + *, + llm: AdjudicationClient | None = None, +) -> LineageAnalysisResult: + """Analyze bounded caller evidence and return a deterministic result. + + The function performs no persistence or network access itself. An optional + client is used only when ``request.policy.allow_llm`` is true and the + supplied client explicitly reports availability. + """ + + validated = _validated_request(request) + _validate_explicit_parent_relations(validated.records) + included, excluded = _included_records(validated) + pair_evaluations = _enforce_pair_budget(included, validated) + selected_llm, llm_status = _selected_llm(validated, llm) + if llm_status == "completed" and pair_evaluations == 0: + llm_status = "not_invoked" + + inferred = _inferred_edges( + included, + selected_llm, + validated, + ) + explicit, explicit_children, explicit_limitations = _explicit_edges( + included + ) + edges = [ + edge + for edge in inferred + if edge.child_evidence_ref not in explicit_children + ] + edges.extend(explicit) + + limitations = [ + LineageLimitation( + "evidence_after_cutoff_excluded", + record.evidence_ref, + ( + "Evidence was first available after the requested " + "knowledge cutoff." + ), + ) + for record in excluded + ] + limitations.extend(explicit_limitations) + + edge_order = { + record.evidence_ref: (record.group_ref, record.occurred_at, record.evidence_ref) + for record in included + } + result = LineageAnalysisResult( + contract_version=CONTRACT_VERSION, + analysis_id=validated.analysis_id, + analysis_scope_code=validated.analysis_scope_code, + knowledge_cutoff=validated.knowledge_cutoff, + included_evidence_refs=tuple( + sorted(record.evidence_ref for record in included) + ), + excluded_evidence_refs=tuple( + sorted(record.evidence_ref for record in excluded) + ), + llm_status_code=llm_status, # type: ignore[arg-type] + edges=tuple( + sorted( + edges, + key=lambda item: ( + edge_order[item.child_evidence_ref], + item.parent_evidence_ref, + item.relation_type_code, + ), + ) + ), + project_projections=_project_groups(included), + limitations=tuple( + sorted( + limitations, + key=lambda item: ( + item.limitation_code, + item.evidence_ref or "", + item.message, + ), + ) + ), + result_digest="", + ) + return replace( + result, + result_digest=result_digest(result), + ) diff --git a/lineageweave/external_lineage_contract.py b/lineageweave/external_lineage_contract.py new file mode 100644 index 000000000..a08670ba8 --- /dev/null +++ b/lineageweave/external_lineage_contract.py @@ -0,0 +1,949 @@ +"""Versioned store-agnostic contract for external lineage consumers. + +The contract accepts only bounded caller-authorized evidence references. It +contains no provider credential, database, mailbox, or network behavior. A +consumer such as Naruon can therefore submit a minimized evidence projection +without granting LineageWeave authority over the consumer's source records. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from dataclasses import dataclass, replace +from datetime import datetime, timezone +from typing import Final, Literal, cast + +CONTRACT_VERSION: Final = "1.0.0" +MAX_RECORD_COUNT: Final = 500 +MAX_REFERENCE_LENGTH: Final = 160 +MAX_LABEL_LENGTH: Final = 2_000 +MAX_CANDIDATE_WINDOW: Final = 200 +MAX_PAIR_EVALUATIONS: Final = 5_000 + +AnalysisScopeCode = Literal["email_lineage", "project_history", "generic_lineage"] +SourceKindCode = Literal["email", "task", "commitment", "project_event", "generic"] +CallerTruthStatusCode = Literal["observed", "authoritative_in_caller"] +ExplicitRelationCode = Literal["rfc_reply", "provider_reply", "manual_parent"] +ResultTruthStatusCode = Literal["observed", "inferred", "proposed"] +LlmStatusCode = Literal["not_requested", "unavailable", "not_invoked", "completed"] + +_ANALYSIS_SCOPES = frozenset({"email_lineage", "project_history", "generic_lineage"}) +_SOURCE_KINDS = frozenset({"email", "task", "commitment", "project_event", "generic"}) +_CALLER_TRUTH_STATUSES = frozenset({"observed", "authoritative_in_caller"}) +_EXPLICIT_RELATIONS = frozenset({"rfc_reply", "provider_reply", "manual_parent"}) +_EDGE_TRUTH_STATUSES = frozenset({"observed", "inferred"}) +_LLM_STATUSES = frozenset({"not_requested", "unavailable", "not_invoked", "completed"}) +_OPAQUE_REFERENCE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@+\-]*$") +_RESULT_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") +_SCORE_TOLERANCE: Final = 1e-9 + + +class LineageContractError(ValueError): + """A fail-closed request or result contract violation. + + Attributes: + code: Stable machine-readable reason code. + field: Optional dotted field path associated with the violation. + """ + + def __init__(self, code: str, message: str, *, field: str | None = None) -> None: + """Initialize one stable contract error without embedding source evidence.""" + + self.code = code + self.field = field + suffix = f" ({field})" if field else "" + super().__init__(f"{message}{suffix}") + + +@dataclass(frozen=True) +class ExplicitParent: + """One caller-observed immediate parent relation.""" + + evidence_ref: str + relation_code: ExplicitRelationCode + + +@dataclass(frozen=True) +class LineageEvidenceRecord: + """One bounded caller-owned evidence record admitted for analysis.""" + + evidence_ref: str + group_ref: str + source_kind_code: SourceKindCode + truth_status_code: CallerTruthStatusCode + label: str + occurred_at: datetime + available_at: datetime + secondary_key: str | None = None + project_ref: str | None = None + explicit_parent: ExplicitParent | None = None + + +@dataclass(frozen=True) +class LineageAnalysisPolicy: + """Bounded reconstruction policy selected by the caller.""" + + candidate_window: int + maximum_pair_evaluations: int + minimum_fused_score: float + allow_llm: bool + + +@dataclass(frozen=True) +class LineageAnalysisRequest: + """Strict versioned request for external lineage reconstruction.""" + + contract_version: str + analysis_id: str + analysis_scope_code: AnalysisScopeCode + knowledge_cutoff: datetime | None + policy: LineageAnalysisPolicy + records: tuple[LineageEvidenceRecord, ...] + + +@dataclass(frozen=True) +class ChannelEvidence: + """One active reconstruction channel's exact normalized contribution.""" + + channel_code: str + score: float + weight: float + contribution: float + + +@dataclass(frozen=True) +class LineageEdgeResult: + """One observed or inferred edge between caller-owned evidence records.""" + + parent_evidence_ref: str + child_evidence_ref: str + relation_type_code: str + truth_status_code: Literal["observed", "inferred"] + fused_score: float + channel_evidence: tuple[ChannelEvidence, ...] + + +@dataclass(frozen=True) +class ProjectProjection: + """A proposed project grouping bounded to one caller group.""" + + group_ref: str + project_ref: str + evidence_refs: tuple[str, ...] + truth_status_code: Literal["proposed"] = "proposed" + + +@dataclass(frozen=True) +class LineageLimitation: + """A machine-readable limitation disclosed with an analysis result.""" + + limitation_code: str + evidence_ref: str | None + message: str + + +@dataclass(frozen=True) +class LineageAnalysisResult: + """Deterministic external lineage result containing no caller credential.""" + + contract_version: str + analysis_id: str + analysis_scope_code: AnalysisScopeCode + knowledge_cutoff: datetime | None + included_evidence_refs: tuple[str, ...] + excluded_evidence_refs: tuple[str, ...] + llm_status_code: LlmStatusCode + edges: tuple[LineageEdgeResult, ...] + project_projections: tuple[ProjectProjection, ...] + limitations: tuple[LineageLimitation, ...] + result_digest: str + + +def _raise(code: str, message: str, field: str | None = None) -> None: + """Raise one stable contract error.""" + + raise LineageContractError(code, message, field=field) + + +def _object( + value: object, + *, + field: str, + allowed: frozenset[str], + required: frozenset[str], +) -> dict[str, object]: + """Validate a strict object and reject unknown or missing fields.""" + + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + _raise("invalid_field_type", "expected an object", field) + typed = cast(dict[str, object], value) + unknown = sorted(set(typed) - allowed) + if unknown: + _raise("unknown_field", f"unknown field {unknown[0]!r}", f"{field}.{unknown[0]}") + missing = sorted(required - set(typed)) + if missing: + _raise("missing_field", f"missing required field {missing[0]!r}", f"{field}.{missing[0]}") + return typed + + +def _string(value: object, *, field: str, minimum: int = 1, maximum: int) -> str: + """Return one trimmed bounded string or fail closed.""" + + if not isinstance(value, str): + _raise("invalid_field_type", "expected a string", field) + normalized = value.strip() + if not minimum <= len(normalized) <= maximum: + _raise( + "text_length_out_of_bounds", + f"length must be {minimum}..{maximum}", + field, + ) + return normalized + + +def _opaque_reference( + value: object, + *, + field: str, + optional: bool = False, +) -> str | None: + """Validate one bounded opaque identifier that cannot be a URL.""" + + if value is None and optional: + return None + normalized = _string(value, field=field, maximum=MAX_REFERENCE_LENGTH) + if "://" in normalized or not _OPAQUE_REFERENCE.fullmatch(normalized): + _raise( + "unsafe_opaque_reference", + "reference must be opaque and whitespace-free", + field, + ) + return normalized + + +def _timestamp(value: object, *, field: str, optional: bool = False) -> datetime | None: + """Parse an offset-aware RFC 3339 timestamp and normalize it to UTC.""" + + if value is None and optional: + return None + if not isinstance(value, str): + _raise("invalid_field_type", "expected an RFC 3339 string", field) + candidate = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + parsed = datetime.fromisoformat(candidate) + except ValueError as exc: + raise LineageContractError( + "invalid_timestamp", + "invalid RFC 3339 timestamp", + field=field, + ) from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + _raise( + "timestamp_must_be_offset_aware", + "timestamp must carry an offset", + field, + ) + return parsed.astimezone(timezone.utc) + + +def _enum( + value: object, + *, + field: str, + allowed: frozenset[str], + code: str, +) -> str: + """Validate one controlled vocabulary value.""" + + if not isinstance(value, str): + _raise("invalid_field_type", "expected a controlled string", field) + if value not in allowed: + _raise(code, f"unsupported value {value!r}", field) + return value + + +def _integer(value: object, *, field: str, minimum: int, maximum: int) -> int: + """Validate one integer policy value within an inclusive range.""" + + if isinstance(value, bool) or not isinstance(value, int): + _raise("invalid_field_type", "expected an integer", field) + if not minimum <= value <= maximum: + _raise( + "policy_value_out_of_bounds", + f"value must be {minimum}..{maximum}", + field, + ) + return value + + +def _number(value: object, *, field: str, minimum: float, maximum: float) -> float: + """Validate one finite numeric policy value within an inclusive range.""" + + if isinstance(value, bool) or not isinstance(value, (int, float)): + _raise("invalid_field_type", "expected a finite number", field) + number = float(value) + if not math.isfinite(number) or not minimum <= number <= maximum: + _raise( + "policy_value_out_of_bounds", + f"value must be {minimum}..{maximum}", + field, + ) + return number + + +def _boolean(value: object, *, field: str) -> bool: + """Validate a real boolean without accepting integer substitutes.""" + + if not isinstance(value, bool): + _raise("invalid_field_type", "expected a boolean", field) + return value + + +def _parse_explicit_parent(value: object, *, field: str) -> ExplicitParent | None: + """Parse one optional caller-observed parent relation.""" + + if value is None: + return None + payload = _object( + value, + field=field, + allowed=frozenset({"evidence_ref", "relation_code"}), + required=frozenset({"evidence_ref", "relation_code"}), + ) + reference = _opaque_reference(payload["evidence_ref"], field=f"{field}.evidence_ref") + relation = _enum( + payload["relation_code"], + field=f"{field}.relation_code", + allowed=_EXPLICIT_RELATIONS, + code="unknown_explicit_relation", + ) + return ExplicitParent( + cast(str, reference), + cast(ExplicitRelationCode, relation), + ) + + +def _parse_record(value: object, *, index: int) -> LineageEvidenceRecord: + """Parse one bounded evidence record from the request array.""" + + field = f"records[{index}]" + payload = _object( + value, + field=field, + allowed=frozenset( + { + "evidence_ref", + "group_ref", + "source_kind_code", + "truth_status_code", + "label", + "occurred_at", + "available_at", + "secondary_key", + "project_ref", + "explicit_parent", + } + ), + required=frozenset( + { + "evidence_ref", + "group_ref", + "source_kind_code", + "truth_status_code", + "label", + "occurred_at", + "available_at", + } + ), + ) + return LineageEvidenceRecord( + evidence_ref=cast( + str, + _opaque_reference(payload["evidence_ref"], field=f"{field}.evidence_ref"), + ), + group_ref=cast( + str, + _opaque_reference(payload["group_ref"], field=f"{field}.group_ref"), + ), + source_kind_code=cast( + SourceKindCode, + _enum( + payload["source_kind_code"], + field=f"{field}.source_kind_code", + allowed=_SOURCE_KINDS, + code="unknown_source_kind", + ), + ), + truth_status_code=cast( + CallerTruthStatusCode, + _enum( + payload["truth_status_code"], + field=f"{field}.truth_status_code", + allowed=_CALLER_TRUTH_STATUSES, + code="unknown_caller_truth_status", + ), + ), + label=_string( + payload["label"], + field=f"{field}.label", + maximum=MAX_LABEL_LENGTH, + ), + occurred_at=cast( + datetime, + _timestamp(payload["occurred_at"], field=f"{field}.occurred_at"), + ), + available_at=cast( + datetime, + _timestamp(payload["available_at"], field=f"{field}.available_at"), + ), + secondary_key=_opaque_reference( + payload.get("secondary_key"), + field=f"{field}.secondary_key", + optional=True, + ), + project_ref=_opaque_reference( + payload.get("project_ref"), + field=f"{field}.project_ref", + optional=True, + ), + explicit_parent=_parse_explicit_parent( + payload.get("explicit_parent"), + field=f"{field}.explicit_parent", + ), + ) + + +def _parse_policy(value: object) -> LineageAnalysisPolicy: + """Parse the bounded reconstruction policy.""" + + payload = _object( + value, + field="policy", + allowed=frozenset( + { + "candidate_window", + "maximum_pair_evaluations", + "minimum_fused_score", + "allow_llm", + } + ), + required=frozenset( + { + "candidate_window", + "maximum_pair_evaluations", + "minimum_fused_score", + "allow_llm", + } + ), + ) + return LineageAnalysisPolicy( + candidate_window=_integer( + payload["candidate_window"], + field="policy.candidate_window", + minimum=1, + maximum=MAX_CANDIDATE_WINDOW, + ), + maximum_pair_evaluations=_integer( + payload["maximum_pair_evaluations"], + field="policy.maximum_pair_evaluations", + minimum=1, + maximum=MAX_PAIR_EVALUATIONS, + ), + minimum_fused_score=_number( + payload["minimum_fused_score"], + field="policy.minimum_fused_score", + minimum=0.0, + maximum=1.0, + ), + allow_llm=_boolean(payload["allow_llm"], field="policy.allow_llm"), + ) + + +def parse_lineage_analysis_request(payload: object) -> LineageAnalysisRequest: + """Parse and strictly validate one external lineage analysis request.""" + + data = _object( + payload, + field="request", + allowed=frozenset( + { + "contract_version", + "analysis_id", + "analysis_scope_code", + "knowledge_cutoff", + "policy", + "records", + } + ), + required=frozenset( + { + "contract_version", + "analysis_id", + "analysis_scope_code", + "policy", + "records", + } + ), + ) + version = _string( + data["contract_version"], + field="contract_version", + maximum=16, + ) + if version != CONTRACT_VERSION: + _raise( + "unsupported_contract_version", + f"only contract version {CONTRACT_VERSION!r} is accepted", + "contract_version", + ) + records_payload = data["records"] + if not isinstance(records_payload, list): + _raise("invalid_field_type", "records must be an array", "records") + if not 1 <= len(records_payload) <= MAX_RECORD_COUNT: + _raise( + "record_count_out_of_bounds", + f"records must contain 1..{MAX_RECORD_COUNT} entries", + "records", + ) + records = tuple( + _parse_record(value, index=index) + for index, value in enumerate(records_payload) + ) + seen: set[str] = set() + for record in records: + if record.evidence_ref in seen: + _raise( + "duplicate_evidence_ref", + f"duplicate evidence reference {record.evidence_ref!r}", + "records", + ) + seen.add(record.evidence_ref) + return LineageAnalysisRequest( + contract_version=version, + analysis_id=cast( + str, + _opaque_reference(data["analysis_id"], field="analysis_id"), + ), + analysis_scope_code=cast( + AnalysisScopeCode, + _enum( + data["analysis_scope_code"], + field="analysis_scope_code", + allowed=_ANALYSIS_SCOPES, + code="unknown_analysis_scope", + ), + ), + knowledge_cutoff=_timestamp( + data.get("knowledge_cutoff"), + field="knowledge_cutoff", + optional=True, + ), + policy=_parse_policy(data["policy"]), + records=records, + ) + + +def _time_text(value: datetime | None) -> str | None: + """Serialize an aware timestamp canonically in UTC with a ``Z`` suffix.""" + + if value is None: + return None + if value.tzinfo is None or value.utcoffset() is None: + _raise( + "timestamp_must_be_offset_aware", + "result timestamp must carry an offset", + ) + utc = value.astimezone(timezone.utc) + text = utc.isoformat(timespec="microseconds").replace( + ".000000+00:00", + "Z", + ) + return text.replace("+00:00", "Z") + + +def _record_dict(record: LineageEvidenceRecord) -> dict[str, object]: + """Serialize one evidence record without adding derived authority.""" + + explicit_parent: dict[str, object] | None = None + if record.explicit_parent is not None: + explicit_parent = { + "evidence_ref": record.explicit_parent.evidence_ref, + "relation_code": record.explicit_parent.relation_code, + } + return { + "evidence_ref": record.evidence_ref, + "group_ref": record.group_ref, + "source_kind_code": record.source_kind_code, + "truth_status_code": record.truth_status_code, + "label": record.label, + "occurred_at": _time_text(record.occurred_at), + "available_at": _time_text(record.available_at), + "secondary_key": record.secondary_key, + "project_ref": record.project_ref, + "explicit_parent": explicit_parent, + } + + +def serialize_lineage_analysis_request( + request: LineageAnalysisRequest, +) -> dict[str, object]: + """Serialize a request canonically with records ordered by evidence reference.""" + + return { + "contract_version": request.contract_version, + "analysis_id": request.analysis_id, + "analysis_scope_code": request.analysis_scope_code, + "knowledge_cutoff": _time_text(request.knowledge_cutoff), + "policy": { + "candidate_window": request.policy.candidate_window, + "maximum_pair_evaluations": request.policy.maximum_pair_evaluations, + "minimum_fused_score": request.policy.minimum_fused_score, + "allow_llm": request.policy.allow_llm, + }, + "records": [ + _record_dict(record) + for record in sorted( + request.records, + key=lambda item: item.evidence_ref, + ) + ], + } + + +def _score(value: float, *, field: str) -> float: + """Validate and canonically round one result score in ``[0, 1]``.""" + + if isinstance(value, bool) or not isinstance(value, (int, float)): + _raise("invalid_field_type", "score must be numeric", field) + number = float(value) + if not math.isfinite(number) or not 0.0 <= number <= 1.0: + _raise( + "score_out_of_bounds", + "score must be finite and within 0..1", + field, + ) + return round(number, 12) + + +def _validated_reference_partition( + values: tuple[str, ...], + *, + field: str, +) -> frozenset[str]: + """Validate one unique result evidence-reference partition.""" + + if len(set(values)) != len(values): + _raise( + "duplicate_evidence_ref", + "result partition contains duplicate references", + field, + ) + for value in values: + _opaque_reference(value, field=field) + return frozenset(values) + + +def _channel_dict(channel: ChannelEvidence) -> dict[str, object]: + """Serialize one exact active-channel contribution.""" + + return { + "channel_code": _string( + channel.channel_code, + field="channel.channel_code", + maximum=64, + ), + "score": _score(channel.score, field="channel.score"), + "weight": _score(channel.weight, field="channel.weight"), + "contribution": _score( + channel.contribution, + field="channel.contribution", + ), + } + + +def _edge_dict( + edge: LineageEdgeResult, + *, + included_refs: frozenset[str], +) -> dict[str, object]: + """Serialize one edge and verify its evidence math and references.""" + + _opaque_reference( + edge.parent_evidence_ref, + field="edge.parent_evidence_ref", + ) + _opaque_reference( + edge.child_evidence_ref, + field="edge.child_evidence_ref", + ) + if edge.parent_evidence_ref == edge.child_evidence_ref: + _raise("self_lineage_edge", "lineage edge cannot reference itself", "edge") + if ( + edge.parent_evidence_ref not in included_refs + or edge.child_evidence_ref not in included_refs + ): + _raise( + "edge_reference_not_included", + "edge references evidence outside the included partition", + "edge", + ) + _enum( + edge.truth_status_code, + field="edge.truth_status_code", + allowed=_EDGE_TRUTH_STATUSES, + code="unknown_result_truth_status", + ) + fused_score = _score(edge.fused_score, field="edge.fused_score") + channels = tuple(edge.channel_evidence) + if not channels: + _raise( + "missing_channel_evidence", + "edge must disclose at least one channel", + "edge.channel_evidence", + ) + channel_codes = [channel.channel_code for channel in channels] + if len(set(channel_codes)) != len(channel_codes): + _raise( + "duplicate_channel_code", + "edge contains duplicate channel codes", + "edge.channel_evidence", + ) + serialized_channels = [_channel_dict(channel) for channel in channels] + weight_sum = sum(float(item["weight"]) for item in serialized_channels) + if not math.isclose(weight_sum, 1.0, abs_tol=_SCORE_TOLERANCE): + _raise( + "channel_weight_sum_mismatch", + "active channel weights must sum to one", + "edge.channel_evidence", + ) + for item in serialized_channels: + expected = float(item["score"]) * float(item["weight"]) + if not math.isclose( + float(item["contribution"]), + expected, + abs_tol=_SCORE_TOLERANCE, + ): + _raise( + "channel_contribution_mismatch", + "each contribution must equal score multiplied by weight", + str(item["channel_code"]), + ) + contribution_sum = sum( + float(item["contribution"]) + for item in serialized_channels + ) + if not math.isclose( + contribution_sum, + fused_score, + abs_tol=_SCORE_TOLERANCE, + ): + _raise( + "channel_contribution_mismatch", + "channel contributions must reconcile to the fused score", + "edge.channel_evidence", + ) + return { + "parent_evidence_ref": edge.parent_evidence_ref, + "child_evidence_ref": edge.child_evidence_ref, + "relation_type_code": _string( + edge.relation_type_code, + field="edge.relation_type_code", + maximum=64, + ), + "truth_status_code": edge.truth_status_code, + "fused_score": fused_score, + "channel_evidence": sorted( + serialized_channels, + key=lambda item: cast(str, item["channel_code"]), + ), + } + + +def _project_dict( + project: ProjectProjection, + *, + included_refs: frozenset[str], +) -> dict[str, object]: + """Serialize one proposed project grouping and validate its references.""" + + _opaque_reference(project.group_ref, field="project.group_ref") + _opaque_reference(project.project_ref, field="project.project_ref") + if project.truth_status_code != "proposed": + _raise( + "unknown_result_truth_status", + "project projection must remain proposed", + "project.truth_status_code", + ) + evidence_refs = tuple(project.evidence_refs) + if len(set(evidence_refs)) != len(evidence_refs): + _raise( + "duplicate_evidence_ref", + "project projection contains duplicate evidence references", + "project.evidence_refs", + ) + for evidence_ref in evidence_refs: + _opaque_reference(evidence_ref, field="project.evidence_refs") + if evidence_ref not in included_refs: + _raise( + "project_reference_not_included", + "project projection references evidence outside the included partition", + evidence_ref, + ) + return { + "group_ref": project.group_ref, + "project_ref": project.project_ref, + "evidence_refs": sorted(evidence_refs), + "truth_status_code": project.truth_status_code, + } + + +def _limitation_dict(limitation: LineageLimitation) -> dict[str, object]: + """Serialize one bounded machine-readable limitation.""" + + if limitation.evidence_ref is not None: + _opaque_reference( + limitation.evidence_ref, + field="limitation.evidence_ref", + ) + return { + "limitation_code": _string( + limitation.limitation_code, + field="limitation.limitation_code", + maximum=96, + ), + "evidence_ref": limitation.evidence_ref, + "message": _string( + limitation.message, + field="limitation.message", + maximum=500, + ), + } + + +def serialize_lineage_analysis_result( + result: LineageAnalysisResult, + *, + include_digest: bool = True, +) -> dict[str, object]: + """Serialize a result with deterministic ordering and full invariants.""" + + if result.contract_version != CONTRACT_VERSION: + _raise( + "unsupported_contract_version", + "result contract version is unsupported", + "contract_version", + ) + _opaque_reference(result.analysis_id, field="analysis_id") + _enum( + result.analysis_scope_code, + field="analysis_scope_code", + allowed=_ANALYSIS_SCOPES, + code="unknown_analysis_scope", + ) + _enum( + result.llm_status_code, + field="llm_status_code", + allowed=_LLM_STATUSES, + code="unknown_llm_status", + ) + included_refs = _validated_reference_partition( + result.included_evidence_refs, + field="included_evidence_refs", + ) + excluded_refs = _validated_reference_partition( + result.excluded_evidence_refs, + field="excluded_evidence_refs", + ) + if included_refs & excluded_refs: + _raise( + "evidence_partition_overlap", + "included and excluded evidence partitions must be disjoint", + "evidence_refs", + ) + payload: dict[str, object] = { + "contract_version": result.contract_version, + "analysis_id": result.analysis_id, + "analysis_scope_code": result.analysis_scope_code, + "knowledge_cutoff": _time_text(result.knowledge_cutoff), + "included_evidence_refs": sorted(included_refs), + "excluded_evidence_refs": sorted(excluded_refs), + "llm_status_code": result.llm_status_code, + "edges": [ + _edge_dict(edge, included_refs=included_refs) + for edge in sorted( + result.edges, + key=lambda item: ( + item.child_evidence_ref, + item.parent_evidence_ref, + item.relation_type_code, + ), + ) + ], + "project_projections": [ + _project_dict(project, included_refs=included_refs) + for project in sorted( + result.project_projections, + key=lambda item: (item.group_ref, item.project_ref), + ) + ], + "limitations": [ + _limitation_dict(limitation) + for limitation in sorted( + result.limitations, + key=lambda item: ( + item.limitation_code, + item.evidence_ref or "", + item.message, + ), + ) + ], + } + if include_digest: + if not _RESULT_DIGEST.fullmatch(result.result_digest): + _raise( + "invalid_result_digest", + "result digest must be a lowercase SHA-256 identifier", + "result_digest", + ) + expected_digest = _digest(payload) + if result.result_digest != expected_digest: + _raise( + "result_digest_mismatch", + "result digest does not match canonical result content", + "result_digest", + ) + payload["result_digest"] = result.result_digest + return payload + + +def _digest(payload: dict[str, object]) -> str: + """Return a SHA-256 digest over canonical UTF-8 JSON.""" + + canonical = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def request_digest(request: LineageAnalysisRequest) -> str: + """Return the deterministic digest of one semantic request.""" + + return _digest(serialize_lineage_analysis_request(request)) + + +def result_digest(result: LineageAnalysisResult) -> str: + """Return the deterministic digest of a result excluding its digest field.""" + + without_digest = replace(result, result_digest="") + return _digest( + serialize_lineage_analysis_result( + without_digest, + include_digest=False, + ) + ) diff --git a/lineageweave/image_content.py b/lineageweave/image_content.py index f6500ffd3..a0d298a91 100644 --- a/lineageweave/image_content.py +++ b/lineageweave/image_content.py @@ -141,7 +141,8 @@ class ImageDescription: Attributes: extracted_text: OCR result -- every piece of legible text found in the image, empty string if none. - caption: one-sentence description of what the image shows. + caption: factual description of the visible entities, relationships, + and layout that make the image useful as semantic evidence. tags: short tags for the main objects/subjects, for independent keyword search separate from the free-text caption. """ @@ -177,15 +178,18 @@ def describe(self, image_bytes: bytes, mime_type: str) -> ImageDescription: # p _RESPONSE_FORMAT = ( - "Examine this image. Reply with EXACTLY three lines, no extra commentary:\n" + "Examine this image. Reply with exactly the three labeled sections below and no " + "extra commentary. TEXT may span multiple lines; CAPTION and TAGS stay on their " + "labeled lines.\n" "TEXT: \n" + "If the image contains a table, preserve its row/column structure as a Markdown " + "pipe table: one row per line, with a separator row immediately after the visible " + "header. Never flatten a table into an unstructured word list or invent a header " + "that is not visible.>\n" "CAPTION: <2-4 concise, evidence-grounded sentences describing the visible layout, " "objects, relationships, directions, measurements, and labels; do not guess " - "anything that is not visible>\n" - "TAGS: " + "anything that is not visible. Omit anything the pixels do not support.>\n" + "TAGS: " ) _REGION_RESPONSE_FORMAT = ( "Find distinct meaningful visual regions in this image for separate OCR and description. " @@ -251,14 +255,15 @@ def _parse_description(content: str) -> ImageDescription: remainder = _strip_outer_markdown_emphasis(match.group(2)) if remainder: fields[label].append(remainder) - multiline_field = "TEXT" if label == "TEXT" else None + multiline_field = label if label in {"TEXT", "CAPTION"} else None continue - if re.match(r"^\s*[*_`>#\-\s]*[A-Za-z][A-Za-z0-9 _-]*\s*:", line): - multiline_field = None - continue - if multiline_field == "TEXT" and line.strip(): - fields["TEXT"].append(_strip_outer_markdown_emphasis(line)) + if multiline_field in {"TEXT", "CAPTION"} and line.strip(): + # A colon is common inside OCR (for example ``Date: 2026-08-21``). + # Only the known response labels above end the active section; + # treating every colon as a provider label loses real image text + # or the continuation of a detailed caption. + fields[multiline_field].append(_strip_outer_markdown_emphasis(line)) if not fields["TEXT"] and not fields["CAPTION"]: raise ImageDescriptionParseError("vision response had no usable TEXT or CAPTION content") @@ -289,7 +294,7 @@ def __init__( api_key: str, model: str | None = None, *, - timeout: float = 180.0, + timeout: float = 600.0, allow_insecure_http: bool = False, ) -> None: parsed = urlparse(base_url) diff --git a/lineageweave/lineage_contract.py b/lineageweave/lineage_contract.py new file mode 100644 index 000000000..19520d70a --- /dev/null +++ b/lineageweave/lineage_contract.py @@ -0,0 +1,452 @@ +"""Versioned, store-agnostic provider contract for bounded lineage analysis. + +The contract accepts evidence that a caller has already authorized. It never +opens a database, accepts provider credentials, or returns a source identifier +that was not present in the request. +""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from datetime import UTC, datetime +from hashlib import sha256 + +from .adjudication_client import AdjudicationClient, NullAdjudicationClient +from .models import Record +from .reconstruct import reconstruct + +CONTRACT_VERSION = "lineage-analysis/v1" +_MAX_REFERENCE_LENGTH = 200 +_MAX_TEXT_LENGTH = 8_000 +_MAX_PROJECT_HINTS = 500 +_MAX_EMAIL_REFS_PER_RECORD = 64 + + +class _ProviderChannelUnavailable(Exception): + """Private signal used to drop a failed provider channel safely.""" + + +class _SafeAdjudicationClient: + """Keep raw provider failures outside the public contract boundary.""" + + available = True + + def __init__(self, client: AdjudicationClient) -> None: + """Wrap one available provider client.""" + self._client = client + + def judge(self, candidate_label: str, record_label: str) -> float: + """Return a bounded score or signal that the channel is unavailable.""" + try: + score = self._client.judge(candidate_label, record_label) + except Exception as exc: + raise _ProviderChannelUnavailable from exc + if isinstance(score, bool) or not isinstance(score, (int, float)): + raise _ProviderChannelUnavailable + number = float(score) + if not math.isfinite(number) or not 0.0 <= number <= 1.0: + raise _ProviderChannelUnavailable + return number + + +def _required_text(value: str, field_name: str, *, maximum: int = _MAX_REFERENCE_LENGTH) -> str: + """Validate and return a bounded non-empty contract string.""" + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field_name} must be a non-empty string") + if len(value) > maximum: + raise ValueError(f"{field_name} exceeds the {maximum}-character limit") + return value + + +def _utc_timestamp(value: datetime, field_name: str) -> datetime: + """Require an explicitly timezone-aware timestamp and normalize it to UTC.""" + if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{field_name} must be timezone-aware") + return value.astimezone(UTC) + + +def _timestamp_json(value: datetime) -> str: + """Serialize a timestamp in one canonical UTC representation.""" + return _utc_timestamp(value, "timestamp").isoformat().replace("+00:00", "Z") + + +@dataclass(frozen=True) +class EmailEvidence: + """Email evidence kept separate from semantic/project lineage signals.""" + + rfc_message_id: str | None = None + references: tuple[str, ...] = () + in_reply_to: str | None = None + provider_thread_id: str | None = None + raw_content_hash: str | None = None + participant_refs: tuple[str, ...] = () + attachment_refs: tuple[str, ...] = () + + def to_dict(self) -> dict[str, object]: + """Return the RFC/provider evidence without flattening it into a score.""" + return { + "rfc_message_id": self.rfc_message_id, + "references": list(self.references), + "in_reply_to": self.in_reply_to, + "provider_thread_id": self.provider_thread_id, + "raw_content_hash": self.raw_content_hash, + "participant_refs": list(self.participant_refs), + "attachment_refs": list(self.attachment_refs), + } + + +@dataclass(frozen=True) +class LineageProjectHint: + """Caller-supplied project hint; it is never an authoritative project state.""" + + evidence_ref: str + project_ref: str + label: str + + def validate(self, evidence_refs: set[str]) -> None: + """Ensure the hint only points at evidence in this request.""" + _required_text(self.evidence_ref, "project_hint.evidence_ref") + _required_text(self.project_ref, "project_hint.project_ref") + _required_text(self.label, "project_hint.label", maximum=_MAX_TEXT_LENGTH) + if self.evidence_ref not in evidence_refs: + raise ValueError("project hint references evidence outside the request") + + def to_dict(self) -> dict[str, str]: + """Return the bounded project hint representation.""" + return { + "evidence_ref": self.evidence_ref, + "project_ref": self.project_ref, + "label": self.label, + } + + +@dataclass(frozen=True) +class LineageEvidenceRecord: + """One already-authorized record submitted under an opaque caller reference.""" + + evidence_ref: str + group_key: str + label: str + occurred_at: datetime + available_at: datetime + secondary_key: str = "" + body_text: str = "" + email: EmailEvidence | None = None + + def validate(self, *, max_body_chars: int, max_email_refs_per_record: int) -> None: + """Validate identity, content bounds, and both independent clocks.""" + _required_text(self.evidence_ref, "evidence.evidence_ref") + _required_text(self.group_key, "evidence.group_key") + _required_text(self.label, "evidence.label", maximum=_MAX_TEXT_LENGTH) + if not isinstance(self.body_text, str) or len(self.body_text) > max_body_chars: + raise ValueError(f"evidence.body_text exceeds the {max_body_chars}-character limit") + if self.secondary_key and len(self.secondary_key) > _MAX_REFERENCE_LENGTH: + raise ValueError("evidence.secondary_key exceeds the reference limit") + _utc_timestamp(self.occurred_at, "evidence.occurred_at") + _utc_timestamp(self.available_at, "evidence.available_at") + if self.email is not None: + for field_name in ( + "rfc_message_id", + "in_reply_to", + "provider_thread_id", + "raw_content_hash", + ): + value = getattr(self.email, field_name) + if value is not None: + _required_text(value, f"evidence.email.{field_name}", maximum=_MAX_TEXT_LENGTH) + for field_name in ("references", "participant_refs", "attachment_refs"): + if len(getattr(self.email, field_name)) > max_email_refs_per_record: + raise ValueError( + f"evidence.email.{field_name} exceeds the " + f"{max_email_refs_per_record}-item limit" + ) + for value in getattr(self.email, field_name): + _required_text(value, f"evidence.email.{field_name}") + + def to_dict(self) -> dict[str, object]: + """Return the source-shaped record used for canonical request hashing.""" + return { + "evidence_ref": self.evidence_ref, + "group_key": self.group_key, + "label": self.label, + "occurred_at": _timestamp_json(self.occurred_at), + "available_at": _timestamp_json(self.available_at), + "secondary_key": self.secondary_key, + "body_text": self.body_text, + "email": self.email.to_dict() if self.email is not None else None, + } + + +@dataclass(frozen=True) +class LineageAnalysisPolicy: + """Bounded, caller-visible policy for one stateless provider invocation.""" + + max_evidence_records: int = 500 + max_body_chars: int = 4_000 + max_project_hints: int = 500 + max_email_refs_per_record: int = _MAX_EMAIL_REFS_PER_RECORD + + def validate(self) -> None: + """Reject unbounded or nonsensical request budgets.""" + if not 1 <= self.max_evidence_records <= 5_000: + raise ValueError("max_evidence_records must be between 1 and 5000") + if not 0 <= self.max_body_chars <= _MAX_TEXT_LENGTH: + raise ValueError(f"max_body_chars must be between 0 and {_MAX_TEXT_LENGTH}") + if not 0 <= self.max_project_hints <= _MAX_PROJECT_HINTS: + raise ValueError(f"max_project_hints must be between 0 and {_MAX_PROJECT_HINTS}") + if not 0 <= self.max_email_refs_per_record <= _MAX_EMAIL_REFS_PER_RECORD: + raise ValueError( + "max_email_refs_per_record must be between " + f"0 and {_MAX_EMAIL_REFS_PER_RECORD}" + ) + + def to_dict(self) -> dict[str, int]: + """Return the policy values included in request identity.""" + return { + "max_evidence_records": self.max_evidence_records, + "max_body_chars": self.max_body_chars, + "max_project_hints": self.max_project_hints, + "max_email_refs_per_record": self.max_email_refs_per_record, + } + + +@dataclass(frozen=True) +class LineageAnalysisRequest: + """Immutable request that binds authorization, evidence, policy, and cutoff.""" + + analysis_id: str + authorization_scope_ref: str + knowledge_cutoff: datetime + evidence: tuple[LineageEvidenceRecord, ...] + project_hints: tuple[LineageProjectHint, ...] = () + policy: LineageAnalysisPolicy = LineageAnalysisPolicy() + + def validate(self) -> None: + """Validate the complete request before any reconstruction occurs.""" + _required_text(self.analysis_id, "analysis_id") + _required_text(self.authorization_scope_ref, "authorization_scope_ref") + _utc_timestamp(self.knowledge_cutoff, "knowledge_cutoff") + self.policy.validate() + if len(self.evidence) > self.policy.max_evidence_records: + raise ValueError("evidence exceeds max_evidence_records") + if len(self.project_hints) > self.policy.max_project_hints: + raise ValueError("project_hints exceeds max_project_hints") + evidence_refs: set[str] = set() + for record in self.evidence: + record.validate( + max_body_chars=self.policy.max_body_chars, + max_email_refs_per_record=self.policy.max_email_refs_per_record, + ) + if record.evidence_ref in evidence_refs: + raise ValueError("evidence_ref values must be unique") + evidence_refs.add(record.evidence_ref) + for hint in self.project_hints: + hint.validate(evidence_refs) + + def to_dict(self) -> dict[str, object]: + """Return the versioned request envelope for transport and hashing.""" + return { + "contract_version": CONTRACT_VERSION, + "analysis_id": self.analysis_id, + "authorization_scope_ref": self.authorization_scope_ref, + "knowledge_cutoff": _timestamp_json(self.knowledge_cutoff), + "evidence": [record.to_dict() for record in self.evidence], + "project_hints": [hint.to_dict() for hint in self.project_hints], + "policy": self.policy.to_dict(), + } + + def canonical_json(self) -> str: + """Return deterministic JSON suitable for an idempotency digest.""" + self.validate() + return json.dumps(self.to_dict(), ensure_ascii=True, sort_keys=True, separators=(",", ":")) + + def request_digest(self) -> str: + """Return the stable identity of this exact analysis request.""" + return sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class LineageChannelEvidence: + """One channel's evidence for one reconstructed edge.""" + + channel_code: str + truth_status: str + score: float | None + evidence_refs: tuple[str, ...] + + def to_dict(self) -> dict[str, object]: + """Return channel evidence with explicit unavailable scores.""" + return { + "channel_code": self.channel_code, + "truth_status": self.truth_status, + "score": self.score, + "evidence_refs": list(self.evidence_refs), + } + + +@dataclass(frozen=True) +class LineageEdgeResult: + """One inferred parent-to-child relation bounded to submitted evidence.""" + + parent_evidence_ref: str + child_evidence_ref: str + truth_status: str + fused_score: float + channel_evidence: tuple[LineageChannelEvidence, ...] + + def to_dict(self) -> dict[str, object]: + """Return an edge without introducing database or provider identifiers.""" + return { + "parent_evidence_ref": self.parent_evidence_ref, + "child_evidence_ref": self.child_evidence_ref, + "truth_status": self.truth_status, + "fused_score": self.fused_score, + "channel_evidence": [item.to_dict() for item in self.channel_evidence], + } + + +@dataclass(frozen=True) +class LineageLimitation: + """Explicit unavailable or non-authoritative result condition.""" + + code: str + detail: str + + def to_dict(self) -> dict[str, str]: + """Return the bounded limitation description.""" + return {"code": self.code, "detail": self.detail} + + +@dataclass(frozen=True) +class LineageAnalysisResult: + """Serializable analysis result with provenance and explicit limitations.""" + + analysis_id: str + request_digest: str + knowledge_cutoff: datetime + edges: tuple[LineageEdgeResult, ...] + limitations: tuple[LineageLimitation, ...] + + def to_dict(self) -> dict[str, object]: + """Return the versioned result envelope.""" + return { + "contract_version": CONTRACT_VERSION, + "analysis_id": self.analysis_id, + "request_digest": self.request_digest, + "knowledge_cutoff": _timestamp_json(self.knowledge_cutoff), + "edges": [edge.to_dict() for edge in self.edges], + "limitations": [limitation.to_dict() for limitation in self.limitations], + } + + def to_json(self) -> str: + """Return deterministic JSON for a service response or fixture.""" + return json.dumps(self.to_dict(), ensure_ascii=True, sort_keys=True, separators=(",", ":")) + + +def analyze_lineage( + request: LineageAnalysisRequest, + *, + llm: AdjudicationClient | None = None, +) -> LineageAnalysisResult: + """Run the existing reconstruction over only evidence known at the cutoff. + + The function is deliberately store-agnostic. A caller owns authorization + and persistence; this boundary only validates bounded evidence, reuses the + reviewed reconstruction pipeline, and maps every output edge back to an + input-owned opaque reference. + """ + request.validate() + cutoff = _utc_timestamp(request.knowledge_cutoff, "knowledge_cutoff") + eligible = [ + record + for record in request.evidence + if ( + _utc_timestamp(record.available_at, "evidence.available_at") <= cutoff + and _utc_timestamp(record.occurred_at, "evidence.occurred_at") <= cutoff + ) + ] + limitations: list[LineageLimitation] = [] + excluded_count = len(request.evidence) - len(eligible) + if excluded_count: + limitations.append( + LineageLimitation( + "evidence_after_cutoff_excluded", + f"{excluded_count} evidence record(s) were unavailable or occurred after the knowledge cutoff.", + ) + ) + if llm is None or not getattr(llm, "available", False): + limitations.append( + LineageLimitation( + "llm_channel_unavailable", + "The optional LLM channel was unavailable; channel weights were renormalized.", + ) + ) + if request.project_hints: + limitations.append( + LineageLimitation( + "project_hints_are_non_authoritative", + "Project hints remain caller evidence and do not change authoritative project status.", + ) + ) + + records = [ + Record( + record_id=record.evidence_ref, + group_key=record.group_key, + label=record.label, + occurred_at=_utc_timestamp(record.occurred_at, "evidence.occurred_at").replace(tzinfo=None), + secondary_key=record.secondary_key, + ) + for record in eligible + ] + effective_llm = ( + _SafeAdjudicationClient(llm) + if llm is not None and getattr(llm, "available", False) + else llm + ) + if not records: + trees = [] + else: + try: + trees = reconstruct(records, llm=effective_llm) + except _ProviderChannelUnavailable: + limitations.append( + LineageLimitation( + "llm_channel_unavailable", + "The LLM channel failed safely; channel weights were renormalized.", + ) + ) + trees = reconstruct(records, llm=NullAdjudicationClient()) + edges = [edge for tree in trees for edge in tree.edges] + eligible_refs = {record.evidence_ref for record in eligible} + result_edges: list[LineageEdgeResult] = [] + for edge in edges: + if edge.parent_id not in eligible_refs or edge.child_id not in eligible_refs: + continue + channel_evidence = tuple( + LineageChannelEvidence( + channel_code=channel, + truth_status="inferred", + score=score, + evidence_refs=(edge.parent_id, edge.child_id), + ) + for channel, score in sorted(edge.channel_scores.items()) + ) + result_edges.append( + LineageEdgeResult( + parent_evidence_ref=edge.parent_id, + child_evidence_ref=edge.child_id, + truth_status="inferred", + fused_score=edge.fused_score, + channel_evidence=channel_evidence, + ) + ) + return LineageAnalysisResult( + analysis_id=request.analysis_id, + request_digest=request.request_digest(), + knowledge_cutoff=cutoff, + edges=tuple(result_edges), + limitations=tuple(limitations), + ) diff --git a/lineageweave/post_content_normalization.py b/lineageweave/post_content_normalization.py index 196b7e7b4..2e2fe2c42 100644 --- a/lineageweave/post_content_normalization.py +++ b/lineageweave/post_content_normalization.py @@ -40,8 +40,8 @@ # what chunk_by_dom already splits on, plus the inline/replaced tags # that carry images or wrap rich-text fragments. _HTML_OPEN_TAG = re.compile( - r"<\s*/?\s*(?:article|section|nav|aside|header|footer|div|p|li|td|th|tr|" - r"table|blockquote|h[1-6]|img|br|hr|ul|ol|span|strong|em|b|i|u|a|" + r"<\s*/?\s*(?:article|section|nav|aside|header|footer|div|p|li|td|th|tr|sup|" + r"table|blockquote|h[1-6]|img|br|hr|ul|ol|oi|span|strong|em|b|i|u|a|" r"html|body|head|style|script|font|center|pre)\b", re.IGNORECASE, ) @@ -262,6 +262,11 @@ def normalize_post_body( vision_client = NullImageContentClient() if not _looks_like_html(body): + markdown_chunks = chunk_by_dom(body) + if any(chunk.label == "markdown_tr" for chunk in markdown_chunks): + return NormalizedPostContent( + text="\n\n".join(chunk.text for chunk in markdown_chunks if chunk.text) + ) return NormalizedPostContent(text=normalize_semantic_text(body)) chunks: list[Chunk] = chunk_by_dom(body) diff --git a/lineageweave/post_content_persistence.py b/lineageweave/post_content_persistence.py index 32caa944b..745e15599 100644 --- a/lineageweave/post_content_persistence.py +++ b/lineageweave/post_content_persistence.py @@ -30,6 +30,10 @@ _LOGGER = logging.getLogger(__name__) +class ImageOcrPreservationError(RuntimeError): + """A retry would erase stronger OCR already persisted for the same image.""" + + def _bounded_unit_batches( # noqa: UP047 - retain Python 3.10 compatibility. units: list[tuple[_BatchKey, str]], ) -> list[list[tuple[_BatchKey, str]]]: @@ -84,12 +88,24 @@ async def persist_post_content( Provider calls happen before the short database transaction. A failed or unavailable embedding call writes no vector row; it never writes a zero or - guessed vector. The raw body remains in ``source_post`` for future retry. + guessed vector. A same-image retry cannot replace non-empty persisted OCR + with an empty result. The raw body remains in ``source_post`` for future + retry. """ normalized = normalized_result or normalize_post_body(body, vision_client) chunks = chunk_by_source_body(body) image_results = {result.chunk_index: result for result in normalized.image_results} formatting = {hint.chunk_index: hint.style for hint in normalized.formatting_hints} + image_ocr_by_sha256: dict[str, bool] = {} + for chunk in chunks: + if chunk.unit_type != "image" or chunk.image_data is None: + continue + result = image_results.get(chunk.index) + description = result.description if result else None + content_sha256 = hashlib.sha256(chunk.image_data).hexdigest() + image_ocr_by_sha256[content_sha256] = image_ocr_by_sha256.get( + content_sha256, False + ) or bool(description and description.extracted_text.strip()) prepared: list[tuple[Chunk, str, str | None]] = [] for chunk in chunks: @@ -226,6 +242,29 @@ async def persist_post_content( ) async with conn.transaction(): + if image_ocr_by_sha256: + await conn.fetchval( + "select post_id from source_post where post_id = $1 for update", + post_id, + ) + previous_images = await conn.fetch( + """ + select image.content_sha256, image.extracted_text + from post_content_unit unit + join post_content_image image using (post_content_unit_id) + where unit.post_id = $1 + and nullif(btrim(image.extracted_text), '') is not null + """, + post_id, + ) + if any( + row["content_sha256"] in image_ocr_by_sha256 + and not image_ocr_by_sha256[row["content_sha256"]] + for row in previous_images + ): + raise ImageOcrPreservationError( + "refusing to replace non-empty image OCR with an empty retry result" + ) await conn.execute("delete from post_content_unit where post_id = $1", post_id) unit_ids: dict[int, str] = {} for chunk, unit_text, style in prepared: diff --git a/lineageweave/reconstruct.py b/lineageweave/reconstruct.py index e0a6f89d0..b8d4f4147 100644 --- a/lineageweave/reconstruct.py +++ b/lineageweave/reconstruct.py @@ -16,7 +16,11 @@ import rankweave as rw import threadweave as tw -from .adjudication_client import AdjudicationClient, NullAdjudicationClient +from .adjudication_client import ( + AdjudicationClient, + AdjudicationClientError, + NullAdjudicationClient, +) from .channels import secondary_key_match_score, temporal_score, text_similarity_score from .models import Edge, Record, Tree @@ -83,7 +87,12 @@ def _best_parent( "text": text_similarity_score(candidate, record), } if "llm" in weights: - scores["llm"] = llm.judge(candidate.label, record.label) + try: + scores["llm"] = llm.judge(candidate.label, record.label) + except AdjudicationClientError: + # A malformed provider response invalidates only this optional + # pair; deterministic channels still produce the edge. + scores["llm"] = 0.0 for channel, score in scores.items(): channel_results[channel].append((candidate.record_id, score)) per_candidate_scores[candidate.record_id][channel] = score diff --git a/lineageweave/relation_verification.py b/lineageweave/relation_verification.py index 509c7552a..e670f2262 100644 --- a/lineageweave/relation_verification.py +++ b/lineageweave/relation_verification.py @@ -1,25 +1,10 @@ -"""Verifies whether an LLM-inferred Ontology relation has any real-world -corroborating evidence, via an external web search -- catching the case -where :mod:`lineageweave.entity_relationship_classification` (or any other -LLM-driven relation inference over the Knowledge Graph) names an -organization or relationship that does not actually exist, rather than -letting a hallucinated node/edge sit in the graph indistinguishable from a -verified one. - -Grounded in FEVER-style open-domain claim verification (Thorne, Vlachos, -Christodoulopoulos, & Mittal, 2018): retrieve external evidence for a -claim, then classify the claim as supported, refuted, or not-enough-info -against what was retrieved. This module implements the practical subset -that fits a same-request check -- retrieval plus a presence/absence -signal (:data:`STATUS_CORROBORATED` / :data:`STATUS_UNCORROBORATED`) -- -not full NLI-based entailment scoring against the retrieved passages; -that upgrade is a real one once real usage shows the presence/absence -signal under- or over-trusting results in practice, not implemented here -because nothing yet demonstrates the need for it over this cheaper stage. - -Same pluggable-client, never-fake-a-missing-channel discipline as every -other channel in this package: :class:`NullRelationVerificationClient` -makes the channel unavailable, never fabricates a verification result. +"""Verify LLM-inferred relations against external search evidence. + +This module implements the retrieval-and-presence subset of FEVER-style claim +verification (Thorne, Vlachos, Christodoulopoulos, & Mittal, 2018). It catches +invented organization names without claiming that a search hit proves the +specific relationship. Missing search transport remains unavailable rather +than becoming a fabricated negative result. """ from __future__ import annotations @@ -40,9 +25,8 @@ "yandex.", "searx", ) -# Distinctive name tokens only. Latin legal suffixes ("Corp", "Ltd") and -# 1-syllable Hangul particles must not corroborate a random host that -# happens to contain them. +# Distinctive name tokens only. Legal suffixes, fixture descriptors, and +# one-syllable Hangul particles cannot corroborate a random search result. _ORG_TOKEN = re.compile(r"[A-Za-z]{4,}|[가-힣]{2,}") _ORG_TOKEN_STOPWORDS = frozenset( { @@ -56,11 +40,22 @@ "group", "holdings", "limited", + "fictitious", + "nonexistent", + "placeholder", + "sample", + "example", + "demo", "foundation", "the", "and", } ) +_HANGUL_TOKEN = re.compile(r"[가-힣]+") +_KOREAN_PARTICLE_SUFFIX = re.compile( + r"(?:에게서|한테서|에서는|으로는|이라고|에서|에게|한테|께서|부터|까지|처럼|보다|만큼|" + r"으로|이랑|라고|이|가|은|는|을|를|의|에|께|와|과|도|만|로|랑|하고)+" +) STATUS_PENDING = "verify_pending" STATUS_CORROBORATED = "verify_corroborated" @@ -69,77 +64,61 @@ @dataclass(frozen=True) class RelationVerificationResult: - """One claim's verification outcome. - - Attributes: - status_code: one of ``STATUS_CORROBORATED`` / ``STATUS_UNCORROBORATED`` - -- ``common_lookup_value.lookup_code`` for category - ``relation_verification_status``. - evidence_url: the first corroborating search result's URL, or - ``None`` when uncorroborated (there is nothing to cite). - """ + """One claim's verification outcome and its optional evidence URL.""" status_code: str evidence_url: str | None class RelationVerificationClient(Protocol): - """Checks a claimed organization/relationship against external search.""" + """Check a claimed organization/relationship against external search.""" available: bool - def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: - """Search for corroborating evidence of ``organization_name`` - having the relationship ``relationship_label`` describes. + def verify( + self, organization_name: str, relationship_label: str + ) -> RelationVerificationResult: + """Return search evidence or raise when the search itself fails. - Implementations must raise if the search itself fails (network - error, non-JSON response) -- a failed search is not the same - claim as "searched and found nothing," and must not be recorded - as ``STATUS_UNCORROBORATED``. Protocol stubs raise - ``NotImplementedError`` so a no-op body is never treated as a - successful result. + A failed search is not the same claim as "searched and found nothing" + and must not be recorded as :data:`STATUS_UNCORROBORATED`. """ raise NotImplementedError class NullRelationVerificationClient: - """No search provider configured -- the verification channel is skipped.""" + """No search provider configured; the verification channel is skipped.""" available = False - def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: # pragma: no cover - """Verify whether the relationship has supporting external evidence.""" + def verify( + self, organization_name: str, relationship_label: str + ) -> RelationVerificationResult: # pragma: no cover + """Reject verification because this client has no search transport.""" raise RuntimeError( "NullRelationVerificationClient has no search channel; check .available first" ) class SearxngRelationVerificationClient: - """Queries a self-hosted Searxng instance's JSON API for corroborating - evidence of a claimed organization/relationship. - - The presence/absence signal is deliberately coarse: any search result - for "```` ````" is treated as - corroboration that the named organization has a real-world footprint - consistent with the claim, not proof the specific relationship is - true (a genuinely false relationship between two REAL organizations - would still return results about each organization separately). This - catches the failure mode actually observed from LLM classification -- - an invented organization name with zero web footprint -- rather than - claiming to adjudicate relationship truth from search snippets alone. - """ + """Query a self-hosted Searxng JSON API for corroborating evidence.""" available = True def __init__(self, base_url: str, *, timeout: float = 15.0) -> None: + """Configure a validated Searxng base URL and request timeout.""" parsed = urlparse(base_url) if parsed.scheme not in {"http", "https"}: - raise ValueError(f"unsupported Searxng base URL scheme: {parsed.scheme or 'missing'}") + raise ValueError( + f"unsupported Searxng base URL scheme: {parsed.scheme or 'missing'}" + ) self._base_url = base_url.rstrip("/") self._timeout = timeout - def verify(self, organization_name: str, relationship_label: str) -> RelationVerificationResult: - """Verify whether the relationship has supporting external evidence.""" + def verify( + self, organization_name: str, relationship_label: str + ) -> RelationVerificationResult: + """Return the first corroborating result or an explicit negative.""" query = f"{organization_name} {relationship_label}" body = get_json( f"{self._base_url}/search?q={quote(query, safe='')}&format=json", @@ -147,46 +126,85 @@ def verify(self, organization_name: str, relationship_label: str) -> RelationVer ) results = body.get("results") if not isinstance(results, list): - return RelationVerificationResult(status_code=STATUS_UNCORROBORATED, evidence_url=None) + return RelationVerificationResult( + status_code=STATUS_UNCORROBORATED, evidence_url=None + ) for result in results: if not isinstance(result, dict): continue evidence_url = corroborating_evidence_url(organization_name, result) if evidence_url is not None: - return RelationVerificationResult(status_code=STATUS_CORROBORATED, evidence_url=evidence_url) - return RelationVerificationResult(status_code=STATUS_UNCORROBORATED, evidence_url=None) - - -def corroborating_evidence_url(organization_name: str, result: dict[str, Any]) -> str | None: - """Return ``result['url']`` when it is a real-world footprint of ``organization_name``. - - Search engines echo the query in result titles, so "any hit" is not - corroboration. A single distinctive token is not enough either -- an - invented name can still contain an ordinary dictionary word (e.g. - "Fictitious", "Nonexistent") that coincidentally appears on an - unrelated page, so a genuine multi-token name requires a majority of - its tokens to co-occur in the same result; a one-token name has no - majority to require and falls back to that single token. The host - must also not itself be a search page. Missing or empty URLs are not - evidence. + return RelationVerificationResult( + status_code=STATUS_CORROBORATED, evidence_url=evidence_url + ) + return RelationVerificationResult( + status_code=STATUS_UNCORROBORATED, evidence_url=None + ) + + +def corroborating_evidence_url( + organization_name: str, result: dict[str, Any] +) -> str | None: + """Return a safe result URL when all distinctive name tokens are present. + + Search engines echo query text in titles, so only the result host and + snippet are considered. A result must contain every distinctive token; + missing, search-host, non-HTTP, and title-only URLs are not evidence. """ url = result.get("url") if not isinstance(url, str) or not url.strip(): return None - host = urlparse(url).netloc.lower() + try: + parsed_url = urlparse(url) + host = (parsed_url.hostname or "").lower() + except ValueError: + return None + if parsed_url.scheme not in {"http", "https"}: + return None if not host or any(marker in host for marker in _SEARCH_HOST_MARKERS): return None - tokens = [ - token.lower() - for token in _ORG_TOKEN.findall(organization_name) - if token.lower() not in _ORG_TOKEN_STOPWORDS + organization_tokens = [ + token.lower() for token in _ORG_TOKEN.findall(organization_name) ] + tokens = { + token for token in organization_tokens if token not in _ORG_TOKEN_STOPWORDS + } if not tokens: return None - haystack = f"{host} {result.get('content') or ''}".lower() - # Every distinctive token must occur in the same result. Matching one - # token lets generic pages about words such as "fictitious" corroborate a - # made-up multi-word organization. - if all(token in haystack for token in tokens): + haystack_tokens = { + token.lower() + for token in _ORG_TOKEN.findall(f"{host} {result.get('content') or ''}") + } + if all( + any( + _organization_token_matches(token, candidate) + for candidate in haystack_tokens + ) + for token in tokens + ) or _concatenated_hangul_name_matches(organization_tokens, haystack_tokens): return url return None + + +def _concatenated_hangul_name_matches( + expected_tokens: list[str], observed_tokens: set[str] +) -> bool: + """Accept a spaced Hangul name when a page writes its parts contiguously.""" + if len(expected_tokens) < 2 or not all( + _HANGUL_TOKEN.fullmatch(token) for token in expected_tokens + ): + return False + compact_name = "".join(expected_tokens) + return any( + _organization_token_matches(compact_name, observed) + for observed in observed_tokens + ) + + +def _organization_token_matches(expected: str, observed: str) -> bool: + """Match exact tokens or a Hangul token followed only by particles.""" + if expected == observed: + return True + if not _HANGUL_TOKEN.fullmatch(expected) or not observed.startswith(expected): + return False + return _KOREAN_PARTICLE_SUFFIX.fullmatch(observed[len(expected) :]) is not None diff --git a/lineageweave/tepp_client.py b/lineageweave/tepp_client.py index 7dbfd886f..72e996fb2 100644 --- a/lineageweave/tepp_client.py +++ b/lineageweave/tepp_client.py @@ -8,30 +8,31 @@ lineage scores as TEPP's calibrated psychometric measurement (they answer different questions -- see docs/lineage-bi-research-notes.md). -TEPP does not expose a live HTTP endpoint yet (as of this writing it is -Rust-crate-only; see ``docs/API_CONTRACT.md`` in that repo). This client -builds and validates the exact wire shape TEPP has published -(``schemas/analysis_run_request_v1.json``) so wiring in a real transport is -a one-line change (:meth:`TeppClient.__init__`'s ``transport`` argument) once -that endpoint exists, instead of a redesign. +TEPP's current protected main exposes Rust library/domain contracts and an +accepted target API contract, not a deployed HTTP service (see +``docs/API_CONTRACT.md`` in that repo). This client builds and validates the +exact wire shape TEPP has published +(``schemas/analysis_run_request_v1.json``), so an executable transport can be +added through :meth:`TeppClient.__init__` without a consumer redesign. """ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass -from typing import Any, Callable +from typing import Any class TeppNotAvailable(RuntimeError): - """Raised by the default transport: TEPP has no live REST API yet.""" + """Raised when no executable TEPP transport is configured.""" def _no_transport(request: dict[str, Any]) -> dict[str, Any]: - """Implement the _no_transport operation for this channel.""" + """Fail closed while TEPP exposes no executable transport.""" raise TeppNotAvailable( - "TEPP has no live HTTP endpoint yet (Rust-crate-only as of this writing). " - "Pass a transport= callable to TeppClient once one exists, or consume TEPP " - "as a Rust crate directly per its own docs/API_CONTRACT.md." + "No executable TEPP transport is configured. TEPP currently publishes " + "Rust library/domain contracts and an accepted target API contract; " + "configure transport= when an executable service is available." ) @@ -51,8 +52,26 @@ class AnalysisRunRequest: output_profile: str contract_version: int = 1 + def __post_init__(self) -> None: + """Reject payloads that violate TEPP's v1 schema before transport.""" + if type(self.contract_version) is not int or self.contract_version != 1: + raise ValueError("TEPP AnalysisRunRequest requires contract_version=1") + for field_name in ( + "idempotency_key", + "tenant_workspace_id", + "snapshot_id", + "knowledge_cutoff", + "model_contract_version", + "output_profile", + ): + value = getattr(self, field_name) + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"TEPP AnalysisRunRequest field {field_name} must be non-blank text" + ) + def to_json(self) -> dict[str, Any]: - """Serialize the accepted TEPP result into its wire representation.""" + """Serialize the validated request into TEPP's v1 wire representation.""" return { "contract_version": self.contract_version, "idempotency_key": self.idempotency_key, @@ -80,4 +99,9 @@ def __init__(self, transport: Callable[[dict[str, Any]], dict[str, Any]] = _no_t def submit_analysis_run(self, request: AnalysisRunRequest) -> dict[str, Any]: """Submit a request; returns TEPP's ``AnalysisRunAccepted`` envelope.""" - return self._transport(request.to_json()) + try: + return self._transport(request.to_json()) + except TeppNotAvailable: + raise + except Exception as exc: + raise TeppNotAvailable("TEPP transport request failed") from exc diff --git a/pyproject.toml b/pyproject.toml index e7d731270..3257416b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lineageweave" -version = "2.14.0" +version = "2.16.0" description = "Reconstructs git-branch-style lineage DAGs from scattered short records using multi-channel score fusion and LLM adjudication." readme = "README.md" license = { text = "MIT" } diff --git a/scripts/backfill_post_content.py b/scripts/backfill_post_content.py index 0e566129b..7be9c3ab6 100644 --- a/scripts/backfill_post_content.py +++ b/scripts/backfill_post_content.py @@ -27,7 +27,10 @@ from lineageweave.image_content import NullImageContentClient, orchestrator_vision_client from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata from lineageweave.post_content_normalization import normalize_post_body -from lineageweave.post_content_persistence import persist_post_content +from lineageweave.post_content_persistence import ( + ImageOcrPreservationError, + persist_post_content, +) from lineageweave.post_structure import ContextualOrchestratorPostStructureClient, NullPostStructureClient @@ -55,6 +58,15 @@ def _parser() -> argparse.ArgumentParser: return parser +async def _ensure_open_connection( + conn: asyncpg.Connection, target_dsn: str +) -> asyncpg.Connection: + """Reconnect after a database restart without repeating VISION work.""" + if not conn.is_closed(): + return conn + return await asyncpg.connect(target_dsn) + + async def backfill_post_content( target_dsn: str, raw_post_ids: list[str] | None, @@ -217,17 +229,22 @@ async def backfill_post_content( if described_images == 0 and not normalized.text.strip(): result["skipped_posts"] += 1 continue - await persist_post_content( - conn, - str(row["post_id"]), - row["post_body"], - vision_client=vision_client, - embedding_client=embedding_client, - embedding_model_code=embedding_model or None, - normalized_result=normalized, - structure_client=structure_client, - post_title=row["post_title"], - ) + conn = await _ensure_open_connection(conn, target_dsn) + try: + await persist_post_content( + conn, + str(row["post_id"]), + row["post_body"], + vision_client=vision_client, + embedding_client=embedding_client, + embedding_model_code=embedding_model or None, + normalized_result=normalized, + structure_client=structure_client, + post_title=row["post_title"], + ) + except ImageOcrPreservationError: + result["skipped_posts"] += 1 + continue async with conn.transaction(): await record_post_content_backfill_success( conn, diff --git a/tests/test_adjudication_client.py b/tests/test_adjudication_client.py index 576f5e9c4..f0e061f32 100644 --- a/tests/test_adjudication_client.py +++ b/tests/test_adjudication_client.py @@ -1,6 +1,73 @@ +"""Provider-response contract tests for LLM adjudication.""" + from __future__ import annotations -from lineageweave.adjudication_client import ContextualOrchestratorAdjudicationClient +import pytest + +import lineageweave.adjudication_client as module +from lineageweave.adjudication_client import ( + AdjudicationClientError, + ContextualOrchestratorAdjudicationClient, + parse_confidence_response, +) + + +@pytest.mark.parametrize("content", ["0", "0.75", "1", "1.000"]) +def test_parse_confidence_response_accepts_only_bounded_numbers(content: str) -> None: + """A compliant number-only response becomes its exact unit score.""" + + assert parse_confidence_response(content) == float(content) + + +@pytest.mark.parametrize("content", ["", "maybe 0.75", "2.0", "0.75 extra", ".5"]) +def test_parse_confidence_response_rejects_malformed_or_out_of_range_text( + content: str, +) -> None: + """Malformed provider output is not silently converted to confidence zero.""" + + with pytest.raises(AdjudicationClientError): + parse_confidence_response(content) + + +def test_parse_confidence_response_rejects_non_text_payload() -> None: + """A structured provider payload cannot masquerade as a score.""" + + with pytest.raises(AdjudicationClientError, match="not text"): + parse_confidence_response({"score": 0.5}) + + +def test_adjudication_client_rejects_malformed_provider_shape(monkeypatch) -> None: + """A provider response without one chat message fails explicitly.""" + + monkeypatch.setattr(module, "post_json", lambda *args, **kwargs: {}) + client = ContextualOrchestratorAdjudicationClient( + "https://orchestrator.invalid", + "synthetic-key", + ) + + with pytest.raises(AdjudicationClientError, match="one chat message"): + client.judge("Parent", "Child") + + +def test_adjudication_client_rejects_provider_score_outside_unit_interval( + monkeypatch, +) -> None: + """A raw out-of-range score is rejected instead of being clamped.""" + + monkeypatch.setattr( + module, + "post_json", + lambda *args, **kwargs: { + "choices": [{"message": {"content": "1.2"}}] + }, + ) + client = ContextualOrchestratorAdjudicationClient( + "https://orchestrator.invalid", + "synthetic-key", + ) + + with pytest.raises(AdjudicationClientError, match="0..1"): + client.judge("Parent", "Child") def test_adjudication_uses_supported_auto_mode_and_long_local_timeout(monkeypatch) -> None: diff --git a/tests/test_backfill_post_content.py b/tests/test_backfill_post_content.py new file mode 100644 index 000000000..d5f034297 --- /dev/null +++ b/tests/test_backfill_post_content.py @@ -0,0 +1,135 @@ +"""Operator backfill connection recovery contracts.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +from scripts import backfill_post_content +from lineageweave.post_content_persistence import ImageOcrPreservationError + + +def test_reconnects_only_after_database_connection_closes(monkeypatch) -> None: + replacement_connection = object() + connected_dsns: list[str] = [] + + class Connection: + def __init__(self, closed: bool) -> None: + self._closed = closed + + def is_closed(self) -> bool: + return self._closed + + async def connect(dsn: str): + connected_dsns.append(dsn) + return replacement_connection + + monkeypatch.setattr(backfill_post_content.asyncpg, "connect", connect) + + current_connection = Connection(False) + assert ( + asyncio.run(backfill_post_content._ensure_open_connection(current_connection, "dsn")) + is current_connection + ) + assert asyncio.run(backfill_post_content._ensure_open_connection(Connection(True), "dsn")) is replacement_connection + assert connected_dsns == ["dsn"] + + +def test_backfill_skips_ocr_protected_post_and_continues(monkeypatch) -> None: + post_ids = [ + "00505695-0000-1fd1-8000-000000000001", + "00505695-0000-1fd1-8000-000000000002", + ] + + class Transaction: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback): + return False + + class Connection: + def is_closed(self) -> bool: + return False + + async def fetch(self, query, *args): + return [{"post_id": post_id} for post_id in post_ids] + + async def fetchrow(self, query, post_id): + return { + "post_id": post_id, + "post_title": "Synthetic title", + "post_body": "Synthetic body", + "author_account_id": None, + "source_process_unit_code": None, + "source_author_code": None, + "source_company_code": None, + "source_customer_code": None, + "source_project_code": None, + "source_sales_pool_code": None, + "corporate_entity_code": None, + } + + def transaction(self): + return Transaction() + + async def fetchval(self, query, *args): + return 0 + + async def close(self): + return None + + connection = Connection() + persisted: list[str] = [] + + async def connect(_dsn): + return connection + + async def persist(conn, post_id, body, **kwargs): + persisted.append(post_id) + if post_id == post_ids[0]: + raise ImageOcrPreservationError("protected") + return 1 + + async def record_success(conn, post_id, body): + return None + + class MetadataContext: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + monkeypatch.setattr(backfill_post_content.asyncpg, "connect", connect) + monkeypatch.setattr(backfill_post_content, "persist_post_content", persist) + monkeypatch.setattr( + backfill_post_content, + "record_post_content_backfill_success", + record_success, + ) + monkeypatch.setattr( + backfill_post_content, + "normalize_post_body", + lambda body, vision_client: SimpleNamespace(image_results=(), text="text"), + ) + monkeypatch.setattr( + backfill_post_content, + "build_post_llm_metadata", + lambda post_id, row: {}, + ) + monkeypatch.setattr( + backfill_post_content, + "use_llm_metadata", + lambda metadata: MetadataContext(), + ) + + result = asyncio.run( + backfill_post_content.backfill_post_content( + "dsn", post_ids, limit=None, normalize_only=True + ) + ) + + assert persisted == post_ids + assert result["processed_posts"] == 1 + assert result["skipped_posts"] == 1 diff --git a/tests/test_chunking.py b/tests/test_chunking.py index fe8792a07..b3ecc3d67 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -2,6 +2,8 @@ from lineageweave.chunking import ( ConversationTurn, + _length_to_indent_units, + _shorthand_left_value, chunk_by_conversation_turn, chunk_by_dom, chunk_by_paragraph, @@ -12,6 +14,7 @@ def test_chunk_by_paragraph_splits_on_blank_lines() -> None: + """Blank lines delimit ordered paragraph chunks.""" text = "First paragraph about budgets.\n\nSecond paragraph about logistics.\n\nThird." chunks = chunk_by_paragraph(text) @@ -25,17 +28,20 @@ def test_chunk_by_paragraph_splits_on_blank_lines() -> None: def test_chunk_by_paragraph_ignores_extra_blank_lines_and_whitespace() -> None: + """Extra blank lines and outer whitespace do not create chunks.""" text = " A. \n\n\n\n B. " chunks = chunk_by_paragraph(text) assert [c.text for c in chunks] == ["A.", "B."] def test_chunk_by_paragraph_empty_text_yields_no_chunks() -> None: + """Empty paragraph input produces no semantic units.""" assert chunk_by_paragraph("") == [] assert chunk_by_paragraph(" \n\n ") == [] def test_chunk_by_sentence_splits_on_sentence_boundaries() -> None: + """Sentence punctuation followed by a new sentence creates a boundary.""" text = "This is one sentence. This is another! Is this a third?" chunks = chunk_by_sentence(text) @@ -47,7 +53,22 @@ def test_chunk_by_sentence_splits_on_sentence_boundaries() -> None: assert all(c.unit_type == "sentence" for c in chunks) +def test_chunk_by_sentence_empty_text_yields_no_chunks() -> None: + """Whitespace alone has no sentence-level semantic unit.""" + assert chunk_by_sentence(" \n ") == [] + + +def test_indent_helpers_cover_invalid_lengths_and_css_shorthand_shapes() -> None: + """Invalid and non-positive lengths stay flat; shorthand picks the left side.""" + assert _length_to_indent_units("auto") == 0 + assert _length_to_indent_units("-8px") == 0 + assert _shorthand_left_value("") == "" + assert _shorthand_left_value("8px") == "8px" + assert _shorthand_left_value("8px 16px") == "16px" + + def test_chunk_by_dom_splits_on_block_element_boundaries() -> None: + """Sibling DOM blocks remain separate semantic units.""" html = ( "

      First block of text.

      Second block of text.

      " "" @@ -62,6 +83,7 @@ def test_chunk_by_dom_splits_on_block_element_boundaries() -> None: def test_chunk_by_dom_nested_blocks_do_not_duplicate_text() -> None: + """The innermost block owns text without duplicating its ancestor.""" html = "

      Nested paragraph text.

      " chunks = chunk_by_dom(html) @@ -72,6 +94,16 @@ def test_chunk_by_dom_nested_blocks_do_not_duplicate_text() -> None: assert chunks[0].label == "p" +def test_chunk_by_dom_flushes_parent_text_before_nested_block() -> None: + """Direct parent text remains before a later child block.""" + chunks = chunk_by_dom("
      Parent text

      Child text

      ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("div", "Parent text"), + ("p", "Child text"), + ] + + def test_chunk_by_dom_groups_table_cells_by_row_instead_of_flattening() -> None: """Live bug (2026-08-19): each used to push its own independent chunk with no row grouping, so a real table (headers + N data rows) @@ -98,13 +130,33 @@ def test_chunk_by_dom_groups_table_cells_by_row_instead_of_flattening() -> None: def test_chunk_by_dom_keeps_nested_table_cell_blocks_in_their_row() -> None: + """Nested cell blocks retain their table-row grouping.""" chunks = chunk_by_dom( "

      No.

      Company
      " ) assert [(chunk.label, chunk.text) for chunk in chunks] == [("tr", "No. | Company")] +def test_chunk_by_dom_keeps_cell_lists_inside_their_table_row() -> None: + """A list inside a cell stays readable and grouped with its row.""" + chunks = chunk_by_dom( + "" + "
      Items:
      • A
      • B
      		Owner
      " + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Items: A B | Owner") + ] + + leading_list = chunk_by_dom( + "" + "
      • A
      • B
      		Owner
      " + ) + assert [chunk.text for chunk in leading_list] == ["A B | Owner"] + + def test_chunk_by_dom_labels_markerless_footnotes() -> None: + """A leading footnote marker assigns the footnote label.""" chunks = chunk_by_dom("

      Body text

      *Tier 2: follow-up note

      ") assert [(chunk.label, chunk.text) for chunk in chunks] == [ ("p", "Body text"), @@ -112,6 +164,154 @@ def test_chunk_by_dom_labels_markerless_footnotes() -> None: ] +def test_chunk_by_dom_labels_numeric_superscript_footnotes() -> None: + """A leading numeric superscript assigns the footnote label.""" + chunks = chunk_by_dom("

      1 Source note attached to the record.

      ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("footnote", "1 Source note attached to the record."), + ] + + +def test_chunk_by_dom_labels_numeric_superscript_after_body_text() -> None: + """A numeric superscript anywhere in a paragraph marks its evidence role.""" + chunks = chunk_by_dom("

      Body claim1 source note.

      ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("footnote", "Body claim1 source note."), + ] + + +def test_chunk_by_dom_does_not_treat_non_numeric_superscript_as_footnote() -> None: + """A formula superscript remains ordinary prose.""" + chunks = chunk_by_dom("

      Formula xn remains prose.

      ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Formula xn remains prose."), + ] + + +def test_chunk_by_dom_preserves_explicit_metric_superscripts() -> None: + """A unit exponent remains searchable mathematical evidence.""" + chunks = chunk_by_dom("

      Volume: 5m3.

      ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Volume: 5m³."), + ] + + +def test_chunk_by_dom_preserves_explicit_metric_subscripts() -> None: + """A unit subscript is retained without changing ordinary footnotes.""" + chunks = chunk_by_dom("

      Index m3 is measured.

      ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("p", "Index m₃ is measured."), + ] + + +def test_chunk_by_source_body_normalizes_plain_metric_scripts() -> None: + """Plain-text metric scripts retain searchable exponent/index semantics.""" + chunks = chunk_by_source_body("Volume: 5m^3; index m_3; braced m^{2}.") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("", "Volume: 5m³; index m₃; braced m²."), + ] + + +def test_chunk_by_source_body_normalizes_metric_scripts_in_markdown_table_cells() -> None: + """Markdown table cells retain the same searchable metric semantics as prose.""" + chunks = chunk_by_source_body( + "| Metric | Index |\n| --- | --- |\n| 5m^3 | m_3 |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("tr", "Metric | Index"), + ("tr", "5m³ | m₃"), + ] + + +def test_chunk_by_dom_preserves_nested_list_order_and_depth() -> None: + """Nested list items retain source order and increasing depth.""" + chunks = chunk_by_dom( + "
      1. Parent item
        • Child item
      " + ) + + assert [chunk.text for chunk in chunks] == ["Parent item", "Child item"] + assert [chunk.indent_width for chunk in chunks] == [4, 8] + + +def test_chunk_by_dom_accepts_exporter_oi_list_container() -> None: + """The exporter-specific oi tag behaves as an ordered-list container.""" + chunks = chunk_by_dom("
    • First item
    • Second item
      • ") + + assert [chunk.text for chunk in chunks] == ["First item", "Second item"] + assert [chunk.indent_width for chunk in chunks] == [4, 4] + + +def test_chunk_by_dom_keeps_markdown_table_rows_as_searchable_units() -> None: + """Markdown rows become independently searchable row units.""" + chunks = chunk_by_dom( + "| Project | Status |\n| :--- | ---: |\n| Alpha | Ready |" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("markdown_tr", "Project | Status"), + ("markdown_tr", "Alpha | Ready"), + ] + + +def test_chunk_by_dom_preserves_escaped_markdown_pipes_and_rejects_short_delimiters() -> None: + escaped = chunk_by_dom( + "| Field | Notes |\n| --- | --- |\n| Owner | Ready \\| review |" + ) + assert [(chunk.label, chunk.text) for chunk in escaped] == [ + ("markdown_tr", "Field | Notes"), + ("markdown_tr", "Owner | Ready | review"), + ] + + short_delimiter = chunk_by_dom( + "| Field | Value |\n| -- | -- |\n| Owner | Buyer |" + ) + assert all(chunk.label != "markdown_tr" for chunk in short_delimiter) + + +def test_chunk_by_dom_keeps_prose_around_markdown_table_rows() -> None: + """Prose surrounding a Markdown table stays in document order.""" + chunks = chunk_by_dom( + "Intro.\n\n| Project | Status |\n| --- | --- |\n| Alpha | Ready |\n\nNext action." + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("", "Intro."), + ("markdown_tr", "Project | Status"), + ("markdown_tr", "Alpha | Ready"), + ("", "Next action."), + ] + + +def test_chunk_by_dom_accepts_markdown_tables_without_outer_pipes() -> None: + """Outer pipes are optional while columns remain row-scoped evidence.""" + chunks = chunk_by_dom("Project | Status\n--- | ---\nAlpha | Ready") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("markdown_tr", "Project | Status"), + ("markdown_tr", "Alpha | Ready"), + ] + + +def test_chunk_by_dom_keeps_non_table_text_after_a_markdown_table() -> None: + """A malformed next row ends the table and remains ordinary prose.""" + chunks = chunk_by_dom( + "Project | Status\n--- | ---\nAlpha | Ready\nNext action without cells" + ) + + assert [(chunk.label, chunk.text) for chunk in chunks] == [ + ("markdown_tr", "Project | Status"), + ("markdown_tr", "Alpha | Ready"), + ("", "Next action without cells"), + ] + + def test_chunk_by_dom_labels_html_and_word_footnote_markup() -> None: html = ( "

      Body text

      " @@ -173,6 +373,7 @@ def test_chunk_by_dom_preserves_explicit_metric_subscripts() -> None: def test_chunk_by_dom_word_table_rows_also_group_cells() -> None: + """WordprocessingML table cells group by their source row.""" html = "1Acme Corp" chunks = chunk_by_dom(html) @@ -181,6 +382,7 @@ def test_chunk_by_dom_word_table_rows_also_group_cells() -> None: def test_chunk_by_dom_keeps_indentation_as_metadata_not_embedding_text() -> None: + """Non-breaking-space indentation stays metadata, not semantic text.""" html = "

        Level one

          Level two

      " chunks = chunk_by_dom(html) @@ -190,6 +392,7 @@ def test_chunk_by_dom_keeps_indentation_as_metadata_not_embedding_text() -> None def test_chunk_by_dom_reads_html_and_word_indentation_declarations() -> None: + """HTML and Word indentation declarations map to comparable units.""" html = ( '

      HTML

      ' '' @@ -202,6 +405,15 @@ def test_chunk_by_dom_reads_html_and_word_indentation_declarations() -> None: assert [chunk.declared_indent_width for chunk in chunks] == [4, 4] +def test_chunk_by_dom_ignores_invalid_word_indentation() -> None: + """Malformed Word indentation metadata cannot create a false hierarchy.""" + chunks = chunk_by_dom( + 'Word' + ) + + assert [(chunk.text, chunk.indent_width) for chunk in chunks] == [("Word", 0)] + + def test_chunk_by_dom_reads_the_css_margin_shorthand_not_just_margin_left() -> None: """Live bug (2026-08-19): a real editor (Word paste, Outlook compose) declares indentation with the box-model shorthand @@ -217,11 +429,13 @@ def test_chunk_by_dom_reads_the_css_margin_shorthand_not_just_margin_left() -> N assert [chunk.text for chunk in chunks] == ["Outer item", "Nested item"] outer, nested = chunks + assert [outer.indent_width, nested.indent_width] == [7, 10] assert outer.indent_width < nested.indent_width assert outer.indent_width > 0 def test_chunk_by_dom_uses_list_container_depth_as_explicit_indentation() -> None: + """Nested list-container depth contributes explicit indentation.""" html = "
      1. Outer
        1. Nested
      " chunks = chunk_by_dom(html) @@ -231,6 +445,7 @@ def test_chunk_by_dom_uses_list_container_depth_as_explicit_indentation() -> Non def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: + """Plain authored lists and tables split into semantic source units.""" body = """1. Background continuation stays with the first item. 2. Decision @@ -250,7 +465,22 @@ def test_chunk_by_source_body_splits_plain_lists_and_markdown_tables() -> None: ] +def test_chunk_by_source_body_keeps_a_single_pipe_row_as_plain_text() -> None: + """One pipe-delimited row alone is not enough evidence of a table.""" + chunks = chunk_by_source_body("Only | one row") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("", "Only | one row")] + + +def test_chunk_by_source_body_delegates_html_to_dom_chunking() -> None: + """HTML input retains its DOM label instead of entering the plain-text splitter.""" + chunks = chunk_by_source_body("

      HTML evidence

      ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("p", "HTML evidence")] + + def test_chunk_by_dom_joins_visual_continuation_lines_but_keeps_list_items() -> None: + """Visual wraps join while authored list starts retain boundaries.""" html = ( '

      1. 배경
      ' " 1) 기존 대차는 이전이 필요함
      " @@ -269,6 +499,7 @@ def test_chunk_by_dom_joins_visual_continuation_lines_but_keeps_list_items() -> def test_normalize_semantic_text_removes_visual_hanging_indent_breaks() -> None: + """Hanging-indent line wraps normalize without flattening list items.""" text = ( "1. 배경\n\n" " 1) 기존 대차는 이전이 필요함\n" @@ -284,16 +515,19 @@ def test_normalize_semantic_text_removes_visual_hanging_indent_breaks() -> None: def test_normalize_semantic_text_preserves_blank_paragraph_boundaries() -> None: + """Blank lines continue to separate authored paragraphs.""" assert normalize_semantic_text("첫 문단\n\n둘째 문단") == "첫 문단\n\n둘째 문단" def test_normalize_semantic_text_does_not_embed_visual_indentation_markers() -> None: + """Presentation-only non-breaking spaces do not enter semantic text.""" assert normalize_semantic_text("\xa0\xa0계속되는 문장\n\xa0\xa0\xa0\xa0다음 줄") == ( "계속되는 문장 다음 줄" ) def test_chunk_by_dom_does_not_infer_marker_depth_without_source_whitespace() -> None: + """Marker shape alone cannot invent indentation depth.""" chunks = chunk_by_dom("

      1. Root
      1) Child
      - Detail

      ") assert [chunk.text for chunk in chunks] == ["1. Root", "1) Child", "- Detail"] @@ -301,11 +535,26 @@ def test_chunk_by_dom_does_not_infer_marker_depth_without_source_whitespace() -> def test_chunk_by_dom_empty_html_yields_no_chunks() -> None: + """Empty block, self-closing block, and empty input yield no chunks.""" assert chunk_by_dom("
      ") == [] + assert chunk_by_dom("

      ") == [] assert chunk_by_dom("") == [] +def test_chunk_by_dom_keeps_unscoped_superscript_as_plain_text() -> None: + """Orphan formatting remains plain text while empty markup adds nothing.""" + chunks = chunk_by_dom("1

      ") + + assert [(chunk.label, chunk.text) for chunk in chunks] == [("", "1")] + + +def test_chunk_by_dom_decodes_deeply_escaped_entities_with_a_bounded_loop() -> None: + """Nested HTML entity escapes decode without an unbounded parser loop.""" + assert [chunk.text for chunk in chunk_by_dom("

      &amp;amp;amp;

      ")] == ["&"] + + def test_chunk_by_dom_falls_back_for_inline_only_markup() -> None: + """Inline-only markup falls back to one plain-text unit.""" chunks = chunk_by_dom("First inline block.Second inline block.") assert len(chunks) == 1 @@ -314,6 +563,7 @@ def test_chunk_by_dom_falls_back_for_inline_only_markup() -> None: def test_chunk_by_dom_flushes_unclosed_block_at_end_of_document() -> None: + """EOF flushes content from an unclosed source block.""" chunks = chunk_by_dom("
      Unclosed source fragment.") assert len(chunks) == 1 @@ -322,6 +572,7 @@ def test_chunk_by_dom_flushes_unclosed_block_at_end_of_document() -> None: def test_chunk_by_dom_interleaves_images_with_text_in_document_order() -> None: + """Embedded images retain their exact position between text blocks.""" tiny_png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" html = ( f"

      Before the picture.

      " @@ -339,6 +590,7 @@ def test_chunk_by_dom_interleaves_images_with_text_in_document_order() -> None: def test_chunk_by_dom_labels_text_chunks_with_their_tag_name() -> None: + """DOM text chunks expose their source tag as the unit label.""" html = "

      A paragraph.

      " chunks = chunk_by_dom(html) @@ -348,12 +600,20 @@ def test_chunk_by_dom_labels_text_chunks_with_their_tag_name() -> None: def test_chunk_by_dom_skips_malformed_image_data() -> None: + """Malformed base64 cannot create an image chunk.""" html = '

      Text.

      ' chunks = chunk_by_dom(html) assert [c.unit_type for c in chunks] == ["dom"] +def test_chunk_by_dom_skips_images_without_embedded_base64_data() -> None: + """Missing, external, and non-base64 image sources are not embedded images.""" + html = '' + assert chunk_by_dom(html) == [] + + def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None: + """Conversation units retain their sender labels and order.""" turns = [ ConversationTurn(sender="alice@example.com", text="Can we move the meeting?"), ConversationTurn(sender="bob@example.com", text="Sure, how about Thursday?"), @@ -366,6 +626,7 @@ def test_chunk_by_conversation_turn_labels_each_chunk_with_its_sender() -> None: def test_chunk_by_conversation_turn_skips_empty_turns() -> None: + """Empty turns are removed before contiguous chunk indexing.""" turns = [ ConversationTurn(sender="alice@example.com", text="Hello."), ConversationTurn(sender="bob@example.com", text=" "), @@ -408,11 +669,13 @@ def test_chunk_by_dom_captures_style_as_separate_metadata_not_embedded_text() -> def test_chunk_by_dom_style_is_none_when_element_has_no_style_attribute() -> None: + """A missing style attribute remains distinct from an empty style value.""" chunks = chunk_by_dom("

      Plain paragraph.

      ") assert chunks[0].style is None def test_chunk_by_dom_splits_on_heading_boundaries_and_labels_the_level() -> None: + """Heading boundaries preserve their source level label.""" html = "

      Quarterly Review

      Body text follows.

      " chunks = chunk_by_dom(html) diff --git a/tests/test_contextual_orchestrator_start.py b/tests/test_contextual_orchestrator_start.py index e3618aa82..a4f6bcfad 100644 --- a/tests/test_contextual_orchestrator_start.py +++ b/tests/test_contextual_orchestrator_start.py @@ -52,6 +52,15 @@ def test_gateway_api_key_accepts_local_compatibility_alias(monkeypatch) -> None: assert module._pop_first_env("LLM_GATEWAY_API_KEY", "LLM_API_KEY") == "compatibility-key" +def test_env_file_quotes_are_not_part_of_transport_values(monkeypatch) -> None: + module = _load_start_module() + monkeypatch.setenv("LLM_GATEWAY_API_KEY", "'provider-key'") + monkeypatch.setenv("LLM_GATEWAY_API_URL", '"https://gateway.example/v1"') + + assert module._pop_first_env("LLM_GATEWAY_API_KEY") == "provider-key" + assert module._pop_first_env("LLM_GATEWAY_API_URL") == "https://gateway.example/v1" + + def test_bootstrap_registers_embedding_agent_before_deleting_secrets(monkeypatch) -> None: module = _load_start_module() captured: dict[str, object] = {} @@ -90,9 +99,11 @@ def serve() -> None: monkeypatch.setattr(module, "Path", FakePath) monkeypatch.setattr(sys, "argv", ["start.py"]) monkeypatch.setenv("LLM_GATEWAY_API_KEY", "provider-key") - monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "orchestrator-token") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "'orchestrator-token'") monkeypatch.setenv("LLM_GATEWAY_API_URL", "https://gateway.example") - monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "embedding-model") + monkeypatch.setenv("LLM_GATEWAY_EMBEDDING_MODEL", "'embedding-model'") + monkeypatch.setenv("LLM_GATEWAY_MAX_OUTPUT_TOKENS", "'2048'") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES", '"65536"') module.main() @@ -100,6 +111,9 @@ def serve() -> None: assert isinstance(argv, list) assert "--embedding-provider-url" not in argv assert "--embedding-model" not in argv + assert argv[argv.index("--auth-token") + 1] == "orchestrator-token" + assert argv[argv.index("--max-output-tokens") + 1] == "2048" + assert argv[argv.index("--max-body-bytes") + 1] == "65536" assert captured["credentials"] == [ ("NVIDIA_NIM_API_KEY", "provider-key"), ("LLM_GATEWAY_API_KEY", "provider-key"), diff --git a/tests/test_external_lineage_analysis.py b/tests/test_external_lineage_analysis.py new file mode 100644 index 000000000..937374332 --- /dev/null +++ b/tests/test_external_lineage_analysis.py @@ -0,0 +1,729 @@ +"""Execution tests for the external Naruon-facing lineage adapter.""" + +from __future__ import annotations + +import pytest + +from lineageweave.external_lineage_analysis import ( + _channel_evidence, + analyze_external_lineage, +) +from lineageweave.external_lineage_contract import ( + LineageContractError, + parse_lineage_analysis_request, + request_digest, + result_digest, +) + + +class AvailableLlm: + """Deterministic available adjudication channel for contract tests.""" + + available = True + + def judge(self, candidate_label: str, record_label: str) -> float: + """Return a high score for labels sharing their first token.""" + + return ( + 0.9 + if candidate_label.split()[0] == record_label.split()[0] + else 0.1 + ) + + +class InvalidLlm: + """Available client returning an invalid score for fail-closed coverage.""" + + available = True + + def judge(self, candidate_label: str, record_label: str) -> float: + """Return an intentionally invalid value.""" + + return 2.0 + + +class TextLlm: + """Available client returning a non-numeric score.""" + + available = True + + def judge(self, candidate_label: str, record_label: str) -> str: + """Return an intentionally malformed score.""" + + return "unknown" + + +class BrokenProviderLlm: + """Available client surfacing an unexpected raw provider failure.""" + + available = True + + def judge(self, candidate_label: str, record_label: str) -> float: + """Raise a raw provider message that must not cross the contract.""" + + raise RuntimeError("provider secret response body") + + +class CountingLlm: + """Available client recording calls for pre-provider budget tests.""" + + available = True + + def __init__(self) -> None: + """Initialize an empty call counter.""" + + self.call_count = 0 + + def judge(self, candidate_label: str, record_label: str) -> float: + """Count one call and return a bounded score.""" + + self.call_count += 1 + return 0.5 + + +def _record( + evidence_ref: str, + label: str, + occurred_at: str, + *, + available_at: str | None = None, + secondary_key: str | None = "thread:opaque", + project_ref: str | None = "project:opaque", + explicit_parent: dict[str, str] | None = None, + group_ref: str = "workspace:demo", +) -> dict[str, object]: + return { + "evidence_ref": evidence_ref, + "group_ref": group_ref, + "source_kind_code": "email", + "truth_status_code": "observed", + "label": label, + "occurred_at": occurred_at, + "available_at": available_at or occurred_at, + "secondary_key": secondary_key, + "project_ref": project_ref, + "explicit_parent": explicit_parent, + } + + +def _request( + records: list[dict[str, object]], + *, + cutoff: str | None = None, + allow_llm: bool = False, + scope: str = "email_lineage", +): + return parse_lineage_analysis_request( + { + "contract_version": "1.0.0", + "analysis_id": "analysis:integration-001", + "analysis_scope_code": scope, + "knowledge_cutoff": cutoff, + "policy": { + "candidate_window": 50, + "maximum_pair_evaluations": 1000, + "minimum_fused_score": 0.1, + "allow_llm": allow_llm, + }, + "records": records, + } + ) + + +def test_cutoff_uses_available_time_and_discloses_excluded_evidence() -> None: + request = _request( + [ + _record( + "email:early", + "Project update", + "2026-08-18T09:00:00Z", + available_at="2026-08-18T09:01:00Z", + ), + _record( + "email:late", + "Earlier event reported late", + "2026-08-17T09:00:00Z", + available_at="2026-08-20T09:00:00Z", + ), + ], + cutoff="2026-08-19T00:00:00Z", + ) + + result = analyze_external_lineage(request) + + assert result.included_evidence_refs == ("email:early",) + assert result.excluded_evidence_refs == ("email:late",) + assert result.edges == () + assert [ + (item.limitation_code, item.evidence_ref) + for item in result.limitations + ] == [ + ("evidence_after_cutoff_excluded", "email:late"), + ] + + +def test_explicit_rfc_reply_overrides_semantic_parent_and_remains_observed() -> None: + request = _request( + [ + _record( + "email:observed-parent", + "Unrelated root", + "2026-08-20T09:00:00Z", + ), + _record( + "email:semantic-parent", + "Phoenix status", + "2026-08-20T09:01:00Z", + ), + _record( + "email:child", + "Phoenix status follow-up", + "2026-08-20T09:02:00Z", + explicit_parent={ + "evidence_ref": "email:observed-parent", + "relation_code": "rfc_reply", + }, + ), + ] + ) + + result = analyze_external_lineage(request) + child_edges = [ + edge + for edge in result.edges + if edge.child_evidence_ref == "email:child" + ] + + assert len(child_edges) == 1 + assert child_edges[0].parent_evidence_ref == "email:observed-parent" + assert child_edges[0].relation_type_code == "rfc_reply" + assert child_edges[0].truth_status_code == "observed" + assert child_edges[0].channel_evidence[0].channel_code == "rfc_reply" + + +def test_inferred_edge_exposes_active_channel_weights_and_contributions() -> None: + request = _request( + [ + _record( + "email:001", + "Phoenix delivery status", + "2026-08-20T09:00:00Z", + ), + _record( + "email:002", + "Phoenix delivery status update", + "2026-08-20T09:05:00Z", + ), + ] + ) + + result = analyze_external_lineage(request) + + assert len(result.edges) == 1 + edge = result.edges[0] + assert edge.truth_status_code == "inferred" + assert edge.relation_type_code == "reconstructed_continuation" + assert {item.channel_code for item in edge.channel_evidence} == { + "temporal", + "secondary_key", + "text", + } + assert sum(item.weight for item in edge.channel_evidence) == pytest.approx( + 1.0 + ) + assert sum( + item.contribution + for item in edge.channel_evidence + ) == pytest.approx(edge.fused_score) + + +@pytest.mark.parametrize( + ("allow_llm", "client", "expected_status", "llm_present"), + [ + (False, AvailableLlm(), "not_requested", False), + (True, None, "unavailable", False), + (True, AvailableLlm(), "completed", True), + ], +) +def test_llm_policy_is_explicit_and_never_fabricates_absent_scores( + allow_llm: bool, + client, + expected_status: str, + llm_present: bool, +) -> None: + request = _request( + [ + _record( + "email:001", + "Phoenix delivery status", + "2026-08-20T09:00:00Z", + ), + _record( + "email:002", + "Phoenix delivery status update", + "2026-08-20T09:05:00Z", + ), + ], + allow_llm=allow_llm, + ) + + result = analyze_external_lineage(request, llm=client) + + assert result.llm_status_code == expected_status + channels = { + channel.channel_code + for channel in result.edges[0].channel_evidence + } + assert ("llm" in channels) is llm_present + + +def test_available_llm_is_not_claimed_when_no_inferred_pair_exists() -> None: + """A configured LLM is not reported as completed when no call was needed.""" + request = _request( + [_record("email:001", "Only record", "2026-08-20T09:00:00Z")], + allow_llm=True, + ) + + result = analyze_external_lineage(request, llm=AvailableLlm()) + + assert result.llm_status_code == "not_invoked" + assert result.edges == () + + +def test_project_projection_is_proposed_and_uses_only_included_evidence() -> None: + request = _request( + [ + _record( + "email:001", + "One", + "2026-08-20T09:00:00Z", + ), + _record( + "email:002", + "Two", + "2026-08-20T09:01:00Z", + ), + _record( + "email:003", + "Late", + "2026-08-18T09:00:00Z", + available_at="2026-08-22T09:00:00Z", + ), + ], + cutoff="2026-08-21T00:00:00Z", + scope="project_history", + ) + + result = analyze_external_lineage(request) + + assert result.project_projections[0].project_ref == "project:opaque" + assert result.project_projections[0].evidence_refs == ( + "email:001", + "email:002", + ) + assert result.project_projections[0].truth_status_code == "proposed" + + +def test_analysis_is_deterministic_for_reordered_input_and_has_digest() -> None: + records = [ + _record( + "email:001", + "Phoenix delivery status", + "2026-08-20T09:00:00Z", + ), + _record( + "email:002", + "Phoenix delivery status update", + "2026-08-20T09:05:00Z", + ), + ] + first_request = _request(records) + second_request = _request(list(reversed(records))) + + first = analyze_external_lineage(first_request) + second = analyze_external_lineage(second_request) + + assert request_digest(first_request) == request_digest(second_request) + assert first == second + assert first.result_digest.startswith("sha256:") + assert result_digest(first) == first.result_digest + + +@pytest.mark.parametrize( + ("records", "expected_code"), + [ + ( + [ + _record( + "email:child", + "Child", + "2026-08-20T09:00:00Z", + explicit_parent={ + "evidence_ref": "email:missing", + "relation_code": "rfc_reply", + }, + ) + ], + "explicit_parent_missing", + ), + ( + [ + _record( + "email:child", + "Child", + "2026-08-20T09:00:00Z", + explicit_parent={ + "evidence_ref": "email:child", + "relation_code": "rfc_reply", + }, + ) + ], + "explicit_parent_self_reference", + ), + ( + [ + _record( + "email:parent", + "Parent", + "2026-08-20T10:00:00Z", + ), + _record( + "email:child", + "Child", + "2026-08-20T09:00:00Z", + explicit_parent={ + "evidence_ref": "email:parent", + "relation_code": "rfc_reply", + }, + ), + ], + "explicit_parent_after_child", + ), + ( + [ + _record( + "email:parent", + "Parent", + "2026-08-20T09:00:00Z", + group_ref="workspace:one", + ), + _record( + "email:child", + "Child", + "2026-08-20T10:00:00Z", + group_ref="workspace:two", + explicit_parent={ + "evidence_ref": "email:parent", + "relation_code": "rfc_reply", + }, + ), + ], + "explicit_parent_group_mismatch", + ), + ], +) +def test_invalid_explicit_parent_semantics_fail_closed( + records: list[dict[str, object]], + expected_code: str, +) -> None: + request = _request(records) + + with pytest.raises(LineageContractError) as captured: + analyze_external_lineage(request) + + assert captured.value.code == expected_code + + +def test_explicit_parent_cycle_fails_closed_even_when_timestamps_tie() -> None: + request = _request( + [ + _record( + "email:one", + "One", + "2026-08-20T09:00:00Z", + explicit_parent={ + "evidence_ref": "email:two", + "relation_code": "rfc_reply", + }, + ), + _record( + "email:two", + "Two", + "2026-08-20T09:00:00Z", + explicit_parent={ + "evidence_ref": "email:one", + "relation_code": "rfc_reply", + }, + ), + ] + ) + + with pytest.raises(LineageContractError) as captured: + analyze_external_lineage(request) + + assert captured.value.code == "explicit_parent_cycle" + + +def test_cutoff_excluded_explicit_parent_creates_limitation_not_edge() -> None: + request = _request( + [ + _record( + "email:parent", + "Parent", + "2026-08-18T09:00:00Z", + available_at="2026-08-22T09:00:00Z", + ), + _record( + "email:child", + "Child", + "2026-08-20T09:00:00Z", + available_at="2026-08-20T09:01:00Z", + explicit_parent={ + "evidence_ref": "email:parent", + "relation_code": "rfc_reply", + }, + ), + ], + cutoff="2026-08-21T00:00:00Z", + ) + + result = analyze_external_lineage(request) + + assert all( + edge.relation_type_code != "rfc_reply" + for edge in result.edges + ) + assert any( + item.limitation_code == "explicit_parent_after_cutoff" + and item.evidence_ref == "email:child" + for item in result.limitations + ) + + +def test_all_evidence_after_cutoff_returns_empty_bounded_result() -> None: + request = _request( + [ + _record( + "email:late", + "Late", + "2026-08-18T09:00:00Z", + available_at="2026-08-22T09:00:00Z", + project_ref=None, + ) + ], + cutoff="2026-08-21T00:00:00Z", + ) + + result = analyze_external_lineage(request) + + assert result.included_evidence_refs == () + assert result.edges == () + assert result.project_projections == () + + +def test_invalid_llm_score_fails_closed_before_result_projection() -> None: + request = _request( + [ + _record( + "email:001", + "Phoenix one", + "2026-08-20T09:00:00Z", + ), + _record( + "email:002", + "Phoenix two", + "2026-08-20T09:01:00Z", + ), + ], + allow_llm=True, + ) + + with pytest.raises(LineageContractError) as captured: + analyze_external_lineage(request, llm=InvalidLlm()) + + assert captured.value.code == "channel_score_out_of_bounds" + + +def test_non_numeric_llm_score_fails_closed_at_the_contract_boundary() -> None: + """A provider score with the wrong type becomes a stable contract error.""" + + request = _request( + [ + _record("email:001", "Phoenix one", "2026-08-20T09:00:00Z"), + _record("email:002", "Phoenix two", "2026-08-20T09:01:00Z"), + ], + allow_llm=True, + ) + + with pytest.raises(LineageContractError) as captured: + analyze_external_lineage(request, llm=TextLlm()) + + assert captured.value.code == "channel_score_out_of_bounds" + + +def test_raw_provider_response_error_is_stable_at_the_contract_boundary() -> None: + """A raw provider failure is not exposed as an arbitrary exception.""" + + request = _request( + [ + _record("email:001", "Phoenix one", "2026-08-20T09:00:00Z"), + _record("email:002", "Phoenix two", "2026-08-20T09:01:00Z"), + ], + allow_llm=True, + ) + + with pytest.raises(LineageContractError) as captured: + analyze_external_lineage(request, llm=BrokenProviderLlm()) + + assert captured.value.code == "llm_channel_error" + assert "provider secret" not in str(captured.value) + + +def test_channel_evidence_rejects_invalid_score_before_serialization() -> None: + """Defense in depth keeps direct channel projection fail-closed.""" + + with pytest.raises(LineageContractError) as captured: + _channel_evidence({"text": 2.0}, {"text": 1.0}) + + assert captured.value.code == "channel_score_out_of_bounds" + + +def test_records_without_project_reference_are_not_projected() -> None: + request = _request( + [ + _record( + "email:001", + "No project", + "2026-08-20T09:00:00Z", + project_ref=None, + ) + ] + ) + + result = analyze_external_lineage(request) + + assert result.project_projections == () + + +def test_cutoff_excluded_explicit_parent_suppresses_alternative_inference() -> None: + request = _request( + [ + _record( + "email:alternative", + "Phoenix child", + "2026-08-20T08:00:00Z", + available_at="2026-08-20T08:01:00Z", + ), + _record( + "email:observed-parent", + "Observed parent", + "2026-08-18T09:00:00Z", + available_at="2026-08-22T09:00:00Z", + ), + _record( + "email:child", + "Phoenix child", + "2026-08-20T09:00:00Z", + available_at="2026-08-20T09:01:00Z", + explicit_parent={ + "evidence_ref": "email:observed-parent", + "relation_code": "rfc_reply", + }, + ), + ], + cutoff="2026-08-21T00:00:00Z", + ) + + result = analyze_external_lineage(request) + + assert all( + edge.child_evidence_ref != "email:child" + for edge in result.edges + ) + + +def test_project_projections_do_not_merge_across_groups() -> None: + request = _request( + [ + _record( + "email:one", + "One", + "2026-08-20T09:00:00Z", + group_ref="workspace:one", + ), + _record( + "email:two", + "Two", + "2026-08-20T09:00:00Z", + group_ref="workspace:two", + ), + ], + scope="project_history", + ) + + result = analyze_external_lineage(request) + projections = [ + (item.group_ref, item.project_ref, item.evidence_refs) + for item in result.project_projections + ] + + assert projections == [ + ("workspace:one", "project:opaque", ("email:one",)), + ("workspace:two", "project:opaque", ("email:two",)), + ] + + +def test_pair_budget_rejects_before_any_optional_llm_call() -> None: + records = [ + _record( + f"email:{index}", + f"Message {index}", + f"2026-08-20T09:0{index}:00Z", + ) + for index in range(4) + ] + payload = { + "contract_version": "1.0.0", + "analysis_id": "analysis:pair-budget", + "analysis_scope_code": "email_lineage", + "knowledge_cutoff": None, + "policy": { + "candidate_window": 50, + "maximum_pair_evaluations": 2, + "minimum_fused_score": 0.1, + "allow_llm": True, + }, + "records": records, + } + request = parse_lineage_analysis_request(payload) + client = CountingLlm() + + with pytest.raises(LineageContractError) as captured: + analyze_external_lineage(request, llm=client) + + assert captured.value.code == "pair_evaluation_budget_exceeded" + assert client.call_count == 0 + + +def test_missing_cutoff_includes_all_records() -> None: + request = _request( + [ + _record( + "email:one", + "One", + "2026-08-20T09:00:00Z", + ), + _record( + "email:two", + "Two", + "2026-08-21T09:00:00Z", + available_at="2026-09-01T09:00:00Z", + ), + ], + cutoff=None, + ) + + result = analyze_external_lineage(request) + + assert result.included_evidence_refs == ("email:one", "email:two") + assert result.excluded_evidence_refs == () diff --git a/tests/test_external_lineage_contract.py b/tests/test_external_lineage_contract.py new file mode 100644 index 000000000..db4a36275 --- /dev/null +++ b/tests/test_external_lineage_contract.py @@ -0,0 +1,718 @@ +"""Contract tests for the future Naruon-facing LineageWeave boundary.""" + +from __future__ import annotations + +import json +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from lineageweave.external_lineage_contract import ( + CONTRACT_VERSION, + ChannelEvidence, + ExplicitParent, + LineageAnalysisResult, + LineageContractError, + LineageEdgeResult, + LineageLimitation, + ProjectProjection, + parse_lineage_analysis_request, + request_digest, + result_digest, + serialize_lineage_analysis_request, + serialize_lineage_analysis_result, +) + +_ROOT = Path(__file__).resolve().parents[1] + + +def _record( + evidence_ref: str, + *, + occurred_at: str = "2026-08-20T09:00:00Z", + available_at: str = "2026-08-20T09:01:00Z", + explicit_parent: dict[str, str] | None = None, +) -> dict[str, object]: + return { + "evidence_ref": evidence_ref, + "group_ref": "workspace:demo", + "source_kind_code": "email", + "truth_status_code": "observed", + "label": f"Subject {evidence_ref}", + "occurred_at": occurred_at, + "available_at": available_at, + "secondary_key": "provider-thread:opaque", + "project_ref": "project:opaque", + "explicit_parent": explicit_parent, + } + + +def _payload() -> dict[str, object]: + return { + "contract_version": "1.0.0", + "analysis_id": "analysis:demo-001", + "analysis_scope_code": "email_lineage", + "knowledge_cutoff": "2026-08-20T18:00:00+09:00", + "policy": { + "candidate_window": 50, + "maximum_pair_evaluations": 1000, + "minimum_fused_score": 0.3, + "allow_llm": False, + }, + "records": [ + _record("email:001"), + _record( + "email:002", + occurred_at="2026-08-20T09:05:00Z", + available_at="2026-08-20T09:06:00Z", + explicit_parent={ + "evidence_ref": "email:001", + "relation_code": "rfc_reply", + }, + ), + ], + } + + +def _result_fixture() -> LineageAnalysisResult: + return LineageAnalysisResult( + contract_version=CONTRACT_VERSION, + analysis_id="analysis:fixture", + analysis_scope_code="generic_lineage", + knowledge_cutoff=None, + included_evidence_refs=("record:001",), + excluded_evidence_refs=(), + llm_status_code="not_requested", + edges=(), + project_projections=(), + limitations=(), + result_digest="", + ) + + +def test_parse_request_is_strict_immutable_and_canonicalizes_timestamps() -> None: + request = parse_lineage_analysis_request(_payload()) + + assert request.contract_version == CONTRACT_VERSION + assert request.analysis_id == "analysis:demo-001" + assert request.analysis_scope_code == "email_lineage" + assert request.knowledge_cutoff == datetime( + 2026, + 8, + 20, + 9, + 0, + tzinfo=timezone.utc, + ) + assert request.records[1].explicit_parent == ExplicitParent( + evidence_ref="email:001", + relation_code="rfc_reply", + ) + assert serialize_lineage_analysis_request(request)[ + "knowledge_cutoff" + ] == "2026-08-20T09:00:00Z" + with pytest.raises(AttributeError): + request.analysis_id = "changed" # type: ignore[misc] + + +def test_request_digest_is_stable_when_keys_and_records_are_reordered() -> None: + payload = _payload() + reordered = { + "records": list(reversed(payload["records"])), # type: ignore[arg-type] + "policy": { + "allow_llm": False, + "minimum_fused_score": 0.3, + "maximum_pair_evaluations": 1000, + "candidate_window": 50, + }, + "knowledge_cutoff": payload["knowledge_cutoff"], + "analysis_scope_code": payload["analysis_scope_code"], + "analysis_id": payload["analysis_id"], + "contract_version": payload["contract_version"], + } + + assert request_digest( + parse_lineage_analysis_request(payload) + ) == request_digest(parse_lineage_analysis_request(reordered)) + + +@pytest.mark.parametrize( + ("mutator", "expected_code"), + [ + (lambda payload: payload.update({"unexpected": True}), "unknown_field"), + ( + lambda payload: payload["policy"].update( # type: ignore[union-attr] + {"unexpected": True} + ), + "unknown_field", + ), + ( + lambda payload: payload["records"][0].update( # type: ignore[index,union-attr] + {"unexpected": True} + ), + "unknown_field", + ), + ( + lambda payload: payload.update({"contract_version": "2.0.0"}), + "unsupported_contract_version", + ), + ( + lambda payload: payload.update( + {"analysis_scope_code": "mailbox_dump"} + ), + "unknown_analysis_scope", + ), + ], +) +def test_parser_rejects_unknown_fields_and_vocabularies( + mutator, + expected_code: str, +) -> None: + payload = _payload() + mutator(payload) + + with pytest.raises(LineageContractError) as captured: + parse_lineage_analysis_request(payload) + + assert captured.value.code == expected_code + + +def test_parser_rejects_duplicate_references_and_record_count_bounds() -> None: + payload = _payload() + payload["records"] = [_record("email:001"), _record("email:001")] + with pytest.raises(LineageContractError) as duplicate: + parse_lineage_analysis_request(payload) + assert duplicate.value.code == "duplicate_evidence_ref" + + payload["records"] = [] + with pytest.raises(LineageContractError) as empty: + parse_lineage_analysis_request(payload) + assert empty.value.code == "record_count_out_of_bounds" + + payload["records"] = [ + _record(f"email:{index:03d}") + for index in range(501) + ] + with pytest.raises(LineageContractError) as oversized: + parse_lineage_analysis_request(payload) + assert oversized.value.code == "record_count_out_of_bounds" + + +@pytest.mark.parametrize( + ("field_name", "value", "expected_code"), + [ + ( + "occurred_at", + "2026-08-20T09:00:00", + "timestamp_must_be_offset_aware", + ), + ("available_at", "not-a-time", "invalid_timestamp"), + ( + "evidence_ref", + "https://mail.example/message/1", + "unsafe_opaque_reference", + ), + ("evidence_ref", "contains whitespace", "unsafe_opaque_reference"), + ("label", "", "text_length_out_of_bounds"), + ("label", "x" * 2001, "text_length_out_of_bounds"), + ], +) +def test_parser_rejects_unsafe_identifiers_timestamps_and_text( + field_name: str, + value: str, + expected_code: str, +) -> None: + payload = _payload() + payload["records"][0][field_name] = value # type: ignore[index] + + with pytest.raises(LineageContractError) as captured: + parse_lineage_analysis_request(payload) + + assert captured.value.code == expected_code + + +@pytest.mark.parametrize( + ("field_name", "value", "expected_code"), + [ + ("candidate_window", 0, "policy_value_out_of_bounds"), + ("candidate_window", 201, "policy_value_out_of_bounds"), + ("maximum_pair_evaluations", 0, "policy_value_out_of_bounds"), + ("maximum_pair_evaluations", 5_001, "policy_value_out_of_bounds"), + ("minimum_fused_score", -0.1, "policy_value_out_of_bounds"), + ("minimum_fused_score", 1.1, "policy_value_out_of_bounds"), + ("allow_llm", "yes", "invalid_field_type"), + ], +) +def test_parser_rejects_invalid_policy_values( + field_name: str, + value: object, + expected_code: str, +) -> None: + payload = _payload() + payload["policy"][field_name] = value # type: ignore[index] + + with pytest.raises(LineageContractError) as captured: + parse_lineage_analysis_request(payload) + + assert captured.value.code == expected_code + + +def test_result_serialization_is_deterministic_and_digest_is_external() -> None: + edge = LineageEdgeResult( + parent_evidence_ref="email:001", + child_evidence_ref="email:002", + relation_type_code="reconstructed_continuation", + truth_status_code="inferred", + fused_score=0.75, + channel_evidence=( + ChannelEvidence("text", 0.8, 0.5, 0.4), + ChannelEvidence("temporal", 0.7, 0.5, 0.35), + ), + ) + result = LineageAnalysisResult( + contract_version=CONTRACT_VERSION, + analysis_id="analysis:demo-001", + analysis_scope_code="email_lineage", + knowledge_cutoff=datetime( + 2026, + 8, + 20, + 9, + 0, + tzinfo=timezone.utc, + ), + included_evidence_refs=("email:001", "email:002"), + excluded_evidence_refs=(), + llm_status_code="not_requested", + edges=(edge,), + project_projections=( + ProjectProjection( + "workspace:demo", + "project:opaque", + ("email:001", "email:002"), + "proposed", + ), + ), + limitations=( + LineageLimitation("none", None, "No material limitation."), + ), + result_digest="", + ) + digest = result_digest(result) + finalized = replace(result, result_digest=digest) + + serialized = serialize_lineage_analysis_result(finalized) + assert serialized["result_digest"] == digest + assert serialized["knowledge_cutoff"] == "2026-08-20T09:00:00Z" + assert result_digest(finalized) == digest + assert json.dumps(serialized, sort_keys=True, separators=(",", ":")) + + +def test_public_schema_exists_and_mirrors_contract_vocabularies() -> None: + schema = json.loads( + ( + _ROOT + / "docs" + / "contracts" + / "external-lineage-analysis-v1.schema.json" + ).read_text(encoding="utf-8") + ) + + assert schema["$schema"] == ( + "https://json-schema.org/draft/2020-12/schema" + ) + assert schema["properties"]["contract_version"]["const"] == ( + CONTRACT_VERSION + ) + assert set( + schema["properties"]["analysis_scope_code"]["enum"] + ) == { + "email_lineage", + "project_history", + "generic_lineage", + } + assert schema["additionalProperties"] is False + assert set( + schema["$defs"]["LineageAnalysisResult"]["properties"]["llm_status_code"]["enum"] + ) == {"not_requested", "unavailable", "not_invoked", "completed"} + pair_budget = schema["$defs"]["LineageAnalysisPolicy"][ + "properties" + ]["maximum_pair_evaluations"] + assert pair_budget == { + "type": "integer", + "minimum": 1, + "maximum": 5000, + } + + +def test_parser_rejects_non_object_and_missing_required_field() -> None: + with pytest.raises(LineageContractError) as non_object: + parse_lineage_analysis_request([]) + assert non_object.value.code == "invalid_field_type" + + payload = _payload() + del payload["analysis_id"] + with pytest.raises(LineageContractError) as missing: + parse_lineage_analysis_request(payload) + assert missing.value.code == "missing_field" + + +def test_parser_rejects_wrong_scalar_types_and_non_array_records() -> None: + mutations = [ + ("contract_version", 1, "invalid_field_type"), + ("knowledge_cutoff", 1, "invalid_field_type"), + ("analysis_scope_code", 1, "invalid_field_type"), + ] + for field, value, expected in mutations: + payload = _payload() + payload[field] = value + with pytest.raises(LineageContractError) as captured: + parse_lineage_analysis_request(payload) + assert captured.value.code == expected + + payload = _payload() + payload["policy"]["minimum_fused_score"] = "0.3" # type: ignore[index] + with pytest.raises(LineageContractError) as number: + parse_lineage_analysis_request(payload) + assert number.value.code == "invalid_field_type" + + payload = _payload() + payload["policy"]["candidate_window"] = 50.0 # type: ignore[index] + with pytest.raises(LineageContractError) as integer: + parse_lineage_analysis_request(payload) + assert integer.value.code == "invalid_field_type" + + payload = _payload() + payload["records"] = tuple(payload["records"]) # type: ignore[arg-type] + with pytest.raises(LineageContractError) as records: + parse_lineage_analysis_request(payload) + assert records.value.code == "invalid_field_type" + + +def test_optional_references_may_be_omitted() -> None: + payload = _payload() + record = payload["records"][0] # type: ignore[index] + del record["secondary_key"] + del record["project_ref"] + del record["explicit_parent"] + + parsed = parse_lineage_analysis_request(payload) + + assert parsed.records[0].secondary_key is None + assert parsed.records[0].project_ref is None + assert parsed.records[0].explicit_parent is None + + +def test_result_serializer_rejects_naive_timestamp_and_invalid_scores() -> None: + result = replace( + _result_fixture(), + knowledge_cutoff=datetime(2026, 8, 20, 9, 0), + ) + with pytest.raises(LineageContractError) as naive: + serialize_lineage_analysis_result(result) + assert naive.value.code == "timestamp_must_be_offset_aware" + + invalid_type_edge = LineageEdgeResult( + "record:001", + "record:002", + "reconstructed_continuation", + "inferred", + True, # type: ignore[arg-type] + (ChannelEvidence("text", 0.5, 1.0, 0.5),), + ) + result_with_two_records = replace( + _result_fixture(), + included_evidence_refs=("record:001", "record:002"), + ) + with pytest.raises(LineageContractError) as score_type: + serialize_lineage_analysis_result( + replace( + result_with_two_records, + edges=(invalid_type_edge,), + ) + ) + assert score_type.value.code == "invalid_field_type" + + invalid_range_edge = replace(invalid_type_edge, fused_score=1.1) + with pytest.raises(LineageContractError) as score_range: + serialize_lineage_analysis_result( + replace( + result_with_two_records, + edges=(invalid_range_edge,), + ) + ) + assert score_range.value.code == "score_out_of_bounds" + + +def test_result_serializer_rejects_non_proposed_project_and_wrong_version() -> None: + project = ProjectProjection( + "workspace:one", + "project:one", + ("record:001",), + "observed", + ) # type: ignore[arg-type] + with pytest.raises(LineageContractError) as truth: + serialize_lineage_analysis_result( + replace( + _result_fixture(), + project_projections=(project,), + ) + ) + assert truth.value.code == "unknown_result_truth_status" + + with pytest.raises(LineageContractError) as version: + serialize_lineage_analysis_result( + replace(_result_fixture(), contract_version="2.0.0") + ) + assert version.value.code == "unsupported_contract_version" + + +def test_result_requires_a_valid_digest_for_transport() -> None: + with pytest.raises(LineageContractError) as captured: + serialize_lineage_analysis_result(_result_fixture()) + + assert captured.value.code == "invalid_result_digest" + + +def test_result_rejects_overlapping_or_duplicate_partitions() -> None: + overlap = replace( + _result_fixture(), + included_evidence_refs=("record:001",), + excluded_evidence_refs=("record:001",), + result_digest="sha256:" + "0" * 64, + ) + with pytest.raises(LineageContractError) as captured: + serialize_lineage_analysis_result(overlap) + assert captured.value.code == "evidence_partition_overlap" + + duplicate = replace( + _result_fixture(), + included_evidence_refs=("record:001", "record:001"), + result_digest="sha256:" + "0" * 64, + ) + with pytest.raises(LineageContractError) as duplicate_error: + serialize_lineage_analysis_result(duplicate) + assert duplicate_error.value.code == "duplicate_evidence_ref" + + +def test_result_rejects_unincluded_edge_or_project_references() -> None: + edge = LineageEdgeResult( + "record:001", + "record:missing", + "reconstructed_continuation", + "inferred", + 0.5, + (ChannelEvidence("text", 0.5, 1.0, 0.5),), + ) + result = replace( + _result_fixture(), + edges=(edge,), + result_digest="sha256:" + "0" * 64, + ) + with pytest.raises(LineageContractError) as edge_error: + serialize_lineage_analysis_result(result) + assert edge_error.value.code == "edge_reference_not_included" + + project = ProjectProjection( + "workspace:one", + "project:one", + ("record:missing",), + "proposed", + ) + result = replace( + _result_fixture(), + project_projections=(project,), + result_digest="sha256:" + "0" * 64, + ) + with pytest.raises(LineageContractError) as project_error: + serialize_lineage_analysis_result(result) + assert project_error.value.code == "project_reference_not_included" + + +def test_result_rejects_self_edges_and_channel_math_errors() -> None: + base = replace( + _result_fixture(), + included_evidence_refs=("record:001", "record:002"), + ) + self_edge = LineageEdgeResult( + "record:001", + "record:001", + "reconstructed_continuation", + "inferred", + 0.5, + (ChannelEvidence("text", 0.5, 1.0, 0.5),), + ) + with pytest.raises(LineageContractError) as self_error: + serialize_lineage_analysis_result( + replace( + base, + edges=(self_edge,), + result_digest="sha256:" + "0" * 64, + ) + ) + assert self_error.value.code == "self_lineage_edge" + + duplicate_channels = replace( + self_edge, + parent_evidence_ref="record:002", + channel_evidence=( + ChannelEvidence("text", 0.5, 0.5, 0.25), + ChannelEvidence("text", 0.5, 0.5, 0.25), + ), + ) + with pytest.raises(LineageContractError) as duplicate_error: + serialize_lineage_analysis_result( + replace( + base, + edges=(duplicate_channels,), + result_digest="sha256:" + "0" * 64, + ) + ) + assert duplicate_error.value.code == "duplicate_channel_code" + + bad_weights = replace( + duplicate_channels, + channel_evidence=( + ChannelEvidence("text", 0.5, 0.4, 0.2), + ChannelEvidence("temporal", 0.5, 0.4, 0.2), + ), + ) + with pytest.raises(LineageContractError) as weight_error: + serialize_lineage_analysis_result( + replace( + base, + edges=(bad_weights,), + result_digest="sha256:" + "0" * 64, + ) + ) + assert weight_error.value.code == "channel_weight_sum_mismatch" + + bad_contribution = replace( + bad_weights, + channel_evidence=( + ChannelEvidence("text", 0.5, 0.5, 0.2), + ChannelEvidence("temporal", 0.5, 0.5, 0.2), + ), + ) + with pytest.raises(LineageContractError) as contribution_error: + serialize_lineage_analysis_result( + replace( + base, + edges=(bad_contribution,), + result_digest="sha256:" + "0" * 64, + ) + ) + assert contribution_error.value.code == ( + "channel_contribution_mismatch" + ) + + +def test_result_rejects_unsafe_analysis_identifier() -> None: + result = replace( + _result_fixture(), + analysis_id="https://unsafe.example/run", + result_digest="sha256:" + "0" * 64, + ) + with pytest.raises(LineageContractError) as captured: + serialize_lineage_analysis_result(result) + assert captured.value.code == "unsafe_opaque_reference" + + +def test_result_rejects_missing_channels_and_contribution_mismatch() -> None: + base = replace( + _result_fixture(), + included_evidence_refs=("record:001", "record:002"), + ) + missing_channels = LineageEdgeResult( + "record:001", + "record:002", + "reconstructed_continuation", + "inferred", + 0.5, + (), + ) + with pytest.raises(LineageContractError) as missing: + serialize_lineage_analysis_result( + replace( + base, + edges=(missing_channels,), + result_digest="sha256:" + "0" * 64, + ) + ) + assert missing.value.code == "missing_channel_evidence" + + inconsistent = replace( + missing_channels, + channel_evidence=( + ChannelEvidence("text", 0.5, 0.5, 0.3), + ChannelEvidence("temporal", 0.5, 0.5, 0.2), + ), + ) + with pytest.raises(LineageContractError) as mismatch: + serialize_lineage_analysis_result( + replace( + base, + edges=(inconsistent,), + result_digest="sha256:" + "0" * 64, + ) + ) + assert mismatch.value.code == "channel_contribution_mismatch" + + +def test_result_rejects_channel_sum_that_does_not_equal_fused_score() -> None: + """The fused score must reconcile with all otherwise valid contributions.""" + + edge = LineageEdgeResult( + "record:001", + "record:002", + "reconstructed_continuation", + "inferred", + 0.5, + ( + ChannelEvidence("text", 0.2, 0.5, 0.1), + ChannelEvidence("temporal", 0.2, 0.5, 0.1), + ), + ) + with pytest.raises(LineageContractError) as captured: + serialize_lineage_analysis_result( + replace( + _result_fixture(), + included_evidence_refs=("record:001", "record:002"), + edges=(edge,), + result_digest="sha256:" + "0" * 64, + ) + ) + + assert captured.value.code == "channel_contribution_mismatch" + + +def test_result_rejects_duplicate_project_evidence_references() -> None: + project = ProjectProjection( + "workspace:one", + "project:one", + ("record:001", "record:001"), + "proposed", + ) + with pytest.raises(LineageContractError) as captured: + serialize_lineage_analysis_result( + replace( + _result_fixture(), + project_projections=(project,), + result_digest="sha256:" + "0" * 64, + ) + ) + assert captured.value.code == "duplicate_evidence_ref" + + +def test_result_rejects_digest_not_matching_canonical_content() -> None: + result = replace( + _result_fixture(), + result_digest="sha256:" + "0" * 64, + ) + + with pytest.raises(LineageContractError) as captured: + serialize_lineage_analysis_result(result) + + assert captured.value.code == "result_digest_mismatch" diff --git a/tests/test_external_lineage_explicit_parent_budget.py b/tests/test_external_lineage_explicit_parent_budget.py new file mode 100644 index 000000000..3455278f8 --- /dev/null +++ b/tests/test_external_lineage_explicit_parent_budget.py @@ -0,0 +1,157 @@ +"""Regression tests for explicit-parent budget and provider minimization.""" + +from __future__ import annotations + +from lineageweave.external_lineage_analysis import analyze_external_lineage +from lineageweave.external_lineage_contract import parse_lineage_analysis_request + + +class CountingLlm: + """Available adjudication client that records every disclosed label pair.""" + + available = True + + def __init__(self) -> None: + """Initialize an empty provider-call ledger.""" + + self.calls: list[tuple[str, str]] = [] + + def judge(self, candidate_label: str, record_label: str) -> float: + """Record one adjudication pair and return a bounded score.""" + + self.calls.append((candidate_label, record_label)) + return 0.5 + + +def _record( + evidence_ref: str, + label: str, + occurred_at: str, + *, + explicit_parent: str | None = None, +) -> dict[str, object]: + """Build one synthetic authorized email evidence record.""" + + return { + "evidence_ref": evidence_ref, + "group_ref": "workspace:synthetic", + "source_kind_code": "email", + "truth_status_code": "observed", + "label": label, + "occurred_at": occurred_at, + "available_at": occurred_at, + "secondary_key": "thread:synthetic", + "project_ref": "project:synthetic", + "explicit_parent": ( + { + "evidence_ref": explicit_parent, + "relation_code": "rfc_reply", + } + if explicit_parent is not None + else None + ), + } + + +def _request( + records: list[dict[str, object]], + *, + allow_llm: bool, + maximum_pair_evaluations: int, +): + """Parse one strict external-lineage request for the regression cases.""" + + return parse_lineage_analysis_request( + { + "contract_version": "1.0.0", + "analysis_id": "analysis:explicit-parent-budget", + "analysis_scope_code": "email_lineage", + "knowledge_cutoff": None, + "policy": { + "candidate_window": 50, + "maximum_pair_evaluations": maximum_pair_evaluations, + "minimum_fused_score": 0.1, + "allow_llm": allow_llm, + }, + "records": records, + } + ) + + +def test_explicit_parent_chain_spends_no_inference_budget_or_llm_calls() -> None: + """Caller-observed edges must not be rescored or charged as inferred work.""" + + request = _request( + [ + _record("email:one", "One", "2026-08-21T09:00:00Z"), + _record( + "email:two", + "Two", + "2026-08-21T09:01:00Z", + explicit_parent="email:one", + ), + _record( + "email:three", + "Three", + "2026-08-21T09:02:00Z", + explicit_parent="email:two", + ), + _record( + "email:four", + "Four", + "2026-08-21T09:03:00Z", + explicit_parent="email:three", + ), + ], + allow_llm=True, + maximum_pair_evaluations=1, + ) + client = CountingLlm() + + result = analyze_external_lineage(request, llm=client) + + assert client.calls == [] + assert [ + ( + edge.parent_evidence_ref, + edge.child_evidence_ref, + edge.truth_status_code, + ) + for edge in result.edges + ] == [ + ("email:one", "email:two", "observed"), + ("email:two", "email:three", "observed"), + ("email:three", "email:four", "observed"), + ] + + +def test_explicit_child_remains_available_as_a_later_inference_candidate() -> None: + """Skipping its own scoring must not remove an explicit child from history.""" + + request = _request( + [ + _record("email:root", "Root", "2026-08-21T09:00:00Z"), + _record( + "email:observed-child", + "Phoenix delivery update", + "2026-08-21T09:01:00Z", + explicit_parent="email:root", + ), + _record( + "email:later-child", + "Phoenix delivery update", + "2026-08-21T09:02:00Z", + ), + ], + allow_llm=False, + maximum_pair_evaluations=2, + ) + + result = analyze_external_lineage(request) + + assert any( + edge.parent_evidence_ref == "email:observed-child" + and edge.child_evidence_ref == "email:later-child" + and edge.truth_status_code == "inferred" + for edge in result.edges + ) diff --git a/tests/test_external_lineage_public_api.py b/tests/test_external_lineage_public_api.py new file mode 100644 index 000000000..9d45f5791 --- /dev/null +++ b/tests/test_external_lineage_public_api.py @@ -0,0 +1,16 @@ +"""Public import-surface tests for external lineage consumers.""" + +from __future__ import annotations + +import lineageweave.external_lineage as external_lineage + + +def test_external_lineage_module_exports_the_versioned_contract() -> None: + assert external_lineage.CONTRACT_VERSION == "1.0.0" + assert callable(external_lineage.parse_lineage_analysis_request) + assert callable(external_lineage.analyze_external_lineage) + assert callable(external_lineage.request_digest) + assert callable(external_lineage.result_digest) + assert external_lineage.LineageContractError.__name__ == ( + "LineageContractError" + ) diff --git a/tests/test_image_content.py b/tests/test_image_content.py index de7fc49b5..148a979c5 100644 --- a/tests/test_image_content.py +++ b/tests/test_image_content.py @@ -85,6 +85,35 @@ def test_parse_description_preserves_multiline_ocr_text() -> None: assert description.caption == "A scanned page." +def test_parse_description_preserves_multiline_caption_evidence() -> None: + """Detailed VISION captions remain complete when providers wrap lines.""" + content = ( + "CAPTION: A project status table for the customer meeting.\n" + "The left column lists workstreams and the right column lists owners.\n" + "TEXT: Workstream | Owner\nAlpha | Team A\nTAGS: table, assignment" + ) + + description = _parse_description(content) + + assert description.caption == ( + "A project status table for the customer meeting.\n" + "The left column lists workstreams and the right column lists owners." + ) + assert description.extracted_text == "Workstream | Owner\nAlpha | Team A" + + +def test_parse_description_preserves_ocr_lines_that_contain_colons() -> None: + """A colon in a scanned field is OCR content, not a new response field.""" + content = ( + "TEXT: Invoice\nDate: 2026-08-21\nTotal: 100\n" + "CAPTION: A synthetic invoice.\nTAGS: invoice" + ) + + description = _parse_description(content) + + assert description.extracted_text == "Invoice\nDate: 2026-08-21\nTotal: 100" + + def test_parse_description_preserves_table_row_structure_in_ocr_text() -> None: """Live gap (2026-08-19): an image containing a table used to have its text flattened into an unstructured word list on OCR, the same @@ -202,6 +231,14 @@ def test_orchestrator_vision_client_does_not_double_v1() -> None: assert client._base_url == "https://gateway.example/v1" +def test_orchestrator_vision_client_allows_deep_agent_runtime() -> None: + """A valid VISION result must not be cut off by the former 180s limit.""" + client = orchestrator_vision_client("https://gateway.example", "key") + + assert isinstance(client, OpenAiCompatibleVisionClient) + assert client._timeout == 600.0 + + def test_orchestrator_vision_client_is_null_when_unconfigured() -> None: client = orchestrator_vision_client("", "") assert isinstance(client, NullImageContentClient) @@ -226,6 +263,17 @@ def test_ocr_prompt_asks_for_table_row_structure() -> None: assert "table" in _RESPONSE_FORMAT.lower() +def test_ocr_prompt_allows_multiline_tables_and_requests_semantic_detail() -> None: + """Table rows and ontology-ready captions must fit the response contract.""" + prompt = _RESPONSE_FORMAT.lower() + + assert "text may span multiple lines" in prompt + assert "separator row" in prompt + assert "named entities" in prompt + assert "relationships" in prompt + assert "exactly three lines" not in prompt + + def test_region_prompt_requires_full_image_coverage() -> None: """Live gap (2026-08-19): "distinct meaningful visual regions" alone let the model describe only the most visually striking part of an diff --git a/tests/test_lineage_contract.py b/tests/test_lineage_contract.py new file mode 100644 index 000000000..5fe3a4fc6 --- /dev/null +++ b/tests/test_lineage_contract.py @@ -0,0 +1,376 @@ +"""Synthetic contract tests for the reusable Naruon provider boundary.""" + +import json +from datetime import UTC, datetime, timedelta + +import pytest + +import lineageweave.lineage_contract as contract +from lineageweave.adjudication_client import NullAdjudicationClient +from lineageweave.lineage_contract import ( + EmailEvidence, + LineageAnalysisPolicy, + LineageAnalysisRequest, + LineageEvidenceRecord, + LineageProjectHint, + analyze_lineage, +) +from lineageweave.models import Edge, Record, Tree + +BASE_TIME = datetime(2026, 1, 1, 9, tzinfo=UTC) + + +class RawProviderFailure: + """Available provider client that exposes a secret-bearing failure.""" + + available = True + + def judge(self, candidate_label: str, record_label: str) -> float: + """Raise the raw provider failure that the contract must sanitize.""" + raise RuntimeError("provider secret response body") + + +class InvalidProviderScore: + """Available provider client that returns one invalid score value.""" + + available = True + + def __init__(self, score: object) -> None: + """Store the hostile score for the contract test.""" + self.score = score + + def judge(self, candidate_label: str, record_label: str) -> object: + """Return the hostile score without raising a transport error.""" + return self.score + + +class ValidProvider: + """Available provider client that returns a bounded confidence.""" + + available = True + + def judge(self, candidate_label: str, record_label: str) -> float: + """Return a valid synthetic confidence for the happy path.""" + return 0.7 + + +def evidence( + ref: str, + *, + available_at: datetime = BASE_TIME, + occurred_at: datetime = BASE_TIME, + label: str = "HVDC design review", +) -> LineageEvidenceRecord: + """Create a bounded synthetic record without real organization data.""" + return LineageEvidenceRecord( + evidence_ref=ref, + group_key="synthetic-customer", + label=label, + occurred_at=occurred_at, + available_at=available_at, + secondary_key="project-01", + body_text="Confirm drawing revision and delivery date.", + ) + + +def request(records: tuple[LineageEvidenceRecord, ...], **kwargs) -> LineageAnalysisRequest: + """Build a valid synthetic request with explicit authorization scope.""" + values = { + "analysis_id": "analysis-001", + "authorization_scope_ref": "scope-opaque-001", + "knowledge_cutoff": BASE_TIME + timedelta(hours=1), + } + values.update(kwargs) + return LineageAnalysisRequest(evidence=records, **values) + + +def test_request_json_keeps_email_protocol_evidence_separate_and_deterministic() -> None: + """RFC/thread fields are preserved as a separate evidence object.""" + record = evidence("mail-001") + record = LineageEvidenceRecord( + **{**record.__dict__, "email": EmailEvidence(rfc_message_id="")} + ) + first = request((record,)) + + payload = json.loads(first.canonical_json()) + + assert first.request_digest() == request((record,)).request_digest() + assert payload["contract_version"] == "lineage-analysis/v1" + assert payload["evidence"][0]["email"]["rfc_message_id"] == "" + assert "truth_status" not in payload["evidence"][0]["email"] + + +def test_request_rejects_empty_and_oversized_text() -> None: + """Opaque identifiers and text fields stay bounded at the trust boundary.""" + with pytest.raises(ValueError, match="analysis_id must be a non-empty string"): + request((evidence("text-001"),), analysis_id="").validate() + with pytest.raises(ValueError, match="analysis_id exceeds"): + request((evidence("text-001"),), analysis_id="x" * 201).validate() + too_long_body = LineageEvidenceRecord(**{**evidence("text-002").__dict__, "body_text": "x" * 4_001}) + with pytest.raises(ValueError, match="body_text exceeds"): + request((too_long_body,)).validate() + too_long_secondary = LineageEvidenceRecord(**{**evidence("text-003").__dict__, "secondary_key": "x" * 201}) + with pytest.raises(ValueError, match="secondary_key exceeds"): + request((too_long_secondary,)).validate() + + +def test_request_rejects_unbound_hints_excess_records_and_policy_budget() -> None: + """Hints and work budgets cannot escape the submitted authorization scope.""" + with pytest.raises(ValueError, match="outside the request"): + request( + (evidence("hint-001"),), + project_hints=(LineageProjectHint("missing-001", "project-001", "Missing"),), + ).validate() + with pytest.raises(ValueError, match="exceeds max_evidence_records"): + request( + (evidence("count-001"), evidence("count-002")), + policy=LineageAnalysisPolicy(max_evidence_records=1), + ).validate() + with pytest.raises(ValueError, match="max_body_chars must be between"): + request((evidence("policy-001"),), policy=LineageAnalysisPolicy(max_body_chars=8_001)).validate() + + +def test_request_rejects_unbounded_nested_collections() -> None: + """Nested evidence collections cannot bypass the request work budget.""" + with pytest.raises(ValueError, match="project_hints exceeds max_project_hints"): + request( + (evidence("hint-budget-001"),), + project_hints=( + LineageProjectHint("hint-budget-001", "project-001", "one"), + LineageProjectHint("hint-budget-001", "project-002", "two"), + ), + policy=LineageAnalysisPolicy(max_project_hints=1), + ).validate() + + record = LineageEvidenceRecord( + **{ + **evidence("email-budget-001").__dict__, + "email": EmailEvidence(references=("ref-001", "ref-002")), + } + ) + with pytest.raises(ValueError, match="evidence.email.references exceeds the 1-item limit"): + request((record,), policy=LineageAnalysisPolicy(max_email_refs_per_record=1)).validate() + + +def test_policy_rejects_nested_collection_limits_above_contract_ceiling() -> None: + """Policy fields cannot raise the nested collection ceilings at runtime.""" + with pytest.raises(ValueError, match="max_project_hints must be between"): + request((evidence("hint-ceiling-001"),), policy=LineageAnalysisPolicy(max_project_hints=501)).validate() + with pytest.raises(ValueError, match="max_email_refs_per_record must be between"): + request( + (evidence("email-ceiling-001"),), + policy=LineageAnalysisPolicy(max_email_refs_per_record=65), + ).validate() + + +def test_request_validates_email_collections_and_policy_lower_bound() -> None: + """Participant, attachment, and lower-bound policy values remain valid inputs.""" + record = evidence("email-collections") + record = LineageEvidenceRecord( + **{ + **record.__dict__, + "body_text": "", + "email": EmailEvidence( + references=("",), + participant_refs=("participant-001",), + attachment_refs=("attachment-001",), + ), + } + ) + request((record,), policy=LineageAnalysisPolicy(max_body_chars=0)).validate() + + +def test_analyze_lineage_excludes_late_evidence_and_exposes_unavailable_llm() -> None: + """Later-available evidence cannot influence a cutoff-bound result.""" + late = evidence("late-001", available_at=BASE_TIME + timedelta(hours=2)) + result = analyze_lineage(request((evidence("early-001"), late))) + + serialized = result.to_json() + + assert "late-001" not in serialized + assert any(item.code == "evidence_after_cutoff_excluded" for item in result.limitations) + assert any(item.code == "llm_channel_unavailable" for item in result.limitations) + assert all( + edge.parent_evidence_ref != "late-001" and edge.child_evidence_ref != "late-001" + for edge in result.edges + ) + + +def test_analyze_lineage_reports_an_explicitly_unavailable_llm() -> None: + """An explicit unavailable client follows the same fail-closed path as no client.""" + result = analyze_lineage( + request((evidence("explicit-null-llm-001"),)), + llm=NullAdjudicationClient(), + ) + + assert any(item.code == "llm_channel_unavailable" for item in result.limitations) + + +def test_analyze_lineage_drops_a_failed_provider_channel_without_raw_error() -> None: + """A provider failure degrades to non-LLM channels without leaking text.""" + result = analyze_lineage( + request( + ( + evidence("provider-failure-001"), + evidence("provider-failure-002", occurred_at=BASE_TIME + timedelta(minutes=1)), + ) + ), + llm=RawProviderFailure(), + ) + + assert any(item.code == "llm_channel_unavailable" for item in result.limitations) + assert all( + all(channel.channel_code != "llm" for channel in edge.channel_evidence) + for edge in result.edges + ) + assert "provider secret" not in result.to_json() + + +@pytest.mark.parametrize("score", [True, "not-a-score", float("nan"), -0.1, 1.1]) +def test_analyze_lineage_drops_invalid_provider_scores(score: object) -> None: + """Boolean, non-finite, and out-of-range scores cannot enter fusion.""" + result = analyze_lineage( + request( + ( + evidence("invalid-score-001"), + evidence("invalid-score-002", occurred_at=BASE_TIME + timedelta(minutes=1)), + ) + ), + llm=InvalidProviderScore(score), + ) + + assert any(item.code == "llm_channel_unavailable" for item in result.limitations) + assert all( + all(channel.channel_code != "llm" for channel in edge.channel_evidence) + for edge in result.edges + ) + + +def test_analyze_lineage_does_not_flatten_the_body_into_short_record_labels(monkeypatch) -> None: + """Large bodies stay bounded evidence and do not inflate pairwise text scoring.""" + captured: list[Record] = [] + + def fake_reconstruct(records: list[Record], *, llm) -> list[Tree]: + captured.extend(records) + return [] + + monkeypatch.setattr(contract, "reconstruct", fake_reconstruct) + long_body = "x" * 4_000 + record = LineageEvidenceRecord(**{**evidence("body-bound-001").__dict__, "body_text": long_body}) + + analyze_lineage(request((record,))) + + assert captured[0].label == "HVDC design review" + + +def test_analyze_lineage_accepts_empty_evidence_without_provider_work() -> None: + """An empty authorized evidence set returns no trees or inferred edges.""" + result = analyze_lineage(request(()), llm=RawProviderFailure()) + + assert result.edges == () + + +def test_analyze_lineage_accepts_an_available_llm_without_candidates() -> None: + """An available client is not marked unavailable when no judgment is needed.""" + class AvailableClient: + """Minimal available client for the one-record no-candidate path.""" + + available = True + + result = analyze_lineage( + request((evidence("available-llm-001"),)), + llm=AvailableClient(), + ) + + assert not any(item.code == "llm_channel_unavailable" for item in result.limitations) + + +def test_analyze_lineage_keeps_a_valid_provider_score() -> None: + """A bounded provider score remains available to the fused edge.""" + result = analyze_lineage( + request( + ( + evidence("valid-score-001"), + evidence("valid-score-002", occurred_at=BASE_TIME + timedelta(minutes=1)), + ) + ), + llm=ValidProvider(), + ) + + assert result.edges + assert any(channel.channel_code == "llm" for channel in result.edges[0].channel_evidence) + + +def test_analyze_lineage_excludes_events_that_occur_after_the_cutoff() -> None: + """An early import cannot make a future event visible in a past analysis.""" + future_event = evidence( + "future-event-001", + occurred_at=BASE_TIME + timedelta(hours=2), + available_at=BASE_TIME, + ) + + result = analyze_lineage(request((evidence("early-001"), future_event))) + + serialized = result.to_json() + assert "future-event-001" not in serialized + assert any(item.code == "evidence_after_cutoff_excluded" for item in result.limitations) + + +def test_analyze_lineage_maps_edges_to_opaque_refs_and_project_hints_stay_non_authoritative() -> None: + """The existing reconstruction is reused without creating project authority.""" + records = (evidence("source-001"), evidence("source-002", occurred_at=BASE_TIME + timedelta(hours=1))) + result = analyze_lineage( + request( + records, + project_hints=(LineageProjectHint("source-001", "project-opaque", "Design review"),), + ) + ) + + assert all( + edge.parent_evidence_ref in {"source-001", "source-002"} + and edge.child_evidence_ref in {"source-001", "source-002"} + for edge in result.edges + ) + assert result.edges + assert json.loads(result.to_json())["edges"] + assert any(item.code == "project_hints_are_non_authoritative" for item in result.limitations) + + +@pytest.mark.parametrize( + ("mutator", "message"), + [ + (lambda records: (records[0], records[0]), "evidence_ref values must be unique"), + (lambda records: (LineageEvidenceRecord(**{**records[0].__dict__, "available_at": BASE_TIME.replace(tzinfo=None)}),), "timezone-aware"), + ], +) +def test_request_rejects_duplicate_or_ambiguous_evidence(mutator, message: str) -> None: + """Trust-boundary validation rejects identity collisions and naive clocks.""" + records = (evidence("duplicate-001"), evidence("duplicate-002")) + with pytest.raises(ValueError, match=message): + request(mutator(records)).validate() + + +def test_policy_rejects_unbounded_record_budget() -> None: + """A provider request cannot silently opt into unbounded work.""" + with pytest.raises(ValueError, match="between 1 and 5000"): + request((evidence("bounded-001"),), policy=LineageAnalysisPolicy(max_evidence_records=0)).validate() + + +def test_analyze_lineage_drops_an_unexpected_edge_reference(monkeypatch) -> None: + """The response boundary cannot leak an edge outside cutoff-eligible refs.""" + def fake_reconstruct(records: list[Record], *, llm) -> list[Tree]: + """Return a deliberately malformed internal edge for boundary testing.""" + return [ + Tree( + group_key=records[0].group_key, + records={record.record_id: record for record in records}, + edges=[Edge("late-001", records[0].record_id, 0.9, {"text": 0.9})], + roots=[], + children_of={}, + ) + ] + + monkeypatch.setattr(contract, "reconstruct", fake_reconstruct) + result = analyze_lineage(request((evidence("early-001"),))) + + assert result.edges == () diff --git a/tests/test_ontology_interoperability.py b/tests/test_ontology_interoperability.py new file mode 100644 index 000000000..6fe62a65f --- /dev/null +++ b/tests/test_ontology_interoperability.py @@ -0,0 +1,94 @@ +"""Interoperability contracts for the core LineageWeave ontology and shapes.""" + +from pathlib import Path + +from rdflib import Graph, Namespace, URIRef +from rdflib.namespace import OWL, RDF, RDFS, SKOS + +ONTOLOGY = Path(__file__).parents[1] / "docs" / "ontology" / "lineageweave-kg.ttl" +SHAPES = ( + Path(__file__).parents[1] + / "docs" + / "ontology" + / "lineageweave-kg.shacl.ttl" +) +LW = Namespace("https://contextualwisdomlab.github.io/lineageweave/ontology#") +LWS = Namespace("https://contextualwisdomlab.github.io/lineageweave/ontology/shapes#") +ORG = Namespace("http://www.w3.org/ns/org#") +SH = Namespace("http://www.w3.org/ns/shacl#") + + +def load_ontology_graph() -> Graph: + """Parse the committed core ontology into a fresh graph.""" + graph = Graph() + graph.parse(ONTOLOGY, format="turtle") + return graph + + +def load_shapes_graph() -> Graph: + """Parse the committed SHACL graph after asserting it is published.""" + assert SHAPES.exists(), "the core ontology must publish a SHACL shapes graph" + graph = Graph() + graph.parse(SHAPES, format="turtle") + return graph + + +def property_shape(graph: Graph, node_shape: URIRef, path: URIRef) -> URIRef: + """Return the property shape for one required path.""" + return next( + candidate + for candidate in graph.objects(node_shape, SH.property) + if graph.value(candidate, SH.path) == path + ) + + +def test_real_organizations_are_not_skos_classification_concepts() -> None: + """Corporate entities use W3C ORG while SKOS owns only level concepts.""" + graph = load_ontology_graph() + assert (LW.CorporateEntity, RDFS.subClassOf, ORG.Organization) in graph + assert (LW.CorporateEntity, RDFS.subClassOf, SKOS.Concept) not in graph + assert (LW.CorporateEntityLevel, RDFS.subClassOf, SKOS.Concept) in graph + assert (LW.GroupLevel, RDF.type, LW.CorporateEntityLevel) in graph + assert (LW.CompanyLevel, RDF.type, LW.CorporateEntityLevel) in graph + assert (LW.PlantLevel, RDF.type, LW.CorporateEntityLevel) in graph + assert (LW.hasEntityLevel, RDFS.domain, LW.CorporateEntity) in graph + assert (LW.hasEntityLevel, RDFS.range, LW.CorporateEntityLevel) in graph + + +def test_organizational_containment_reuses_w3c_org_relations() -> None: + """Parent corporations and teams specialize the correct ORG relations.""" + graph = load_ontology_graph() + assert (LW.subOrganizationOf, RDFS.subPropertyOf, ORG.subOrganizationOf) in graph + assert (LW.hasSubOrganization, OWL.inverseOf, LW.subOrganizationOf) in graph + assert (LW.teamAffiliatedWith, RDFS.subPropertyOf, ORG.unitOf) in graph + + +def test_core_ontology_has_stable_version_and_import_metadata() -> None: + """Consumers can bind the exact profile without network dereferencing.""" + graph = load_ontology_graph() + ontology = URIRef("https://contextualwisdomlab.github.io/lineageweave/ontology") + assert graph.value(ontology, OWL.versionIRI) == URIRef( + "https://contextualwisdomlab.github.io/lineageweave/ontology/1.0.0" + ) + assert str(graph.value(ontology, OWL.versionInfo)) == "1.0.0" + for imported_iri in ( + URIRef("http://www.w3.org/ns/org"), + URIRef("http://www.w3.org/ns/prov-o"), + URIRef("http://www.w3.org/2004/02/skos/core"), + ): + assert (ontology, OWL.imports, imported_iri) in graph + + +def test_core_shapes_enforce_level_and_parent_boundaries() -> None: + """SHACL cardinalities complement the ontology's open-world semantics.""" + graph = load_shapes_graph() + assert (LWS.CorporateEntityShape, SH.targetClass, LW.CorporateEntity) in graph + assert (LWS.TeamShape, SH.targetClass, LW.Team) in graph + entity_level = property_shape(graph, LWS.CorporateEntityShape, LW.hasEntityLevel) + assert int(graph.value(entity_level, SH.minCount)) == 1 + assert int(graph.value(entity_level, SH.maxCount)) == 1 + parent = property_shape(graph, LWS.CorporateEntityShape, LW.subOrganizationOf) + assert int(graph.value(parent, SH.maxCount)) == 1 + team_owner = property_shape(graph, LWS.TeamShape, LW.teamAffiliatedWith) + assert int(graph.value(team_owner, SH.minCount)) == 1 + assert int(graph.value(team_owner, SH.maxCount)) == 1 diff --git a/tests/test_person_mention_projection.py b/tests/test_person_mention_projection.py index 03d6fcf82..1d1b96ed4 100644 --- a/tests/test_person_mention_projection.py +++ b/tests/test_person_mention_projection.py @@ -103,6 +103,9 @@ _SOURCE_ORG_NAMED_HINTS_MIGRATION = ( Path(__file__).resolve().parents[1] / "migrations" / "0039_source_org_named_hints.sql" ) +_MAJOR_EVENT_ACTION_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0100_major_event_action.sql" +) def _postgres_available() -> bool: diff --git a/tests/test_post_content_normalization.py b/tests/test_post_content_normalization.py index 18e046b8b..0007df6c9 100644 --- a/tests/test_post_content_normalization.py +++ b/tests/test_post_content_normalization.py @@ -122,6 +122,20 @@ def test_plain_text_visual_continuation_breaks_are_normalized_for_embeddings() - assert result.text == "- 요청 사항 후속 설명은 같은 항목에 속한다.\n· 다음 항목" +def test_markdown_table_rows_remain_separate_in_normalized_evidence() -> None: + result = normalize_post_body( + "| Project | Status |\n| --- | --- |\n| Alpha | Ready |" + ) + + assert result.text == "Project | Status\n\nAlpha | Ready" + + +def test_exporter_oi_lists_are_normalized_as_html_evidence() -> None: + result = normalize_post_body("
    • Parent
      • Child
      • ") + + assert result.text == "Parent\n\nChild" + + def test_html_tags_never_appear_in_the_normalized_text() -> None: html = '

      Confirm delivery by Friday.

      ' result = normalize_post_body(html) diff --git a/tests/test_post_content_persistence_edges.py b/tests/test_post_content_persistence_edges.py index 24bec9e6d..4eac1b77e 100644 --- a/tests/test_post_content_persistence_edges.py +++ b/tests/test_post_content_persistence_edges.py @@ -2,6 +2,7 @@ import asyncio from contextlib import asynccontextmanager +import hashlib from types import SimpleNamespace import pytest @@ -27,9 +28,10 @@ def _persist(*args: object, **kwargs: object) -> int: class _Connection: - def __init__(self) -> None: + def __init__(self, previous_images: tuple[dict[str, object], ...] = ()) -> None: self.executed: list[tuple[str, tuple[object, ...]]] = [] self.fetchvals: list[tuple[str, tuple[object, ...]]] = [] + self.previous_images = previous_images self._next_id = 0 @asynccontextmanager @@ -45,6 +47,10 @@ async def fetchval(self, query: str, *args: object) -> str: self._next_id += 1 return f"id-{self._next_id}" + async def fetch(self, query: str, *args: object) -> list[dict[str, object]]: + assert "post_content_image" in query + return list(self.previous_images) + class _EmbedMany: available = True @@ -190,7 +196,18 @@ def test_persists_image_tags_formatting_and_embeddings() -> None: ), ), ) - conn = _Connection() + conn = _Connection( + ( + { + "content_sha256": hashlib.sha256(b"hello").hexdigest(), + "extracted_text": "previous OCR", + }, + { + "content_sha256": hashlib.sha256(b"replaced image").hexdigest(), + "extracted_text": "removed image OCR", + }, + ) + ) embedder = _EmbedMany() count = _persist( @@ -213,6 +230,37 @@ def test_persists_image_tags_formatting_and_embeddings() -> None: assert sum("post_content_embedding_value" in query for query, _args in conn.executed) == 2 * len(chunks) +def test_same_image_retry_cannot_replace_existing_ocr_with_empty_text() -> None: + body = '' + image_index = chunk_by_dom(body)[0].index + normalized = NormalizedPostContent( + text="[image: updated caption]", + image_results=( + ImageContentResult( + image_index, + "image/png", + "described", + SimpleNamespace(caption="updated caption", extracted_text="", tags=()), + ), + ), + ) + conn = _Connection( + ( + { + "content_sha256": hashlib.sha256(b"hello").hexdigest(), + "description_status_code": "described", + "extracted_text": "prior OCR", + "caption": "prior caption", + }, + ) + ) + + with pytest.raises(RuntimeError, match="refusing to replace non-empty image OCR"): + _persist(conn, "post-4", body, normalized_result=normalized) + + assert not any("delete from post_content_unit" in query for query, _args in conn.executed) + + def test_legacy_embed_and_malformed_vectors_never_write_vectors() -> None: conn = _Connection() legacy = _LegacyEmbed([float("nan")]) diff --git a/tests/test_reconstruct.py b/tests/test_reconstruct.py index 6ae195941..2aa40baa7 100644 --- a/tests/test_reconstruct.py +++ b/tests/test_reconstruct.py @@ -3,6 +3,7 @@ from datetime import datetime from lineageweave import Record, reconstruct +from lineageweave.adjudication_client import AdjudicationClientError from lineageweave.fixtures import sample_records @@ -65,6 +66,25 @@ def test_llm_channel_is_used_and_scored_when_a_client_is_supplied() -> None: assert all("llm" in edge.channel_scores for edge in tree_a.edges) +class _MalformedAdjudicationClient: + """Available client that returns an unusable provider response.""" + + available = True + + def judge(self, candidate_label: str, record_label: str) -> float: + """Raise the typed provider-response error for one candidate pair.""" + raise AdjudicationClientError("synthetic malformed confidence") + + +def test_malformed_llm_confidence_degrades_one_pair_without_aborting_reconstruction() -> None: + """Optional LLM parsing failure must not discard deterministic lineage.""" + trees = reconstruct(sample_records(), llm=_MalformedAdjudicationClient()) + tree_a = next(tree for tree in trees if tree.group_key == "A-100") + + assert tree_a.edges + assert all(edge.channel_scores["llm"] == 0.0 for edge in tree_a.edges) + + def test_candidate_window_bounds_which_priors_are_considered() -> None: records = [ Record(f"r{i}", "G", f"record {i}", datetime(2026, 1, 1, i), "") for i in range(5) diff --git a/tests/test_relation_verification.py b/tests/test_relation_verification.py index f515d5d25..e68ef81c9 100644 --- a/tests/test_relation_verification.py +++ b/tests/test_relation_verification.py @@ -1,12 +1,4 @@ -"""Tests for lineageweave.relation_verification. - -SearxngRelationVerificationClient's tests run against a real local HTTP -server (same pattern as tests/test_http_client.py), not a mocked -transport -- proving the actual URL construction and response-shape -handling, not just that a mock was called correctly. A real Searxng -instance is exercised separately, gated behind docker compose (this -repo's discipline: never fake a channel it can instead genuinely run). -""" +"""Tests for the real HTTP and evidence boundaries of relation verification.""" from __future__ import annotations @@ -21,19 +13,49 @@ STATUS_CORROBORATED, STATUS_UNCORROBORATED, NullRelationVerificationClient, + RelationVerificationClient, SearxngRelationVerificationClient, corroborating_evidence_url, ) class _ResultsHandler(BaseHTTPRequestHandler): + """Serve deterministic search responses over the real HTTP boundary.""" + received_query: str = "" - def do_GET(self) -> None: # noqa: N802 -- BaseHTTPRequestHandler API + def do_GET(self) -> None: + """Return the response shape selected by the received search query.""" parsed = urlparse(self.path) query = parse_qs(parsed.query) type(self).received_query = query.get("q", [""])[0] - if "Acme" in type(self).received_query: + if "Malformed" in type(self).received_query: + payload = { + "query": type(self).received_query, + "results": {"unexpected": True}, + } + elif "Mixed" in type(self).received_query: + payload = { + "query": type(self).received_query, + "results": [ + None, + {"url": "https://other.example/item", "content": "unrelated"}, + { + "url": "https://mixed.example/item", + "content": "Mixed Signal", + }, + ], + } + elif "NoList" in type(self).received_query: + payload = {"query": type(self).received_query, "results": {}} + elif "Skip" in type(self).received_query: + payload = {"query": type(self).received_query, "results": [None]} + elif "NoEvidence" in type(self).received_query: + payload = { + "query": type(self).received_query, + "results": [{"url": "https://example.com/item", "content": ""}], + } + elif "Acme" in type(self).received_query: payload = { "query": type(self).received_query, "results": [ @@ -53,11 +75,13 @@ def do_GET(self) -> None: # noqa: N802 -- BaseHTTPRequestHandler API self.end_headers() self.wfile.write(body) - def log_message(self, format: str, *args) -> None: # noqa: A002 -- stdlib signature + def log_message(self, format: str, *args: object) -> None: + """Keep the test server quiet while preserving the stdlib signature.""" return def _serve() -> tuple[HTTPServer, str]: + """Start an ephemeral local HTTP server and return its base URL.""" server = HTTPServer(("127.0.0.1", 0), _ResultsHandler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -66,17 +90,26 @@ def _serve() -> tuple[HTTPServer, str]: def test_null_client_is_unavailable_not_silently_uncorroborated() -> None: + """A missing search channel is unavailable, not a negative finding.""" client = NullRelationVerificationClient() assert client.available is False with pytest.raises(RuntimeError): client.verify("Acme Corp", "Voice of Customer") +def test_protocol_stub_cannot_be_mistaken_for_a_verification() -> None: + """The protocol's runtime stub fails instead of fabricating a result.""" + with pytest.raises(NotImplementedError): + RelationVerificationClient.verify(object(), "Acme Corp", "Voice of Customer") + + def test_searxng_client_reports_corroborated_with_evidence_url() -> None: + """A matching result returns corroboration and its evidence URL.""" server, base = _serve() try: - client = SearxngRelationVerificationClient(base_url=base) - result = client.verify("Acme Corp", "Voice of Customer") + result = SearxngRelationVerificationClient(base_url=base).verify( + "Acme Corp", "Voice of Customer" + ) finally: server.shutdown() @@ -87,10 +120,12 @@ def test_searxng_client_reports_corroborated_with_evidence_url() -> None: def test_searxng_client_reports_uncorroborated_with_no_evidence_url_when_search_is_empty() -> None: + """An empty result set remains explicitly uncorroborated.""" server, base = _serve() try: - client = SearxngRelationVerificationClient(base_url=base) - result = client.verify("Totally Fictitious Nonexistent Org", "Voice of Customer") + result = SearxngRelationVerificationClient(base_url=base).verify( + "Totally Fictitious Nonexistent Org", "Voice of Customer" + ) finally: server.shutdown() @@ -98,7 +133,53 @@ def test_searxng_client_reports_uncorroborated_with_no_evidence_url_when_search_ assert result.evidence_url is None +@pytest.mark.parametrize("organization_name", ["NoList", "Skip", "NoEvidence"]) +def test_searxng_client_fails_closed_for_unusable_results( + organization_name: str, +) -> None: + """Malformed and unciting search results remain explicitly negative.""" + server, base = _serve() + try: + result = SearxngRelationVerificationClient(base_url=base).verify( + organization_name, "Voice of Customer" + ) + finally: + server.shutdown() + + assert result.status_code == STATUS_UNCORROBORATED + assert result.evidence_url is None + + +def test_searxng_client_rejects_malformed_results_shape() -> None: + """A non-list result collection cannot become corroborating evidence.""" + server, base = _serve() + try: + result = SearxngRelationVerificationClient(base_url=base).verify( + "Malformed Organization", "Supplier" + ) + finally: + server.shutdown() + + assert result.status_code == STATUS_UNCORROBORATED + assert result.evidence_url is None + + +def test_searxng_client_skips_invalid_and_unrelated_results() -> None: + """The client scans past invalid and unrelated entries to real evidence.""" + server, base = _serve() + try: + result = SearxngRelationVerificationClient(base_url=base).verify( + "Mixed Signal", "Supplier" + ) + finally: + server.shutdown() + + assert result.status_code == STATUS_CORROBORATED + assert result.evidence_url == "https://mixed.example/item" + + def test_query_echo_on_a_search_host_is_not_corroboration() -> None: + """A search-result URL and echoed title cannot become evidence.""" assert ( corroborating_evidence_url( "Zzqxvthorp Fictitious Nonexistent Org", @@ -112,7 +193,36 @@ def test_query_echo_on_a_search_host_is_not_corroboration() -> None: ) +@pytest.mark.parametrize("url", [None, "", "relative-path"]) +def test_missing_empty_or_relative_url_is_not_evidence(url: object) -> None: + """Evidence needs a non-empty absolute URL with a real host.""" + assert corroborating_evidence_url("Acme Corp", {"url": url}) is None + + +@pytest.mark.parametrize("url", ["file://acme.example/item", "javascript://acme.example/item"]) +def test_non_http_evidence_url_is_not_accepted(url: str) -> None: + """Evidence links must be browser-safe HTTP(S) resources.""" + assert corroborating_evidence_url("Acme Corp", {"url": url}) is None + + +def test_malformed_ipv6_url_is_not_evidence() -> None: + """Malformed bracketed hosts fail closed instead of crashing verification.""" + assert corroborating_evidence_url("Acme Corp", {"url": "https://[::1/x"}) is None + + +def test_name_with_only_legal_suffixes_has_no_distinctive_token() -> None: + """Legal suffixes alone cannot identify an organization.""" + assert ( + corroborating_evidence_url( + "Corp Ltd", + {"url": "https://business.example/item", "content": "Corp Ltd"}, + ) + is None + ) + + def test_org_token_in_result_host_is_corroboration() -> None: + """A distinctive organization token in the host is evidence.""" assert ( corroborating_evidence_url( "Acme Corp", @@ -122,8 +232,109 @@ def test_org_token_in_result_host_is_corroboration() -> None: ) +def test_short_name_token_inside_another_word_is_not_corroboration() -> None: + """A search snippet must contain the organization token as a word.""" + assert ( + corroborating_evidence_url( + "Alpha Corp", + { + "url": "https://unrelated.example/news", + "title": "Alphabetical index", + "content": "An alphabetical index of sample terms.", + }, + ) + is None + ) + + +def test_partial_multi_token_name_is_not_corroboration() -> None: + """One generic token must not validate an invented multi-token name.""" + assert ( + corroborating_evidence_url( + "Fictitious Nonexistent Org", + { + "url": "https://microsoft.example/news", + "title": "Fictitious names, domains, and addresses", + "content": "This page discusses fictitious names.", + }, + ) + is None + ) + + +def test_all_distinctive_multi_token_name_parts_are_corroboration() -> None: + """All distinctive name tokens may be distributed across host and content.""" + assert ( + corroborating_evidence_url( + "Aurora Grid Power", + { + "url": "https://aurora-grid.example/news", + "title": "Aurora Grid Power", + "content": "Aurora Grid Power announced a delivery window.", + }, + ) + == "https://aurora-grid.example/news" + ) + + +def test_title_only_full_name_is_not_corroboration() -> None: + """A title echo alone is not an organization footprint.""" + assert ( + corroborating_evidence_url( + "Aurora Grid Power", + { + "url": "https://news.example/item", + "title": "Aurora Grid Power", + "content": "", + }, + ) + is None + ) + + +def test_compound_host_token_is_not_two_name_tokens() -> None: + """A compound host word must not match separate organization tokens.""" + assert ( + corroborating_evidence_url( + "Green House", + {"url": "https://greenhouse.example/news", "title": "News", "content": ""}, + ) + is None + ) + + +def test_spaced_hangul_name_matches_contiguous_page_token() -> None: + """A page may concatenate the parts of a spaced Korean name.""" + assert ( + corroborating_evidence_url( + "한빛 그리드", + { + "url": "https://news.example/item", + "title": "News", + "content": "한빛그리드가 공급 일정을 발표했다.", + }, + ) + == "https://news.example/item" + ) + + +def test_userinfo_tokens_are_not_hostname_evidence() -> None: + """URL credentials cannot corroborate an unrelated actual hostname.""" + assert ( + corroborating_evidence_url( + "Aurora Grid Power", + { + "url": "https://aurora-grid-power.example@unrelated.example/news", + "title": "News", + "content": "", + }, + ) + is None + ) + + def test_legal_suffix_alone_is_not_corroboration() -> None: - """'Corp' is in almost every corporate host; it is not evidence.""" + """A legal suffix in a host is not organization evidence.""" assert ( corroborating_evidence_url( "Acme Corp", @@ -133,20 +344,115 @@ def test_legal_suffix_alone_is_not_corroboration() -> None: ) -def test_hangul_org_name_token_is_corroboration() -> None: +def test_fixture_descriptors_do_not_corrobate_an_unrelated_search_hit() -> None: + """Synthetic-data descriptors must not stand in for organization identity.""" + assert ( + corroborating_evidence_url( + "Zzqxvthorp Fictitious Nonexistent Org", + { + "url": "https://learn.microsoft.com/writing-style", + "title": "Fictitious names and addresses", + "content": "Documentation explains fictitious and nonexistent examples.", + }, + ) + is None + ) + + +def test_missing_url_and_all_generic_tokens_are_not_evidence() -> None: + """Missing URLs and fixture-only names cannot provide evidence.""" + assert corroborating_evidence_url("Acme Corp", {"content": "Acme"}) is None + assert ( + corroborating_evidence_url( + "Fictitious Nonexistent Org", + {"url": "https://example.com/item", "content": "Fictitious"}, + ) + is None + ) + + +def test_one_common_token_is_not_multi_token_corroboration() -> None: + """An unrelated page mentioning one name token is insufficient evidence.""" + assert ( + corroborating_evidence_url( + "Aurora Grid Power", + { + "url": "https://news.example/item", + "content": "The power outage affected the region.", + }, + ) + is None + ) + + +@pytest.mark.parametrize("particle", ["가", "에서", "으로"]) +def test_hangul_org_name_with_attached_particle_is_corroboration(particle: str) -> None: + """Attached Korean particles remain bounded evidence.""" + assert ( + corroborating_evidence_url( + "한빛그리드", + { + "url": "https://news.example/item", + "title": "News", + "content": f"한빛그리드{particle} 발표했다.", + }, + ) + == "https://news.example/item" + ) + + +@pytest.mark.parametrize("particles", ["에서는", "으로는", "까지도"]) +def test_hangul_org_name_with_stacked_particles_is_corroboration( + particles: str, +) -> None: + """Stacked Korean particles after a complete name remain bounded evidence.""" assert ( corroborating_evidence_url( "한빛그리드", { "url": "https://news.example/item", "title": "News", - "content": "한빛그리드 announced a delivery window.", + "content": f"한빛그리드{particles} 발표했다.", }, ) == "https://news.example/item" ) +@pytest.mark.parametrize("particle", ["가", "이", "으로"]) +def test_latin_org_name_with_attached_korean_particle_is_corroboration( + particle: str, +) -> None: + """Latin organization tokens also accept directly attached particles.""" + assert ( + corroborating_evidence_url( + "Acme", + { + "url": "https://news.example/item", + "title": "News", + "content": f"Acme{particle} 발표했다.", + }, + ) + == "https://news.example/item" + ) + + +def test_longer_hangul_business_name_is_not_a_particle_match() -> None: + """An unrelated longer Korean name must not satisfy a shorter name token.""" + assert ( + corroborating_evidence_url( + "한빛그리드", + { + "url": "https://news.example/item", + "title": "News", + "content": "한빛그리드솔루션이 공급 일정을 발표했다.", + }, + ) + is None + ) + + def test_searxng_client_refuses_non_http_scheme() -> None: + """The client rejects non-HTTP URLs before making a request.""" with pytest.raises(ValueError, match="unsupported Searxng base URL scheme"): SearxngRelationVerificationClient(base_url="file:///etc/passwd") diff --git a/tests/test_tepp_client.py b/tests/test_tepp_client.py index ea87d5558..aa6b7c433 100644 --- a/tests/test_tepp_client.py +++ b/tests/test_tepp_client.py @@ -1,5 +1,7 @@ from __future__ import annotations +from dataclasses import replace + import pytest from backend.app.analysis_run_start import configured_tepp_client @@ -31,6 +33,35 @@ def test_to_json_matches_tepp_published_schema_shape() -> None: } +@pytest.mark.parametrize( + ("field_name", "value"), + [ + ("idempotency_key", " "), + ("tenant_workspace_id", None), + ("snapshot_id", "\t"), + ("knowledge_cutoff", ""), + ("model_contract_version", "\n"), + ("output_profile", None), + ], +) +def test_request_rejects_non_blank_schema_fields(field_name: str, value: object) -> None: + """A v1 request must not send blank or non-text required fields.""" + with pytest.raises(ValueError, match=field_name): + replace(_sample_request(), **{field_name: value}) + + +def test_request_rejects_unknown_contract_version() -> None: + """The adapter must not silently emit a request for another contract.""" + with pytest.raises(ValueError, match="contract_version=1"): + replace(_sample_request(), contract_version=2) + + +def test_request_rejects_boolean_contract_version() -> None: + """JSON booleans must not pass Python's integer type relationship.""" + with pytest.raises(ValueError, match="contract_version=1"): + replace(_sample_request(), contract_version=True) + + def test_default_transport_fails_closed_until_tepp_ships_http() -> None: client = TeppClient() with pytest.raises(TeppNotAvailable): @@ -52,6 +83,18 @@ def fake_transport(payload: dict) -> dict: assert received["snapshot_id"] == "demo-snapshot-1" +def test_custom_transport_provider_errors_are_not_exposed() -> None: + """Provider response text stays behind the stable unavailable error.""" + + def broken_transport(_payload: dict) -> dict: + raise RuntimeError("provider secret response body") + + with pytest.raises(TeppNotAvailable, match="transport request failed") as error: + TeppClient(transport=broken_transport).submit_analysis_run(_sample_request()) + + assert "provider secret" not in str(error.value) + + def test_configured_transport_sends_optional_bearer_key(monkeypatch: pytest.MonkeyPatch) -> None: received = {} @@ -66,3 +109,22 @@ def fake_post_json(url: str, payload: dict, *, headers: dict, timeout: float) -> assert received["headers"] == {"authorization": "Bearer test-key"} assert received["payload"] == _sample_request().to_json() + + +def test_configured_transport_provider_errors_are_not_exposed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The configured provider boundary does not return raw transport text.""" + + def broken_post_json(*args, **kwargs): + del args, kwargs + raise RuntimeError("provider secret response body") + + monkeypatch.setattr("backend.app.analysis_run_start.post_json", broken_post_json) + + with pytest.raises(TeppNotAvailable, match="transport request failed") as error: + configured_tepp_client("https://tepp.example/v1/analysis-runs").submit_analysis_run( + _sample_request() + ) + + assert "provider secret" not in str(error.value) diff --git a/uv.lock b/uv.lock index eed84dc7a..5b2f39e62 100644 --- a/uv.lock +++ b/uv.lock @@ -454,7 +454,7 @@ wheels = [ [[package]] name = "lineageweave" -version = "2.14.0" +version = "2.16.0" source = { editable = "." } dependencies = [ { name = "certifi" },