diff --git a/CHANGELOG.md b/CHANGELOG.md index b100f9fd6..374e6d5a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ All notable changes to this project are documented here. Format follows ### Added +- Evidence-bound occupational construct semantics now keep cognitive + abilities, work styles, work activities, affective reactions, performance + behaviors, and FJA worker functions distinct. Record-to-construct links + require a reified evidence span and PROV-O derivation/time; unsupported + DPT-to-psychology crosswalks and local scores remain unavailable (ADR 0248). +- Versioned occupational construct vocabularies and semantic-unit assertions + now persist in normalized tables. Database and application validation require + same-Post verbatim evidence, and authorized Post detail exposes provenance + without internal identifiers or numerical scores (ADR 0249). +- An operator-only O*NET 31.0 catalog synchronizer now imports every official + cognitive-ability, work-style, and work-activity Content Model element with + stable IRIs, descriptions, attribution, and a deterministic source digest; + conflicting release metadata fails closed (ADR 0250). - The DOT/FJA Data/People/Things worker-function taxonomy is now published in the canonical ontology: all 24 worker functions carry the official Dictionary of Occupational Titles Appendix B definitions verbatim, their diff --git a/backend/app/main.py b/backend/app/main.py index 6457bbde1..e6e5f3fa5 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -141,6 +141,9 @@ post_content_is_complete, publish_post_content_event, ) +from backend.app.occupational_construct_ingestion import ( + load_occupational_construct_assertions, +) from backend.app.post_content_worker import run_post_content_worker from backend.app.post_eligibility import SOURCE_POST_ELIGIBILITY_SQL, source_post_visible from backend.app.post_evaluation_ingestion import ( @@ -1654,6 +1657,9 @@ async def read_post( project_evidence = await _load_project_evidence( conn, post_id, row["source_project_code"], row["source_project_name"] ) + occupational_construct_assertions = await load_occupational_construct_assertions( + conn, post_id + ) known_at = None if as_of_clock is not None: known_at = await fetch_known_at_revision(conn, post_id, as_of_clock) @@ -1661,6 +1667,7 @@ async def read_post( **_serialize_post(row, labels), "post_body": row["post_body"], "project_evidence": project_evidence, + "occupational_construct_assertions": occupational_construct_assertions, } if known_at is not None: payload["known_at"] = known_at diff --git a/backend/app/occupational_construct_ingestion.py b/backend/app/occupational_construct_ingestion.py new file mode 100644 index 000000000..dd446a09c --- /dev/null +++ b/backend/app/occupational_construct_ingestion.py @@ -0,0 +1,207 @@ +"""Persist evidence-bound occupational constructs under ADR 0249.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlsplit + + +CONSTRUCT_FAMILIES = frozenset( + { + "cognitive_ability", + "work_style", + "work_activity", + "affective_reaction", + "performance_behavior", + } +) +TRUTH_STATUS_CODES = frozenset( + { + "truth_authoritative", + "truth_observed", + "truth_inferred", + "truth_proposed", + "truth_superseded", + "truth_rejected", + } +) + + +def _https_iri(value: str, field_name: str) -> str: + """Return one normalized HTTPS IRI or reject untrusted input.""" + normalized = value.strip() + parsed = urlsplit(normalized) + if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password: + raise ValueError(f"{field_name} must be an absolute credential-free HTTPS IRI") + return normalized + + +@dataclass(frozen=True) +class ConstructVocabulary: + """One immutable external vocabulary release.""" + + vocabulary_iri: str + version_label: str + license_iri: str + attribution_text: str + + def __post_init__(self) -> None: + object.__setattr__(self, "vocabulary_iri", _https_iri(self.vocabulary_iri, "vocabulary_iri")) + object.__setattr__(self, "license_iri", _https_iri(self.license_iri, "license_iri")) + if not self.version_label.strip() or not self.attribution_text.strip(): + raise ValueError("vocabulary version and attribution must be non-empty") + + +@dataclass(frozen=True) +class OccupationalConstruct: + """One versioned external occupational construct.""" + + vocabulary: ConstructVocabulary + construct_iri: str + family_code: str + preferred_label: str + + def __post_init__(self) -> None: + object.__setattr__(self, "construct_iri", _https_iri(self.construct_iri, "construct_iri")) + if self.family_code not in CONSTRUCT_FAMILIES: + raise ValueError(f"unsupported occupational construct family {self.family_code!r}") + if not self.preferred_label.strip(): + raise ValueError("construct preferred label must be non-empty") + + +@dataclass(frozen=True) +class OccupationalConstructAssertion: + """One construct assertion bound to a verbatim semantic-unit span.""" + + post_content_unit_id: str + unit_text: str + construct: OccupationalConstruct + evidence_text: str + truth_status_code: str + extraction_method: str + + def __post_init__(self) -> None: + for name, value in ( + ("post_content_unit_id", self.post_content_unit_id), + ("unit_text", self.unit_text), + ("evidence_text", self.evidence_text), + ("truth_status_code", self.truth_status_code), + ("extraction_method", self.extraction_method), + ): + if not value.strip(): + raise ValueError(f"{name} must be non-empty") + if self.evidence_text not in self.unit_text: + raise ValueError("construct evidence must be a verbatim semantic-unit span") + if self.truth_status_code not in TRUTH_STATUS_CODES: + raise ValueError(f"unsupported ontology truth status {self.truth_status_code!r}") + + +async def persist_occupational_construct_assertions( + conn: Any, + post_id: str, + orchestrator_session_id: str, + assertions: tuple[OccupationalConstructAssertion, ...], +) -> None: + """Atomically replace one Post's construct assertions and shared registry rows.""" + if not post_id.strip() or not orchestrator_session_id.strip(): + raise ValueError("post and orchestrator session identifiers must be non-empty") + async with conn.transaction(): + await conn.execute( + "delete from post_occupational_construct_assertion where post_id = $1", + post_id, + ) + for assertion in assertions: + vocabulary = assertion.construct.vocabulary + vocabulary_id = await conn.fetchval( + """ + insert into occupational_construct_vocabulary + (vocabulary_iri, version_label, license_iri, attribution_text) + values ($1, $2, $3, $4) + on conflict (vocabulary_iri, version_label) do update set + vocabulary_iri = excluded.vocabulary_iri + where occupational_construct_vocabulary.license_iri = excluded.license_iri + and occupational_construct_vocabulary.attribution_text = excluded.attribution_text + returning vocabulary_id + """, + vocabulary.vocabulary_iri, + vocabulary.version_label, + vocabulary.license_iri, + vocabulary.attribution_text, + ) + if vocabulary_id is None: + raise ValueError("vocabulary metadata conflicts with its immutable version") + construct_id = await conn.fetchval( + """ + insert into occupational_construct + (vocabulary_id, construct_iri, construct_family_code, preferred_label) + values ($1, $2, $3, $4) + on conflict (vocabulary_id, construct_iri) do update set + construct_iri = excluded.construct_iri + where occupational_construct.construct_family_code = excluded.construct_family_code + and occupational_construct.preferred_label = excluded.preferred_label + returning construct_id + """, + vocabulary_id, + assertion.construct.construct_iri, + assertion.construct.family_code, + assertion.construct.preferred_label, + ) + if construct_id is None: + raise ValueError("construct metadata conflicts with its immutable vocabulary version") + await conn.execute( + """ + insert into post_occupational_construct_assertion + (post_id, post_content_unit_id, construct_id, evidence_text, + truth_status_code, extraction_method, orchestrator_session_id) + values ($1, $2, $3, $4, $5, $6, $7) + """, + post_id, + assertion.post_content_unit_id, + construct_id, + assertion.evidence_text, + assertion.truth_status_code, + assertion.extraction_method, + orchestrator_session_id, + ) + + +async def load_occupational_construct_assertions( + conn: Any, post_id: str +) -> list[dict[str, object]]: + """Load assertions only after the caller has authorized their Post.""" + rows = await conn.fetch( + """ + select construct.construct_iri, construct.construct_family_code, + construct.preferred_label, vocabulary.vocabulary_iri, + vocabulary.version_label, assertion.evidence_text, + assertion.truth_status_code, assertion.extraction_method, + assertion.generated_at, unit.unit_index + from post_occupational_construct_assertion assertion + join occupational_construct construct on construct.construct_id = assertion.construct_id + join occupational_construct_vocabulary vocabulary + on vocabulary.vocabulary_id = construct.vocabulary_id + join post_content_unit unit + on unit.post_content_unit_id = assertion.post_content_unit_id + where assertion.post_id = $1 + order by unit.unit_index, construct.construct_family_code, + construct.preferred_label, construct.construct_iri + """, + post_id, + ) + return [ + { + "construct_iri": row["construct_iri"], + "construct_family_code": row["construct_family_code"], + "preferred_label": row["preferred_label"], + "vocabulary_iri": row["vocabulary_iri"], + "vocabulary_version": row["version_label"], + "evidence_text": row["evidence_text"], + "truth_status_code": row["truth_status_code"], + "extraction_method": row["extraction_method"], + "generated_at": row["generated_at"], + "unit_index": row["unit_index"], + "provenance": "post_occupational_construct_assertion.evidence_text", + } + for row in rows + ] diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 892c8231a..ee2dc8a93 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -231,6 +231,19 @@ / "migrations" / "0183_source_post_event_occurred_at.sql" ) +_ONTOLOGY_TRUTH_STATUS_MIGRATION = ( + Path(__file__).resolve().parents[2] / "migrations" / "0175_ontology_truth_status.sql" +) +_OCCUPATIONAL_CONSTRUCT_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0238_occupational_construct_assertion.sql" +) +_OCCUPATIONAL_CATALOG_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0239_occupational_construct_catalog.sql" +) def _postgres_available() -> bool: @@ -383,12 +396,17 @@ def seeded_db(demo_analyst_token): cur.execute(_LEFTOVER_MAP_COVERAGE_MIGRATION.read_text()) cur.execute(_GLOBAL_ASK_JOB_MIGRATION.read_text()) cur.execute(_GLOBAL_ASK_SCOPE_MIGRATION.read_text()) - cur.execute(_GLOBAL_ASK_EVIDENCE_SEARCH_MIGRATION.read_text()) + for statement in _GLOBAL_ASK_EVIDENCE_SEARCH_MIGRATION.read_text().split(";\n\n"): + if statement.strip(): + cur.execute(statement + ";") cur.execute(_GLOBAL_ASK_KNOWLEDGE_CUTOFF_MIGRATION.read_text()) cur.execute(_GLOBAL_ASK_PUBLIC_VERIFICATION_MIGRATION.read_text()) cur.execute(_EVENT_OCCURRED_AT_MIGRATION.read_text()) + cur.execute(_ONTOLOGY_TRUTH_STATUS_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_AXIS_MIGRATION.read_text()) cur.execute(_CHANNEL_EVIDENCE_MIGRATION.read_text()) + cur.execute(_OCCUPATIONAL_CONSTRUCT_MIGRATION.read_text()) + cur.execute(_OCCUPATIONAL_CATALOG_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_UNEXPLAINED_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_CROSS_SHARE_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_RECONSTRUCTION_MIGRATION.read_text()) @@ -1945,6 +1963,76 @@ def test_post_detail_exposes_explicit_and_semantic_project_evidence( assert listed_post["project_evidence"][0]["provenance"] == "post_project_mention.evidence_text" +def test_post_detail_exposes_evidence_bound_occupational_construct( + client, demo_analyst_token, seeded_db +) -> None: + """The authorized detail projection preserves its exact synthetic evidence.""" + conn = psycopg2.connect(seeded_db["dsn"]) + try: + with conn.cursor() as cur: + cur.execute( + """ + insert into post_content_unit + (post_id, unit_index, unit_kind_code, unit_text) + values (%s, 90, 'plain_text', %s) + returning post_content_unit_id + """, + ( + seeded_db["public_post_id"], + "Synthetic record requires oral comprehension.", + ), + ) + unit_id = cur.fetchone()[0] + cur.execute( + """ + insert into occupational_construct_vocabulary + (vocabulary_iri, version_label, license_iri, attribution_text) + values (%s, '31.0', %s, 'Synthetic O*NET attribution') + returning vocabulary_id + """, + ( + "https://www.onetcenter.org/database.html", + "https://creativecommons.org/licenses/by/4.0/", + ), + ) + vocabulary_id = cur.fetchone()[0] + cur.execute( + """ + insert into occupational_construct + (vocabulary_id, construct_iri, construct_family_code, preferred_label) + values (%s, %s, 'cognitive_ability', 'Oral Comprehension') + returning construct_id + """, + (vocabulary_id, "https://data.onetcenter.org/element/1.A.1.a.1"), + ) + construct_id = cur.fetchone()[0] + cur.execute( + """ + insert into post_occupational_construct_assertion + (post_id, post_content_unit_id, construct_id, evidence_text, + truth_status_code, extraction_method, orchestrator_session_id) + values (%s, %s, %s, 'oral comprehension', 'truth_inferred', + 'contextual_orchestrator_structured', 'synthetic-session') + """, + (seeded_db["public_post_id"], unit_id, construct_id), + ) + conn.commit() + finally: + conn.close() + + response = client.get( + f"/api/posts/{seeded_db['public_post_id']}", + headers={"Authorization": f"Bearer {demo_analyst_token}"}, + ) + assert response.status_code == 200 + assertions = response.json()["occupational_construct_assertions"] + assert assertions[0]["preferred_label"] == "Oral Comprehension" + assert assertions[0]["evidence_text"] == "oral comprehension" + assert assertions[0]["provenance"] == ( + "post_occupational_construct_assertion.evidence_text" + ) + + def test_post_detail_as_of_returns_the_cutoff_known_body( client, demo_analyst_token, seeded_db ) -> None: diff --git a/docs/adr/0248-occupational-construct-evidence-boundary.md b/docs/adr/0248-occupational-construct-evidence-boundary.md new file mode 100644 index 000000000..1f4056cec --- /dev/null +++ b/docs/adr/0248-occupational-construct-evidence-boundary.md @@ -0,0 +1,92 @@ +# ADR 0248: Evidence-bound occupational constructs + +**Status:** Accepted +**Date:** 2026-08-27 +**Extends:** [ADR 0011](0011-prov-o-standard-relations.md), [ADR 0065](0065-prov-o-provenance-boundary.md), [ADR 0184](0184-ontology-provenance-explorer.md), [ADR 0232](0232-worker-function-taxonomy-in-the-published-ontology.md) + +## Context + +ADR 0232 publishes the 24 DOT/FJA Data, People, and Things worker +functions, but those functions describe how work relates to data, people, +or things. They are not cognitive abilities, affective reactions, +personality tendencies, or observed behavior. Treating Data as cognition, +People as affect, or Things as behavior would invent a crosswalk that the +authorities do not publish. + +The O*NET 31.0 Content Model already supplies maintained identifiers and +hierarchies for abilities, work styles, skills, work activities, work +context, and tasks. It also publishes specific Ability-to-Work-Activity and +Work-Style-to-Work-Activity linkages. Its downloadable RDF graph is the +authoritative reusable semantic source; copying thousands of changing terms +into the LineageWeave namespace would create a stale second vocabulary. +O*NET work styles are personality tendencies, not momentary affect. +EmotionML likewise requires an explicitly named emotion vocabulary because +affective science has no single default category set. + +## Decision + +1. Keep five non-equivalent construct classes in the published ontology: + `CognitiveAbility`, `WorkStyle`, `WorkActivity`, `AffectiveReaction`, and + `PerformanceBehavior`, all subclasses of `OccupationalConstruct`. + FJA `WorkerFunction` remains a separate class and concept scheme. +2. Reuse official, versioned external identifiers and relationships. O*NET + 31.0 RDF is authoritative for O*NET concepts and its published linkages; + LineageWeave does not remint or paraphrase those terms. Affect must name an + EmotionML-compatible vocabulary. No configured source means unavailable. +3. A source Post may `supportsOccupationalConstruct` only through an + `OccupationalConstructAssertion`: an RDF-reified statement with exactly + one Post subject, the fixed predicate, one construct object, a non-empty + verbatim evidence span, `prov:wasDerivedFrom` that same Post, and + `prov:generatedAtTime`. SHACL rejects incomplete projections. +4. The assertion is evidence about record content, not a person trait, + diagnosis, ability score, job requirement, or causal effect. Person-, + job-, task-, or position-level binding needs a separate normalized schema + decision with authorization, subject identity, event/system time, + validity, and measurement provenance. +5. Only source-published cross-scheme links may be materialized. The O*NET + Ability-to-Work-Activity and Work-Style-to-Work-Activity datasets qualify. + A documented loose FJA/GWA orientation may be cited as a qualified + association, never as `owl:equivalentClass`, `owl:sameAs`, + `skos:exactMatch`, `skos:closeMatch`, or a rank mapping. Transitive chains + must not manufacture a DPT-to-ability or DPT-to-work-style assertion. +6. No rank, confidence, intensity, importance, level, or weight is computed + locally. Published measurements remain tied to their source scale and + sampling metadata; TEPP and fast-mlsirm retain numerical authority under + ADR 0208. + +## Considered options + +| Option | Outcome | +|---|---| +| Map Data/People/Things directly to cognitive/affective/behavioral facets | Rejected: intuitive but unsupported, collapses work functions into psychological constructs | +| Copy the complete O*NET vocabulary into local IRIs | Rejected: duplicates a maintained linked-data source and creates quarterly drift | +| Link external constructs through evidence-bearing assertions | Accepted: preserves authoritative identifiers, provenance, uncertainty, and an additive persistence path | + +## Consequences + +- The semantic layer can represent the requested construct families and their + evidence relationship without claiming that the first source mention is a + measured person attribute. +- Complete O*NET breadth remains available through its maintained RDF graph; + ADRs 0249 and 0250 add normalized assertion persistence and official catalog + synchronization. Record extraction remains unavailable until its separate + contextual-orchestrator contract is accepted and shipped. +- Actual affect stays absent unless the evidence names a conforming affect + vocabulary and supports the reaction; work style is never relabeled affect. +- Unsupported equivalence and causal links fail closed rather than becoming + graph navigation facts. + +## Verification + +- `tests/test_ontology.py` pins class separation, direct assertion direction, + prohibited FJA equivalence, and PROV-O requirements. +- `tests/test_ontology_shapes.py` validates complete assertion projections and + rejects missing evidence or derivation. +- Ontology publication tests continue to prove deterministic Turtle, + JSON-LD, N-Triples, SHACL, and human-readable output. + +## References + +See +[`docs/doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md`](../doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md) +for the APA 7 evidence register and adoption limits. diff --git a/docs/adr/0249-occupational-construct-assertion-persistence.md b/docs/adr/0249-occupational-construct-assertion-persistence.md new file mode 100644 index 000000000..bf5da06f3 --- /dev/null +++ b/docs/adr/0249-occupational-construct-assertion-persistence.md @@ -0,0 +1,66 @@ +# ADR 0249: Occupational construct assertion persistence + +**Status:** Accepted +**Date:** 2026-08-27 +**Extends:** [ADR 0062](0062-semantic-unit-embedding.md), [ADR 0184](0184-ontology-provenance-explorer.md), [ADR 0248](0248-occupational-construct-evidence-boundary.md) + +## Context + +ADR 0248 defines the semantic assertion but intentionally leaves persistence +unavailable. A bare Post-to-construct edge would lose the exact semantic unit, +source vocabulary version, extraction session, truth status, and verbatim +evidence that make the assertion reviewable. Reusing `knowledge_graph_edge` +would also mix a provenance-bearing analysis artifact into the navigation +projection forbidden by ADR 0065. + +## Decision + +1. Persist source vocabularies, their versioned external constructs, and Post + assertions in three normalized tables: `occupational_construct_vocabulary`, + `occupational_construct`, and `post_occupational_construct_assertion`. +2. Every assertion references one existing `post_content_unit`, one construct, + one ontology truth-status code, one extraction method, and the shared + post-scoped contextual-orchestrator session identifier. It stores no score, + weight, intensity, importance, causal flag, or person binding. +3. A database trigger rejects an assertion when the semantic unit belongs to a + different Post or its evidence is not a non-empty verbatim substring of the + stored unit text. Application validation mirrors this boundary before SQL. +4. Replacement is atomic per Post. Vocabulary and construct rows use natural + versioned uniqueness and UPSERT; post assertions are deleted and recreated + inside the same transaction so stale analysis cannot coexist with a new + source-derived set. +5. The already-authorized `GET /api/posts/{post_id}` response may include the + assertions after the Post ABAC decision succeeds. It exposes evidence, + construct label/IRI/family, vocabulary/version, truth status, method, and + generated time, but not internal database identifiers. +6. Search, ontology-neighborhood nodes, extraction prompts, UI presentation, + and person/job/task binding remain separate increments. Absence returns an + empty list. + +## Considered options + +| Option | Outcome | +|---|---| +| Store construct JSON on the Post | Rejected: duplicates vocabulary metadata and defeats 3NF/query integrity | +| Add bare knowledge-graph edges | Rejected: loses evidence-unit provenance and confuses navigation with assertion ownership | +| Versioned registry plus evidence-unit assertion | Accepted: normalized, replayable, provenance-bearing, and additive | + +## Consequences + +- Authorized clients can inspect persisted construct evidence without treating + it as a measured person attribute. +- Vocabulary updates create a new version row instead of silently changing the + meaning of historical assertions. +- The trigger adds one indexed unit lookup per written assertion; batch volume + is bounded by the semantic units of one Post. A future measured throughput + problem may replace it with a set-based staging validator. + +## Verification + +- `tests/test_occupational_construct_persistence.py` verifies trust-boundary + validation, atomic replacement SQL, versioned UPSERT, and empty replacement. +- `tests/test_occupational_construct_schema.py` verifies replay safety, 3NF + foreign keys, evidence trigger, prohibited numeric fields, and hot-path + indexes. +- The existing Post-detail ABAC path loads the projection only after its + visibility decision; focused tests verify the returned review model. diff --git a/docs/adr/0250-official-occupational-construct-catalog-sync.md b/docs/adr/0250-official-occupational-construct-catalog-sync.md new file mode 100644 index 000000000..b780ebd1c --- /dev/null +++ b/docs/adr/0250-official-occupational-construct-catalog-sync.md @@ -0,0 +1,69 @@ +# ADR 0250: Official occupational construct catalog synchronization + +**Status:** Accepted +**Date:** 2026-08-27 +**Extends:** [ADR 0248](0248-occupational-construct-evidence-boundary.md), [ADR 0249](0249-occupational-construct-assertion-persistence.md) + +## Context + +The assertion store accepted by ADR 0249 needs a complete, reviewable catalog +before contextual-orchestrator can select a construct. Asking a model to invent +an O*NET label or IRI would defeat ADR 0248. Shipping a hand-maintained subset +would also drift from O*NET's quarterly releases and omit most of the requested +cognitive and behavioral domain. + +O*NET 31.0 publishes a machine-readable Content Model Reference with stable +element identifiers, names, hierarchy positions, descriptions, release +documentation, and CC BY 4.0 attribution terms. Its hierarchy explicitly places +cognitive abilities below `1.A.1`, work styles below `1.D`, and work activities +below `4.A`. These are source classifications, not a LineageWeave heuristic. + +## Decision + +1. An operator-only synchronizer reads the fixed HTTPS O*NET 31.0 Content Model + Reference JSON document. The URL, release, vocabulary IRI, license IRI, and + attribution are code-reviewed constants; runtime input cannot redirect the + process to an arbitrary host. +2. The synchronizer imports every element at or below the three published + hierarchy roots: `1.A.1` as `cognitive_ability`, `1.D` as `work_style`, and + `4.A` as `work_activity`. It preserves official labels, optional descriptions, + and permanent `https://data.onetcenter.org/element/{element_id}` IRIs. +3. The canonical decoded JSON SHA-256 is stored on the vocabulary release. + The reviewed O*NET 31.0 document digest is + `cb25e83a25c355dba035afdfc6b23ed8706a939d5f5021ed772d554ea49afb06`; + synchronization rejects any other digest before opening a database + transaction. Replaying the same document is idempotent. A changed document + under the same release, or conflicting construct metadata, aborts instead + of rewriting history. The reviewed document contains 3,006 source rows and + admits 2,529 governed constructs: 29 cognitive abilities, 26 work styles, + and 2,474 work activities. +4. This catalog does not import occupation ratings, scores, scale values, + ability-to-activity linkages, work-style linkages, FJA crosswalks, affective + vocabularies, or person/job bindings. Those require their own provenance and + decision records. +5. Catalog synchronization is a prerequisite for extraction. The extractor + may select only catalog rows supplied to contextual-orchestrator; it may not + mint a label, family, description, or IRI. + +## Consequences + +- The semantic layer gains the full official breadth needed for catalog-bound + cognitive, work-style, and work-activity assertions without copying these + terms into the LineageWeave ontology namespace. +- O*NET descriptions that are absent remain `NULL`; LineageWeave does not fill + them with generated prose. +- Affective reactions and performance interpretation remain unavailable until + an authoritative vocabulary and evidence contract are accepted. + +## Verification + +- Parser tests cover all three roots, exact IRI construction, ignored unrelated + rows, malformed payloads, and deterministic source hashing. +- Schema tests require replay-safe catalog description and source-hash columns. +- Synchronization tests prove idempotent UPSERTs and post-write exact metadata + comparison. + +## References + +See +[`docs/doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md`](../doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md). diff --git a/docs/adr/README.md b/docs/adr/README.md index e92be6b20..f2ff20d86 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -28,6 +28,7 @@ decision from them. | [`temporal-topic-context-influence-research.md`](../temporal-topic-context-influence-research.md) | [0210](0210-temporal-topic-context-influence-dashboard.md) | | [`python-mathematical-compute-boundary-audit.md`](../doctoring/python-mathematical-compute-boundary-audit.md) | [0208](0208-externalize-local-mathematical-compute.md) | | [`WORKER_FUNCTION_TAXONOMY_REFERENCES.md`](../doctoring/WORKER_FUNCTION_TAXONOMY_REFERENCES.md) | [0232](0232-worker-function-taxonomy-in-the-published-ontology.md) | +| [`OCCUPATIONAL_CONSTRUCT_REFERENCES.md`](../doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md) | [0248](0248-occupational-construct-evidence-boundary.md) | [0011](0011-prov-o-standard-relations.md) and [0065](0065-prov-o-provenance-boundary.md) cite the dated W3C PROV-O and PROV-DM Recommendations (https://www.w3.org/TR/2013/REC-prov-o-20130430/ and https://www.w3.org/TR/2013/REC-prov-dm-20130430/). diff --git a/docs/doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md b/docs/doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md new file mode 100644 index 000000000..7f7aba651 --- /dev/null +++ b/docs/doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md @@ -0,0 +1,84 @@ +# Occupational construct evidence register + +Supporting research for [ADR 0248](../adr/0248-occupational-construct-evidence-boundary.md). +This register does not create mappings. It records which relationships a +source actually supports and which tempting inferences remain prohibited. + +## Adopted sources and limits + +- The O*NET 31.0 database is the maintained source for ability, work-style, + skill, work-activity, work-context, task, and published linkage identifiers. + It is CC BY 4.0; derived products must credit USDOL/ETA, link the license, + and identify modifications. LineageWeave links rather than remints these + resources. +- The O*NET 31.0 Content Model Reference publishes 3,006 hierarchy elements. + ADR 0250 admits only the source-defined cognitive-ability (`1.A.1`), work- + style (`1.D`), and work-activity (`4.A`) roots and descendants; it preserves + blank descriptions as unavailable and stores no occupation rating. +- The O*NET Content Model separates worker characteristics and requirements + from occupational requirements. It does not make FJA worker functions + equivalent to abilities, dispositions, or affect. +- O*NET publishes Ability-to-Work-Activity and Work-Style-to-Work-Activity + linkage datasets. Those source records may be reused with their identifiers + and provenance; transitive DPT mappings may not be inferred from them. +- EmotionML defines a representation mechanism, not a universal emotion + taxonomy. Every affective assertion must identify its vocabulary. +- PROV-O qualified relations and SHACL validation support evidence-bearing, + fail-closed assertions. They do not establish occupational-psychology + validity by themselves. + +## Explicit non-adoptions + +- Data = cognitive, People = affective, and Things = behavioral. +- FJA rank as ability intensity, affect, performance, or interval measurement. +- O*NET work styles as moods or emotions. +- `owl:sameAs`, `owl:equivalentClass`, `skos:exactMatch`, or + `skos:closeMatch` between FJA functions and psychological constructs. +- Causal or person-level claims derived only from a job title, work function, + aggregate occupation rating, or source-system label. + +## APA 7 references + +Hansen, M. C., Norton, J. J., Gregory, C. M., Meade, A. W., Foster Thompson, +L., Rivkin, D., Lewis, P., & Nottingham, J. (2014). *A multi-phase rational +method for developing area work activities*. National Center for O*NET +Development. https://www.onetcenter.org/dl_files/DWA_2014.pdf + +Knublauch, H., & Kontokostas, D. (Eds.). (2017). *Shapes Constraint Language +(SHACL).* World Wide Web Consortium. https://www.w3.org/TR/shacl/ + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV +ontology.* World Wide Web Consortium. https://www.w3.org/TR/prov-o/ + +Miles, A., & Bechhofer, S. (Eds.). (2009). *SKOS simple knowledge +organization system reference.* World Wide Web Consortium. +https://www.w3.org/TR/skos-reference/ + +National Center for O*NET Development. (2026). *O*NET 31.0 database* [Data +set]. U.S. Department of Labor, Employment and Training Administration. +https://www.onetcenter.org/database.html + +National Center for O*NET Development. (2026). *O*NET 31.0 Content Model +Reference* [Data set]. U.S. Department of Labor, Employment and Training +Administration. https://www.onetcenter.org/dl_files/database/db_31_0_json/content_model_reference.json + +Peterson, N. G., Mumford, M. D., Borman, W. C., Jeanneret, P. R., Fleishman, +E. A., Levin, K. Y., Campion, M. A., Mayfield, M. S., Morgeson, F. P., +Pearlman, K., Gowing, M. K., Lancaster, A. R., Silver, M. B., & Dye, D. M. +(2001). Understanding work using the Occupational Information Network +(O*NET): Implications for practice and research. *Personnel Psychology, +54*(2), 451–492. https://doi.org/10.1111/j.1744-6570.2001.tb00100.x + +Putka, D. J., Kell, H. J., Voss, N., Oswald, F. L., & Lewis, P. (2024). +*Revisiting the work styles domain of the O*NET Content Model* (Report No. +090). Human Resources Research Organization. +https://www.onetcenter.org/dl_files/Work_Styles_New.pdf + +Schröder, M., Pirker, H., & Lamolle, M. (Eds.). (2014). *Emotion Markup +Language (EmotionML) 1.0.* World Wide Web Consortium. +https://www.w3.org/TR/emotionml/ + +Weiss, H. M., & Cropanzano, R. (1996). Affective events theory: A theoretical +discussion of the structure, causes and consequences of affective experiences +at work. *Research in Organizational Behavior, 18*, 1–74. +https://web.mit.edu/curhan/www/docs/Articles/15341_Readings/Affect/AffectiveEventsTheory_WeissCropanzano.pdf diff --git a/docs/ontology/lineageweave-kg-shapes.ttl b/docs/ontology/lineageweave-kg-shapes.ttl index 436eb401f..5d5b5c0f2 100644 --- a/docs/ontology/lineageweave-kg-shapes.ttl +++ b/docs/ontology/lineageweave-kg-shapes.ttl @@ -23,10 +23,11 @@ # # Every sh:targetClass must live in the canonical repository-case namespace; # sh:path may additionally use RDF's subject/predicate/object reification -# predicates. scripts/publish_ontology_site.py fails closed on every other -# external or dangling target so a renamed term cannot silently orphan its -# shape. tests/test_ontology_shapes.py validates this graph against the -# ontology source with pyshacl, plus negative violation tests. +# predicates and PROV-O's derivation/time predicates. +# scripts/publish_ontology_site.py fails closed on every other external or +# dangling target so a renamed term cannot silently orphan its shape. +# tests/test_ontology_shapes.py validates this graph against the ontology +# source with pyshacl, plus negative violation tests. ################################################################# @@ -166,6 +167,58 @@ sh:datatype xsd:string ; ] . +:OccupationalConstructAssertionShape a sh:NodeShape ; + rdfs:label "Occupational construct assertion shape" ; + sh:targetClass :OccupationalConstructAssertion ; + sh:property [ + sh:path rdf:subject ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:class :Post ; + ] ; + sh:property [ + sh:path rdf:predicate ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:hasValue :supportsOccupationalConstruct ; + ] ; + sh:property [ + sh:path rdf:object ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:class :OccupationalConstruct ; + ] ; + sh:property [ + sh:path :constructEvidence ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:datatype xsd:string ; + sh:minLength 1 ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:class :Post ; + ] ; + sh:property [ + sh:path ; + sh:minCount 1 ; + sh:maxCount 1 ; + sh:datatype xsd:dateTime ; + ] ; + sh:sparql [ + a sh:SPARQLConstraint ; + sh:message "prov:wasDerivedFrom must identify the same Post as rdf:subject." ; + sh:select """ + SELECT $this WHERE { + $this ?post ; + ?source . + FILTER (?post != ?source) + } + """ ; + ] . + :OurSidePersonShape a sh:NodeShape ; rdfs:label "Our-side person shape" ; sh:comment "Closed-world complement of :OurSidePerson owl:disjointWith :CounterpartyPerson: an instance of one can never be typed as the other." ; diff --git a/docs/ontology/lineageweave-kg.ttl b/docs/ontology/lineageweave-kg.ttl index 59c33ce8c..1adcb94bf 100644 --- a/docs/ontology/lineageweave-kg.ttl +++ b/docs/ontology/lineageweave-kg.ttl @@ -437,6 +437,55 @@ rdfs:domain :ProjectMention ; rdfs:range xsd:decimal . +################################################################# +# Evidence-bound occupational constructs (ADR 0248). +################################################################# + +:OccupationalConstruct a owl:Class ; + rdfs:label "Occupational construct"@en ; + rdfs:comment "A governed cognitive, dispositional, behavioral, or affective concept referenced by authorized record evidence; it is not itself a person-level measurement."@en . + +:CognitiveAbility a owl:Class ; + rdfs:subClassOf :OccupationalConstruct ; + rdfs:label "Cognitive ability"@en . + +:WorkStyle a owl:Class ; + rdfs:subClassOf :OccupationalConstruct ; + rdfs:label "Work style"@en ; + rdfs:comment "A personality tendency exhibited at work; not a mood or emotion."@en . + +:WorkActivity a owl:Class ; + rdfs:subClassOf :OccupationalConstruct ; + rdfs:label "Work activity"@en . + +:AffectiveReaction a owl:Class ; + rdfs:subClassOf :OccupationalConstruct ; + rdfs:label "Affective reaction"@en ; + rdfs:comment "An evidence-supported reaction represented with an explicitly identified EmotionML-compatible vocabulary; no default emotion category is inferred."@en . + +:PerformanceBehavior a owl:Class ; + rdfs:subClassOf :OccupationalConstruct ; + rdfs:label "Performance behavior"@en . + +:supportsOccupationalConstruct a owl:ObjectProperty ; + rdfs:domain :Post ; + rdfs:range :OccupationalConstruct ; + rdfs:label "supports occupational construct"@en ; + rdfs:comment "Record evidence supports discussion of a construct; this does not assert a person trait, score, job requirement, or cause."@en . + +:OccupationalConstructAssertion a owl:Class ; + rdfs:subClassOf rdf:Statement, + [ a owl:Restriction ; owl:onProperty rdf:subject ; owl:allValuesFrom :Post ], + [ a owl:Restriction ; owl:onProperty rdf:predicate ; owl:hasValue :supportsOccupationalConstruct ], + [ a owl:Restriction ; owl:onProperty rdf:object ; owl:allValuesFrom :OccupationalConstruct ] ; + rdfs:label "Occupational construct assertion"@en ; + rdfs:comment "A provenance-bearing reified statement that one authorized Post contains evidence supporting an occupational construct."@en . + +:constructEvidence a owl:DatatypeProperty ; + rdfs:domain :OccupationalConstructAssertion ; + rdfs:range xsd:string ; + rdfs:label "construct evidence"@en . + ################################################################# # Worker-function taxonomy (ADR 0232). # diff --git a/docs/product-requirements.md b/docs/product-requirements.md index b728080b1..63a6cbf14 100644 --- a/docs/product-requirements.md +++ b/docs/product-requirements.md @@ -72,6 +72,25 @@ and lookup round-trip isolation are enforced by `tests/test_worker_function_taxonomy.py`; `tests/test_ontology.py` continues to pass unchanged. +### PRD-FR-2B — Evidence-bound occupational constructs + +- Keep cognitive abilities, work styles, work activities, affective + reactions, and performance behaviors as non-equivalent construct classes + (ADR 0248). FJA worker functions remain separate. +- Reuse official external identifiers and source-published relationships; + never infer a DPT-to-psychology crosswalk or relabel work style as affect. +- Bind a construct to record content only through a provenance-bearing, + evidence-cited assertion. Do not promote record evidence to a person trait, + score, causal effect, or job requirement. + +Acceptance: SHACL rejects incomplete assertions; ontology tests prohibit FJA +equivalence and require exact Post/evidence/PROV statement structure. ADR 0249 +adds normalized, semantic-unit-bound persistence and an authorized Post-detail +projection. ADR 0250 synchronizes all official O*NET cognitive-ability, +work-style, and work-activity Content Model elements into that versioned +registry without importing ratings. Search, graph navigation, extraction, and +UI remain unavailable until their separate ADR acceptance. + ### PRD-FR-3 — Bounded ontology exploration - Apply RBAC/ABAC, source eligibility, and knowledge cutoff before graph diff --git a/frontend/src/api.ts b/frontend/src/api.ts index fca5882d0..03b2cece4 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -100,6 +100,7 @@ export interface PostKnownAt { export interface PostDetail extends PostSummary { post_body: string; + occupational_construct_assertions: OccupationalConstructAssertion[]; known_at?: PostKnownAt; } @@ -264,6 +265,20 @@ export interface ProjectEvidence { provenance: string; } +export interface OccupationalConstructAssertion { + construct_iri: string; + construct_family_code: string; + preferred_label: string; + vocabulary_iri: string; + vocabulary_version: string; + evidence_text: string; + truth_status_code: string; + extraction_method: string; + generated_at: string; + unit_index: number; + provenance: string; +} + export interface PostAiSummary { post_id: string; korean_summary: string; diff --git a/lineageweave/occupational_construct_catalog.py b/lineageweave/occupational_construct_catalog.py new file mode 100644 index 000000000..154190da1 --- /dev/null +++ b/lineageweave/occupational_construct_catalog.py @@ -0,0 +1,192 @@ +"""Synchronize an evidence-safe subset of the official O*NET catalog.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from typing import Any + + +ONET_RELEASE = "31.0" +ONET_CONTENT_MODEL_URL = ( + "https://www.onetcenter.org/dl_files/database/" + "db_31_0_json/content_model_reference.json" +) +ONET_CONTENT_MODEL_CANONICAL_SHA256 = ( + "cb25e83a25c355dba035afdfc6b23ed8706a939d5f5021ed772d554ea49afb06" +) +ONET_VOCABULARY_IRI = "https://www.onetcenter.org/database.html" +ONET_LICENSE_IRI = "https://creativecommons.org/licenses/by/4.0/" +ONET_ATTRIBUTION = ( + "This product includes information from the O*NET 31.0 Database by " + "the U.S. Department of Labor, Employment and Training Administration " + "(USDOL/ETA). Used under the CC BY 4.0 license. O*NET® is a trademark " + "of USDOL/ETA." +) +_ELEMENT_ID = re.compile(r"^[0-9]+(?:\.[A-Za-z0-9]+)*$") +_FAMILY_ROOTS = ( + ("1.A.1", "cognitive_ability"), + ("1.D", "work_style"), + ("4.A", "work_activity"), +) + + +@dataclass(frozen=True) +class CatalogConstruct: + """One exact O*NET Content Model element admitted by ADR 0250.""" + + construct_iri: str + family_code: str + preferred_label: str + description: str | None + + +def catalog_content_sha256(payload: dict[str, Any]) -> str: + """Hash the deterministic canonical JSON representation of one release.""" + canonical = json.dumps( + payload, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +def parse_onet_construct_catalog(payload: dict[str, Any]) -> tuple[CatalogConstruct, ...]: + """Parse only published cognitive, work-style, and work-activity roots.""" + if payload.get("table_id") != "content_model_reference": + raise ValueError("O*NET payload is not the Content Model Reference") + rows = payload.get("row") + if not isinstance(rows, list): + raise ValueError("O*NET Content Model Reference rows must be an array") + + constructs: dict[str, CatalogConstruct] = {} + for row in rows: + if not isinstance(row, dict): + raise ValueError("O*NET Content Model Reference row must be an object") + element_id = row.get("element_id") + label = row.get("element_name") + if not isinstance(element_id, str) or not _ELEMENT_ID.fullmatch(element_id): + raise ValueError("O*NET element_id is malformed") + if not isinstance(label, str) or not label.strip(): + raise ValueError("O*NET element_name must be non-empty") + if label != label.strip(): + raise ValueError("O*NET element_name must not contain outer whitespace") + family = next( + ( + family_code + for root, family_code in _FAMILY_ROOTS + if element_id == root or element_id.startswith(f"{root}.") + ), + None, + ) + if family is None: + continue + description_value = row.get("description") + if description_value is not None and not isinstance(description_value, str): + raise ValueError("O*NET description must be text or null") + description = (description_value or "").strip() or None + iri = f"https://data.onetcenter.org/element/{element_id}" + if iri in constructs: + raise ValueError(f"duplicate O*NET construct IRI: {iri}") + constructs[iri] = CatalogConstruct(iri, family, label, description) + if not constructs: + raise ValueError("O*NET catalog contains no governed construct roots") + return tuple(constructs[iri] for iri in sorted(constructs)) + + +async def sync_onet_construct_catalog( + conn: Any, + payload: dict[str, Any], + *, + expected_source_sha256: str = ONET_CONTENT_MODEL_CANONICAL_SHA256, +) -> int: + """Atomically synchronize one immutable O*NET release and verify it exactly.""" + source_sha256 = catalog_content_sha256(payload) + if source_sha256 != expected_source_sha256: + raise ValueError("O*NET 31.0 source digest differs from the reviewed release") + constructs = parse_onet_construct_catalog(payload) + async with conn.transaction(): + vocabulary_id = await conn.fetchval( + """ + insert into occupational_construct_vocabulary + (vocabulary_iri, version_label, license_iri, attribution_text, + source_content_sha256) + values ($1, $2, $3, $4, $5) + on conflict (vocabulary_iri, version_label) do update set + source_content_sha256 = coalesce( + occupational_construct_vocabulary.source_content_sha256, + excluded.source_content_sha256 + ) + where occupational_construct_vocabulary.license_iri = excluded.license_iri + and occupational_construct_vocabulary.attribution_text = excluded.attribution_text + and ( + occupational_construct_vocabulary.source_content_sha256 is null + or occupational_construct_vocabulary.source_content_sha256 = excluded.source_content_sha256 + ) + returning vocabulary_id + """, + ONET_VOCABULARY_IRI, + ONET_RELEASE, + ONET_LICENSE_IRI, + ONET_ATTRIBUTION, + source_sha256, + ) + if vocabulary_id is None: + raise ValueError("O*NET release metadata conflicts with the stored catalog") + await conn.executemany( + """ + insert into occupational_construct + (vocabulary_id, construct_iri, construct_family_code, + preferred_label, construct_description) + values ($1, $2, $3, $4, $5) + on conflict (vocabulary_id, construct_iri) do update set + construct_description = coalesce( + occupational_construct.construct_description, + excluded.construct_description + ) + where occupational_construct.construct_family_code = excluded.construct_family_code + and occupational_construct.preferred_label = excluded.preferred_label + and ( + occupational_construct.construct_description is null + or occupational_construct.construct_description = excluded.construct_description + ) + """, + [ + ( + vocabulary_id, + construct.construct_iri, + construct.family_code, + construct.preferred_label, + construct.description, + ) + for construct in constructs + ], + ) + rows = await conn.fetch( + """ + select construct_iri, construct_family_code, preferred_label, + construct_description + from occupational_construct + where vocabulary_id = $1 + """, + vocabulary_id, + ) + stored = { + str(row["construct_iri"]): ( + str(row["construct_family_code"]), + str(row["preferred_label"]), + row["construct_description"], + ) + for row in rows + } + expected = { + construct.construct_iri: ( + construct.family_code, + construct.preferred_label, + construct.description, + ) + for construct in constructs + } + if stored != expected: + raise ValueError("stored O*NET catalog differs from the official release") + return len(constructs) diff --git a/migrations/0238_occupational_construct_assertion.sql b/migrations/0238_occupational_construct_assertion.sql new file mode 100644 index 000000000..c78286ddf --- /dev/null +++ b/migrations/0238_occupational_construct_assertion.sql @@ -0,0 +1,84 @@ +-- ADR 0249: versioned occupational constructs and evidence-unit assertions. +-- Replay-safe; numerical measurement remains outside LineageWeave. + +create table if not exists occupational_construct_vocabulary ( + vocabulary_id uuid primary key default gen_random_uuid(), + vocabulary_iri text not null check (btrim(vocabulary_iri) <> ''), + version_label text not null check (btrim(version_label) <> ''), + license_iri text not null check (btrim(license_iri) <> ''), + attribution_text text not null check (btrim(attribution_text) <> ''), + created_at timestamptz not null default now(), + unique (vocabulary_iri, version_label) +); + +create table if not exists occupational_construct ( + construct_id uuid primary key default gen_random_uuid(), + vocabulary_id uuid not null references occupational_construct_vocabulary(vocabulary_id), + construct_iri text not null check (btrim(construct_iri) <> ''), + construct_family_code text not null check (construct_family_code in ( + 'cognitive_ability', + 'work_style', + 'work_activity', + 'affective_reaction', + 'performance_behavior' + )), + preferred_label text not null check (btrim(preferred_label) <> ''), + unique (vocabulary_id, construct_iri) +); + +create table if not exists post_occupational_construct_assertion ( + assertion_id uuid primary key default gen_random_uuid(), + post_id uuid not null references source_post(post_id) on delete cascade, + post_content_unit_id uuid not null references post_content_unit(post_content_unit_id) on delete cascade, + construct_id uuid not null references occupational_construct(construct_id), + evidence_text text not null check (btrim(evidence_text) <> ''), + truth_status_code text not null references common_lookup_value(lookup_code) check ( + truth_status_code in ( + 'truth_authoritative', + 'truth_observed', + 'truth_inferred', + 'truth_proposed', + 'truth_superseded', + 'truth_rejected' + ) + ), + extraction_method text not null check (btrim(extraction_method) <> ''), + orchestrator_session_id text not null check (btrim(orchestrator_session_id) <> ''), + generated_at timestamptz not null default now(), + unique (post_id, post_content_unit_id, construct_id, extraction_method) +); + +create or replace function validate_occupational_construct_evidence() +returns trigger +language plpgsql +as $$ +declare + unit_post_id uuid; + selected_unit_text text; +begin + select unit.post_id, unit.unit_text + into unit_post_id, selected_unit_text + from post_content_unit unit + where unit.post_content_unit_id = new.post_content_unit_id; + if unit_post_id is null or unit_post_id <> new.post_id then + raise exception 'occupational construct evidence unit must belong to the assertion post'; + end if; + if strpos(selected_unit_text, new.evidence_text) = 0 then + raise exception 'occupational construct evidence must be verbatim unit text'; + end if; + return new; +end; +$$; + +drop trigger if exists occupational_construct_evidence_trigger + on post_occupational_construct_assertion; +create trigger occupational_construct_evidence_trigger +before insert or update on post_occupational_construct_assertion +for each row execute function validate_occupational_construct_evidence(); + +create index if not exists occupational_construct_family_iri_idx + on occupational_construct (construct_family_code, construct_iri); +create index if not exists post_occupational_construct_post_time_idx + on post_occupational_construct_assertion (post_id, generated_at desc, assertion_id); +create index if not exists post_occupational_construct_construct_post_idx + on post_occupational_construct_assertion (construct_id, post_id); diff --git a/migrations/0239_occupational_construct_catalog.sql b/migrations/0239_occupational_construct_catalog.sql new file mode 100644 index 000000000..25e01cbfd --- /dev/null +++ b/migrations/0239_occupational_construct_catalog.sql @@ -0,0 +1,25 @@ +-- ADR 0250: preserve the official release document and construct descriptions. + +alter table occupational_construct_vocabulary + add column if not exists source_content_sha256 text; + +alter table occupational_construct_vocabulary + drop constraint if exists occupational_construct_vocabulary_source_content_sha256_check; +alter table occupational_construct_vocabulary + add constraint occupational_construct_vocabulary_source_content_sha256_check + check ( + source_content_sha256 is null + or source_content_sha256 ~ '^[0-9a-f]{64}$' + ); + +alter table occupational_construct + add column if not exists construct_description text; + +alter table occupational_construct + drop constraint if exists occupational_construct_description_nonblank_check; +alter table occupational_construct + add constraint occupational_construct_description_nonblank_check + check ( + construct_description is null + or btrim(construct_description) <> '' + ); diff --git a/scripts/publish_ontology_site.py b/scripts/publish_ontology_site.py index 71ba6918c..677ea4a92 100644 --- a/scripts/publish_ontology_site.py +++ b/scripts/publish_ontology_site.py @@ -17,7 +17,7 @@ from urllib.parse import urlsplit from rdflib import Graph, URIRef -from rdflib.namespace import OWL, RDF, RDFS, SH, SKOS +from rdflib.namespace import OWL, PROV, RDF, RDFS, SH, SKOS try: from scripts.ontology_site_contract import public_fragment @@ -33,7 +33,15 @@ #: lowercase form is the deprecated compatibility vocabulary. CANONICAL_NAMESPACE = "https://contextualwisdomlab.github.io/LineageWeave/ontology#" DEPRECATED_NAMESPACE = "https://contextualwisdomlab.github.io/lineageweave/ontology#" -STANDARD_SHACL_PATHS = frozenset({RDF.subject, RDF.predicate, RDF.object}) +STANDARD_SHACL_PATHS = frozenset( + { + RDF.subject, + RDF.predicate, + RDF.object, + PROV.wasDerivedFrom, + PROV.generatedAtTime, + } +) _MAPPING_FOR_KIND = { OWL.Class: OWL.equivalentClass, @@ -139,9 +147,9 @@ def validate_shapes_graph(shapes: Graph, canonical: Graph) -> None: """Reject SHACL shapes whose targets dangle outside the ontology. A shape that targets a class absent from the canonical graph, or - constrains a path neither declared there nor one of RDF's three - reification predicates, would silently validate nothing -- the - publication boundary refuses it instead (ADR 0207 decision 10). + constrains a path neither declared there nor an allowlisted RDF + reification/PROV-O provenance predicate, would silently validate nothing + -- the publication boundary refuses it instead (ADR 0207 decision 10). Only URI-valued targets and paths are checked; literal sh:path values are not part of this contract. """ diff --git a/scripts/sync_occupational_construct_catalog.py b/scripts/sync_occupational_construct_catalog.py new file mode 100644 index 000000000..412d8d484 --- /dev/null +++ b/scripts/sync_occupational_construct_catalog.py @@ -0,0 +1,59 @@ +"""Synchronize the fixed official O*NET construct catalog into PostgreSQL.""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +import asyncpg + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from backend.app.config import load_settings +from lineageweave.http_client import get_json +from lineageweave.occupational_construct_catalog import ( + ONET_CONTENT_MODEL_URL, + sync_onet_construct_catalog, +) + + +def _parser() -> argparse.ArgumentParser: + """Build the operator-only catalog synchronization parser.""" + parser = argparse.ArgumentParser( + description="Synchronize the governed O*NET occupational construct catalog." + ) + parser.add_argument("--target-dsn") + return parser + + +async def synchronize_catalog(target_dsn: str) -> int: + """Download the fixed release and persist it without exposing credentials.""" + payload = await asyncio.to_thread( + get_json, + ONET_CONTENT_MODEL_URL, + timeout=30.0, + service_peer_name="onet-resource-center", + maximum_response_bytes=8 * 1024 * 1024, + expected_response_media_type="application/json", + ) + conn = await asyncpg.connect(target_dsn) + try: + return await sync_onet_construct_catalog(conn, payload) + finally: + await conn.close() + + +def main() -> None: + """Parse configuration, synchronize the catalog, and print only its count.""" + args = _parser().parse_args() + settings = load_settings() + count = asyncio.run(synchronize_catalog(args.target_dsn or settings.database_url)) + print({"release": "31.0", "construct_count": count}) + + +if __name__ == "__main__": + main() diff --git a/tests/test_occupational_construct_catalog.py b/tests/test_occupational_construct_catalog.py new file mode 100644 index 000000000..61ac937ae --- /dev/null +++ b/tests/test_occupational_construct_catalog.py @@ -0,0 +1,173 @@ +"""Contracts for the official O*NET occupational construct catalog.""" + +from __future__ import annotations + +import asyncio +from contextlib import AbstractAsyncContextManager + +import pytest + +from lineageweave.occupational_construct_catalog import ( + ONET_ATTRIBUTION, + ONET_CONTENT_MODEL_CANONICAL_SHA256, + catalog_content_sha256, + parse_onet_construct_catalog, + sync_onet_construct_catalog, +) + + +def _payload() -> dict[str, object]: + return { + "table_id": "content_model_reference", + "row": [ + { + "element_id": "1.A.1.a.1", + "element_name": "Oral Comprehension", + "description": "Understand spoken words.", + }, + { + "element_id": "1.D.1", + "element_name": "Achievement Orientation", + "description": " ", + }, + { + "element_id": "4.A.1.a.1", + "element_name": "Getting Information", + "description": None, + }, + { + "element_id": "2.C.1", + "element_name": "Education", + "description": "Outside the governed roots.", + }, + ], + } + + +def test_parser_admits_only_the_three_published_hierarchy_roots() -> None: + """Published element positions, not label guesses, determine each family.""" + constructs = parse_onet_construct_catalog(_payload()) + assert [construct.family_code for construct in constructs] == [ + "cognitive_ability", + "work_style", + "work_activity", + ] + assert constructs[0].construct_iri.endswith("/1.A.1.a.1") + assert constructs[1].description is None + + +def test_parser_rejects_malformed_or_duplicate_source_rows() -> None: + """A malformed official document cannot become a partial local catalog.""" + with pytest.raises(ValueError, match="Content Model Reference"): + parse_onet_construct_catalog({"table_id": "other", "row": []}) + payload = _payload() + rows = payload["row"] + assert isinstance(rows, list) + payload["row"] = [rows[0], rows[0]] + with pytest.raises(ValueError, match="duplicate"): + parse_onet_construct_catalog(payload) + + +def test_parser_preserves_labels_and_keeps_blank_description_unavailable() -> None: + """Official labels are exact while whitespace-only descriptions stay absent.""" + payload = _payload() + constructs = parse_onet_construct_catalog(payload) + assert constructs[1].preferred_label == "Achievement Orientation" + assert constructs[1].description is None + rows = payload["row"] + assert isinstance(rows, list) + rows[0]["element_name"] = " Oral Comprehension" + with pytest.raises(ValueError, match="outer whitespace"): + parse_onet_construct_catalog(payload) + + +def test_catalog_hash_is_key_order_independent() -> None: + """Equivalent decoded JSON produces one reproducible release digest.""" + assert catalog_content_sha256({"a": 1, "b": 2}) == catalog_content_sha256( + {"b": 2, "a": 1} + ) + + +class _Transaction(AbstractAsyncContextManager[None]): + async def __aenter__(self) -> None: + return None + + async def __aexit__(self, *_args: object) -> None: + return None + + +class _RecordingConnection: + def __init__(self) -> None: + self.batch: list[tuple[object, ...]] = [] + + def transaction(self) -> _Transaction: + return _Transaction() + + async def fetchval(self, query: str, *args: object) -> str: + assert "source_content_sha256" in query + assert args[3] == ONET_ATTRIBUTION + return "vocabulary-id" + + async def executemany( + self, query: str, args: list[tuple[object, ...]] + ) -> None: + assert "construct_description" in query + self.batch = args + + async def fetch(self, _query: str, *_args: object) -> list[dict[str, object]]: + return [ + { + "construct_iri": row[1], + "construct_family_code": row[2], + "preferred_label": row[3], + "construct_description": row[4], + } + for row in self.batch + ] + + +def test_sync_uses_one_transaction_and_verifies_exact_stored_metadata() -> None: + """The operator sync persists and verifies the whole admitted catalog.""" + conn = _RecordingConnection() + payload = _payload() + assert ( + asyncio.run( + sync_onet_construct_catalog( + conn, + payload, + expected_source_sha256=catalog_content_sha256(payload), + ) + ) + == 3 + ) + assert len(conn.batch) == 3 + + +def test_sync_rejects_conflicting_stored_construct_metadata() -> None: + """A same-version label conflict aborts instead of rewriting history.""" + class ConflictingConnection(_RecordingConnection): + async def fetch( + self, query: str, *args: object + ) -> list[dict[str, object]]: + rows = await super().fetch(query, *args) + rows[0]["preferred_label"] = "Conflicting label" + return rows + + with pytest.raises(ValueError, match="differs from the official release"): + payload = _payload() + asyncio.run( + sync_onet_construct_catalog( + ConflictingConnection(), + payload, + expected_source_sha256=catalog_content_sha256(payload), + ) + ) + + +def test_sync_rejects_unreviewed_release_before_opening_a_transaction() -> None: + """A same-URL document change cannot initialize a new release silently.""" + conn = _RecordingConnection() + with pytest.raises(ValueError, match="digest differs"): + asyncio.run(sync_onet_construct_catalog(conn, _payload())) + assert conn.batch == [] + assert len(ONET_CONTENT_MODEL_CANONICAL_SHA256) == 64 diff --git a/tests/test_occupational_construct_catalog_schema.py b/tests/test_occupational_construct_catalog_schema.py new file mode 100644 index 000000000..f7f16f3fc --- /dev/null +++ b/tests/test_occupational_construct_catalog_schema.py @@ -0,0 +1,17 @@ +"""Replay contracts for the official occupational construct catalog schema.""" + +from pathlib import Path + + +MIGRATION = Path("migrations/0239_occupational_construct_catalog.sql") + + +def test_catalog_migration_is_replay_safe_and_preserves_source_integrity() -> None: + """Existing volumes can replay the catalog metadata extension safely.""" + sql = MIGRATION.read_text(encoding="utf-8").casefold() + assert "add column if not exists source_content_sha256" in sql + assert "source_content_sha256 ~ '^[0-9a-f]{64}$'" in sql + assert "add column if not exists construct_description" in sql + assert "construct_description is null" in sql + assert "btrim(construct_description) <> ''" in sql + assert "drop constraint if exists" in sql diff --git a/tests/test_occupational_construct_ontology.py b/tests/test_occupational_construct_ontology.py new file mode 100644 index 000000000..c3a9781d8 --- /dev/null +++ b/tests/test_occupational_construct_ontology.py @@ -0,0 +1,95 @@ +"""Ontology checks for evidence-bound occupational constructs (ADR 0248).""" + +from __future__ import annotations + +from rdflib import BNode, Graph, Literal, URIRef +from rdflib.namespace import OWL, PROV, RDF, RDFS, XSD + +from lineageweave.ontology import LW, load_ontology + + +def test_construct_families_are_distinct_from_worker_functions() -> None: + """Construct families share a parent but never equate to FJA functions.""" + graph = load_ontology() + families = { + LW.CognitiveAbility, + LW.WorkStyle, + LW.WorkActivity, + LW.AffectiveReaction, + LW.PerformanceBehavior, + } + for family in families: + assert (family, RDF.type, OWL.Class) in graph + assert (family, RDFS.subClassOf, LW.OccupationalConstruct) in graph + assert (family, OWL.equivalentClass, LW.WorkerFunction) not in graph + assert (LW.WorkerFunction, OWL.equivalentClass, family) not in graph + + +def test_construct_assertion_has_fixed_reified_direction() -> None: + """The schema fixes Post -> supports construct and never Person -> trait.""" + graph = load_ontology() + assert (LW.supportsOccupationalConstruct, RDFS.domain, LW.Post) in graph + assert ( + LW.supportsOccupationalConstruct, + RDFS.range, + LW.OccupationalConstruct, + ) in graph + restrictions = set(graph.objects(LW.OccupationalConstructAssertion, RDFS.subClassOf)) + assert any( + (node, OWL.onProperty, RDF.predicate) in graph + and (node, OWL.hasValue, LW.supportsOccupationalConstruct) in graph + for node in restrictions + ) + + +def test_construct_assertion_shape_requires_evidence_and_provenance() -> None: + """SHACL rejects a construct assertion without evidence and PROV metadata.""" + from pyshacl import validate + + shapes = Graph().parse("docs/ontology/lineageweave-kg-shapes.ttl", format="turtle") + ontology = load_ontology() + post = URIRef("https://example.test/post/synthetic") + construct = URIRef("https://example.test/construct/synthetic") + assertion = BNode() + data = Graph() + data.add((post, RDF.type, LW.Post)) + data.add((post, LW.postTitle, Literal("Synthetic post"))) + data.add((post, LW.postBody, Literal("Synthetic body."))) + data.add( + (post, LW.createdAt, Literal("2026-08-27T00:00:00Z", datatype=XSD.dateTime)) + ) + data.add((construct, RDF.type, LW.CognitiveAbility)) + data.add((assertion, RDF.type, LW.OccupationalConstructAssertion)) + data.add((assertion, RDF.subject, post)) + data.add((assertion, RDF.predicate, LW.supportsOccupationalConstruct)) + data.add((assertion, RDF.object, construct)) + conforms, _, _ = validate(data, shacl_graph=shapes, ont_graph=ontology) + assert not conforms + + data.add((assertion, LW.constructEvidence, Literal("Synthetic evidence."))) + data.add((assertion, PROV.wasDerivedFrom, post)) + data.add( + ( + assertion, + PROV.generatedAtTime, + Literal("2026-08-27T00:00:00Z", datatype=XSD.dateTime), + ) + ) + conforms, _, report = validate(data, shacl_graph=shapes, ont_graph=ontology) + assert conforms, report + + other_post = URIRef("https://example.test/post/other-synthetic") + data.add((other_post, RDF.type, LW.Post)) + data.add((other_post, LW.postTitle, Literal("Other synthetic post"))) + data.add((other_post, LW.postBody, Literal("Other synthetic body."))) + data.add( + ( + other_post, + LW.createdAt, + Literal("2026-08-27T00:00:00Z", datatype=XSD.dateTime), + ) + ) + data.remove((assertion, PROV.wasDerivedFrom, post)) + data.add((assertion, PROV.wasDerivedFrom, other_post)) + conforms, _, _ = validate(data, shacl_graph=shapes, ont_graph=ontology) + assert not conforms diff --git a/tests/test_occupational_construct_persistence.py b/tests/test_occupational_construct_persistence.py new file mode 100644 index 000000000..e8485a78a --- /dev/null +++ b/tests/test_occupational_construct_persistence.py @@ -0,0 +1,178 @@ +"""Persistence checks for evidence-bound occupational constructs (ADR 0249).""" + +from __future__ import annotations + +import asyncio +from contextlib import AbstractAsyncContextManager +from datetime import UTC, datetime + +import pytest + +from backend.app.occupational_construct_ingestion import ( + ConstructVocabulary, + OccupationalConstruct, + OccupationalConstructAssertion, + load_occupational_construct_assertions, + persist_occupational_construct_assertions, +) + + +class _Transaction(AbstractAsyncContextManager[None]): + async def __aenter__(self) -> None: + return None + + async def __aexit__(self, *_args: object) -> None: + return None + + +class RecordingConnection: + """Record parameterized persistence calls without a database.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, tuple[object, ...]]] = [] + self.values = iter(("vocabulary-id", "construct-id")) + self.rows: list[dict[str, object]] = [] + + def transaction(self) -> _Transaction: + return _Transaction() + + async def execute(self, query: str, *args: object) -> None: + self.calls.append((" ".join(query.split()), args)) + + async def fetchval(self, query: str, *args: object) -> str: + self.calls.append((" ".join(query.split()), args)) + return next(self.values) + + async def fetch(self, query: str, *args: object) -> list[dict[str, object]]: + self.calls.append((" ".join(query.split()), args)) + return self.rows + + +def _assertion() -> OccupationalConstructAssertion: + vocabulary = ConstructVocabulary( + "https://www.onetcenter.org/database.html", + "31.0", + "https://creativecommons.org/licenses/by/4.0/", + "O*NET 31.0 Database by USDOL/ETA.", + ) + construct = OccupationalConstruct( + vocabulary, + "https://data.onetcenter.org/element/1.A.1.a.1", + "cognitive_ability", + "Oral Comprehension", + ) + return OccupationalConstructAssertion( + "11111111-1111-1111-1111-111111111111", + "The record states a synthetic comprehension requirement.", + construct, + "synthetic comprehension requirement", + "truth_inferred", + "contextual_orchestrator_structured", + ) + + +def test_assertion_rejects_nonverbatim_evidence_and_unsafe_iris() -> None: + """Trust-boundary values fail before any SQL can run.""" + assertion = _assertion() + with pytest.raises(ValueError, match="verbatim"): + OccupationalConstructAssertion( + assertion.post_content_unit_id, + assertion.unit_text, + assertion.construct, + "not present", + assertion.truth_status_code, + assertion.extraction_method, + ) + with pytest.raises(ValueError, match="HTTPS"): + ConstructVocabulary( + "http://unsafe.example/vocabulary", + "1", + "https://example.test/license", + "Synthetic attribution", + ) + with pytest.raises(ValueError, match="truth status"): + OccupationalConstructAssertion( + assertion.post_content_unit_id, + assertion.unit_text, + assertion.construct, + assertion.evidence_text, + "truth_guessed", + assertion.extraction_method, + ) + + +def test_persistence_replaces_then_upserts_versioned_registry_and_assertion() -> None: + """One transaction performs delete, registry UPSERTs, then assertion insert.""" + conn = RecordingConnection() + asyncio.run( + persist_occupational_construct_assertions( + conn, + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + "post-session", + (_assertion(),), + ) + ) + statements = [query for query, _ in conn.calls] + assert statements[0].startswith("delete from post_occupational_construct_assertion") + assert "on conflict (vocabulary_iri, version_label) do update" in statements[1] + assert "where occupational_construct_vocabulary.license_iri" in statements[1] + assert "on conflict (vocabulary_id, construct_iri) do update" in statements[2] + assert "where occupational_construct.construct_family_code" in statements[2] + assert statements[3].startswith("insert into post_occupational_construct_assertion") + assert conn.calls[3][1][-1] == "post-session" + + +def test_conflicting_version_metadata_fails_closed() -> None: + """An existing release cannot be silently rewritten by an UPSERT.""" + conn = RecordingConnection() + conn.values = iter((None,)) + with pytest.raises(ValueError, match="immutable version"): + asyncio.run( + persist_occupational_construct_assertions( + conn, + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + "post-session", + (_assertion(),), + ) + ) + + +def test_empty_replacement_only_removes_stale_assertions() -> None: + """Unavailable analysis writes no placeholder registry or assertion rows.""" + conn = RecordingConnection() + asyncio.run( + persist_occupational_construct_assertions( + conn, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", "post-session", () + ) + ) + assert len(conn.calls) == 1 + assert conn.calls[0][0].startswith("delete from post_occupational_construct_assertion") + + +def test_authorized_projection_omits_internal_ids_and_preserves_provenance() -> None: + """The read model returns review fields in semantic-unit order.""" + conn = RecordingConnection() + conn.rows = [ + { + "construct_iri": "https://data.onetcenter.org/element/1.A.1.a.1", + "construct_family_code": "cognitive_ability", + "preferred_label": "Oral Comprehension", + "vocabulary_iri": "https://www.onetcenter.org/database.html", + "version_label": "31.0", + "evidence_text": "synthetic evidence", + "truth_status_code": "truth_inferred", + "extraction_method": "contextual_orchestrator_structured", + "generated_at": datetime(2026, 8, 27, tzinfo=UTC), + "unit_index": 2, + } + ] + result = asyncio.run( + load_occupational_construct_assertions( + conn, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + ) + ) + assert result[0]["vocabulary_version"] == "31.0" + assert result[0]["provenance"] == ( + "post_occupational_construct_assertion.evidence_text" + ) + assert "construct_id" not in result[0] diff --git a/tests/test_occupational_construct_schema.py b/tests/test_occupational_construct_schema.py new file mode 100644 index 000000000..7de22a82c --- /dev/null +++ b/tests/test_occupational_construct_schema.py @@ -0,0 +1,42 @@ +"""Static schema contracts for ADR 0249 occupational assertions.""" + +from pathlib import Path + + +MIGRATION = Path(__file__).resolve().parents[1] / "migrations/0238_occupational_construct_assertion.sql" + + +def test_construct_migration_is_normalized_replay_safe_and_indexed() -> None: + """The migration owns three replay-safe tables and both query directions.""" + sql = MIGRATION.read_text(encoding="utf-8").casefold() + for table in ( + "occupational_construct_vocabulary", + "occupational_construct", + "post_occupational_construct_assertion", + ): + assert f"create table if not exists {table}" in sql + assert "references post_content_unit(post_content_unit_id)" in sql + assert "references occupational_construct(construct_id)" in sql + assert "references common_lookup_value(lookup_code)" in sql + for truth_status in ( + "truth_authoritative", + "truth_observed", + "truth_inferred", + "truth_proposed", + "truth_superseded", + "truth_rejected", + ): + assert truth_status in sql + assert "validate_occupational_construct_evidence" in sql + assert "strpos(selected_unit_text, new.evidence_text) = 0" in sql + assert "(post_id, generated_at desc, assertion_id)" in sql + assert "(construct_id, post_id)" in sql + + +def test_construct_assertion_schema_contains_no_local_measurement() -> None: + """Persistence cannot smuggle scores, weights, or person traits into the model.""" + assertion_sql = MIGRATION.read_text(encoding="utf-8").split( + "create table if not exists post_occupational_construct_assertion", 1 + )[1].split(");", 1)[0].casefold() + for prohibited in ("score", "weight", "intensity", "importance", "person_id"): + assert prohibited not in assertion_sql diff --git a/tests/test_publish_ontology_site.py b/tests/test_publish_ontology_site.py index d0f954b37..99b60d3e3 100644 --- a/tests/test_publish_ontology_site.py +++ b/tests/test_publish_ontology_site.py @@ -297,6 +297,16 @@ def test_shapes_validation_rejects_dangling_targets_and_outside_namespace() -> N ) +def test_shapes_validation_allows_only_governed_external_statement_paths() -> None: + """RDF reification and adopted PROV paths are the external-path allowlist.""" + publisher = _load_publisher() + from rdflib.namespace import PROV + + assert PROV.wasDerivedFrom in publisher.STANDARD_SHACL_PATHS + assert PROV.generatedAtTime in publisher.STANDARD_SHACL_PATHS + assert URIRef("https://example.test/arbitrary") not in publisher.STANDARD_SHACL_PATHS + + def test_main_publishes_site(tmp_path: Path) -> None: publisher = _load_publisher() repository = _repository_fixture(tmp_path) diff --git a/tests/test_schema.py b/tests/test_schema.py index 49caf6f8c..e4b91c80e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -26,6 +26,10 @@ import pytest from backend.app.post_chat_ingestion import gather_global_chat_sources +from lineageweave.occupational_construct_catalog import ( + catalog_content_sha256, + sync_onet_construct_catalog, +) _ADMIN_DSN = os.environ.get( "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" @@ -40,6 +44,19 @@ _POST_CONTENT_MIGRATION = ( Path(__file__).resolve().parents[1] / "migrations" / "0026_post_content_artifacts.sql" ) +_ONTOLOGY_TRUTH_STATUS_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0175_ontology_truth_status.sql" +) +_OCCUPATIONAL_CONSTRUCT_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0238_occupational_construct_assertion.sql" +) +_OCCUPATIONAL_CATALOG_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0239_occupational_construct_catalog.sql" +) _SOURCE_STATE_MIGRATION = ( Path(__file__).resolve().parents[1] / "migrations" / "0033_source_state_provenance.sql" ) @@ -153,6 +170,9 @@ def schema_db(): with conn.cursor() as cur: cur.execute(_MIGRATION_PATH.read_text()) cur.execute(_POST_CONTENT_MIGRATION.read_text()) + cur.execute(_ONTOLOGY_TRUTH_STATUS_MIGRATION.read_text()) + cur.execute(_OCCUPATIONAL_CONSTRUCT_MIGRATION.read_text()) + cur.execute(_OCCUPATIONAL_CATALOG_MIGRATION.read_text()) cur.execute(_PROJECT_MENTION_MIGRATION.read_text()) cur.execute(_SOURCE_STATE_MIGRATION.read_text()) cur.execute(_SOURCE_CONTEXT_MIGRATION.read_text()) @@ -235,10 +255,87 @@ def test_migration_applies_cleanly(schema_db) -> None: "post_summary_action", "post_chat_result", "post_chat_citation", + "occupational_construct_vocabulary", + "occupational_construct", + "post_occupational_construct_assertion", } assert expected <= tables +def test_occupational_catalog_metadata_columns_exist(schema_db) -> None: + """The real schema preserves catalog descriptions and release integrity.""" + with schema_db.cursor() as cur: + cur.execute( + """ + select table_name, column_name + from information_schema.columns + where (table_name, column_name) in ( + ('occupational_construct_vocabulary', 'source_content_sha256'), + ('occupational_construct', 'construct_description') + ) + """ + ) + columns = set(cur.fetchall()) + assert columns == { + ("occupational_construct_vocabulary", "source_content_sha256"), + ("occupational_construct", "construct_description"), + } + + +def test_occupational_catalog_sync_persists_exact_rows(schema_db) -> None: + """The real PostgreSQL path atomically stores the governed catalog subset.""" + payload = { + "table_id": "content_model_reference", + "row": [ + { + "element_id": "1.A.1.a.1", + "element_name": "Synthetic cognitive ability", + "description": "Synthetic description.", + }, + { + "element_id": "1.D.1", + "element_name": "Synthetic work style", + "description": "", + }, + { + "element_id": "4.A.1", + "element_name": "Synthetic work activity", + "description": None, + }, + ], + } + + async def synchronize() -> int: + parsed_admin_dsn = urlsplit(_ADMIN_DSN) + db_dsn = urlunsplit( + parsed_admin_dsn._replace(path=f"/{schema_db.info.dbname}") + ) + conn = await asyncpg.connect(db_dsn) + try: + return await sync_onet_construct_catalog( + conn, + payload, + expected_source_sha256=catalog_content_sha256(payload), + ) + finally: + await conn.close() + + assert asyncio.run(synchronize()) == 3 + with schema_db.cursor() as cur: + cur.execute( + """ + select construct_family_code, preferred_label, construct_description + from occupational_construct + order by construct_family_code + """ + ) + assert cur.fetchall() == [ + ("cognitive_ability", "Synthetic cognitive ability", "Synthetic description."), + ("work_activity", "Synthetic work activity", None), + ("work_style", "Synthetic work style", None), + ] + + def test_global_ask_evidence_search_indexes_exist_on_normalized_tables(schema_db) -> None: """The real PostgreSQL schema owns all nine evidence-search indexes.""" with schema_db.cursor() as cur: