diff --git a/CHANGELOG.md b/CHANGELOG.md index e19d722fa..ef99773f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ All notable changes to this project are documented here. Format follows now persist in normalized tables. Database and application validation require same-Post verbatim evidence, and authorized Post detail exposes provenance without internal identifiers or numerical scores (ADR 0249). +- An operator-only O*NET 31.0 catalog synchronizer now imports every official + cognitive-ability, work-style, and work-activity Content Model element with + stable IRIs, descriptions, attribution, and a deterministic source digest; + conflicting release metadata fails closed (ADR 0250). - The DOT/FJA Data/People/Things worker-function taxonomy is now published in the canonical ontology: all 24 worker functions carry the official diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 1dee1b294..ee2dc8a93 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -239,6 +239,11 @@ / "migrations" / "0238_occupational_construct_assertion.sql" ) +_OCCUPATIONAL_CATALOG_MIGRATION = ( + Path(__file__).resolve().parents[2] + / "migrations" + / "0239_occupational_construct_catalog.sql" +) def _postgres_available() -> bool: @@ -401,6 +406,7 @@ def seeded_db(demo_analyst_token): cur.execute(_LEFTOVER_MAP_AXIS_MIGRATION.read_text()) cur.execute(_CHANNEL_EVIDENCE_MIGRATION.read_text()) cur.execute(_OCCUPATIONAL_CONSTRUCT_MIGRATION.read_text()) + cur.execute(_OCCUPATIONAL_CATALOG_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_UNEXPLAINED_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_CROSS_SHARE_MIGRATION.read_text()) cur.execute(_LEFTOVER_MAP_RECONSTRUCTION_MIGRATION.read_text()) diff --git a/docs/adr/0248-occupational-construct-evidence-boundary.md b/docs/adr/0248-occupational-construct-evidence-boundary.md index bc0d19cc5..1f4056cec 100644 --- a/docs/adr/0248-occupational-construct-evidence-boundary.md +++ b/docs/adr/0248-occupational-construct-evidence-boundary.md @@ -68,8 +68,9 @@ affective science has no single default category set. evidence relationship without claiming that the first source mention is a measured person attribute. - Complete O*NET breadth remains available through its maintained RDF graph; - runtime ingestion and persistence are still unavailable until a connector - and normalized assertion tables are accepted and shipped. + ADRs 0249 and 0250 add normalized assertion persistence and official catalog + synchronization. Record extraction remains unavailable until its separate + contextual-orchestrator contract is accepted and shipped. - Actual affect stays absent unless the evidence names a conforming affect vocabulary and supports the reaction; work style is never relabeled affect. - Unsupported equivalence and causal links fail closed rather than becoming diff --git a/docs/adr/0250-official-occupational-construct-catalog-sync.md b/docs/adr/0250-official-occupational-construct-catalog-sync.md new file mode 100644 index 000000000..b780ebd1c --- /dev/null +++ b/docs/adr/0250-official-occupational-construct-catalog-sync.md @@ -0,0 +1,69 @@ +# ADR 0250: Official occupational construct catalog synchronization + +**Status:** Accepted +**Date:** 2026-08-27 +**Extends:** [ADR 0248](0248-occupational-construct-evidence-boundary.md), [ADR 0249](0249-occupational-construct-assertion-persistence.md) + +## Context + +The assertion store accepted by ADR 0249 needs a complete, reviewable catalog +before contextual-orchestrator can select a construct. Asking a model to invent +an O*NET label or IRI would defeat ADR 0248. Shipping a hand-maintained subset +would also drift from O*NET's quarterly releases and omit most of the requested +cognitive and behavioral domain. + +O*NET 31.0 publishes a machine-readable Content Model Reference with stable +element identifiers, names, hierarchy positions, descriptions, release +documentation, and CC BY 4.0 attribution terms. Its hierarchy explicitly places +cognitive abilities below `1.A.1`, work styles below `1.D`, and work activities +below `4.A`. These are source classifications, not a LineageWeave heuristic. + +## Decision + +1. An operator-only synchronizer reads the fixed HTTPS O*NET 31.0 Content Model + Reference JSON document. The URL, release, vocabulary IRI, license IRI, and + attribution are code-reviewed constants; runtime input cannot redirect the + process to an arbitrary host. +2. The synchronizer imports every element at or below the three published + hierarchy roots: `1.A.1` as `cognitive_ability`, `1.D` as `work_style`, and + `4.A` as `work_activity`. It preserves official labels, optional descriptions, + and permanent `https://data.onetcenter.org/element/{element_id}` IRIs. +3. The canonical decoded JSON SHA-256 is stored on the vocabulary release. + The reviewed O*NET 31.0 document digest is + `cb25e83a25c355dba035afdfc6b23ed8706a939d5f5021ed772d554ea49afb06`; + synchronization rejects any other digest before opening a database + transaction. Replaying the same document is idempotent. A changed document + under the same release, or conflicting construct metadata, aborts instead + of rewriting history. The reviewed document contains 3,006 source rows and + admits 2,529 governed constructs: 29 cognitive abilities, 26 work styles, + and 2,474 work activities. +4. This catalog does not import occupation ratings, scores, scale values, + ability-to-activity linkages, work-style linkages, FJA crosswalks, affective + vocabularies, or person/job bindings. Those require their own provenance and + decision records. +5. Catalog synchronization is a prerequisite for extraction. The extractor + may select only catalog rows supplied to contextual-orchestrator; it may not + mint a label, family, description, or IRI. + +## Consequences + +- The semantic layer gains the full official breadth needed for catalog-bound + cognitive, work-style, and work-activity assertions without copying these + terms into the LineageWeave ontology namespace. +- O*NET descriptions that are absent remain `NULL`; LineageWeave does not fill + them with generated prose. +- Affective reactions and performance interpretation remain unavailable until + an authoritative vocabulary and evidence contract are accepted. + +## Verification + +- Parser tests cover all three roots, exact IRI construction, ignored unrelated + rows, malformed payloads, and deterministic source hashing. +- Schema tests require replay-safe catalog description and source-hash columns. +- Synchronization tests prove idempotent UPSERTs and post-write exact metadata + comparison. + +## References + +See +[`docs/doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md`](../doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md). diff --git a/docs/doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md b/docs/doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md index 696a1b6a8..7f7aba651 100644 --- a/docs/doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md +++ b/docs/doctoring/OCCUPATIONAL_CONSTRUCT_REFERENCES.md @@ -11,6 +11,10 @@ source actually supports and which tempting inferences remain prohibited. It is CC BY 4.0; derived products must credit USDOL/ETA, link the license, and identify modifications. LineageWeave links rather than remints these resources. +- The O*NET 31.0 Content Model Reference publishes 3,006 hierarchy elements. + ADR 0250 admits only the source-defined cognitive-ability (`1.A.1`), work- + style (`1.D`), and work-activity (`4.A`) roots and descendants; it preserves + blank descriptions as unavailable and stores no occupation rating. - The O*NET Content Model separates worker characteristics and requirements from occupational requirements. It does not make FJA worker functions equivalent to abilities, dispositions, or affect. @@ -54,6 +58,10 @@ National Center for O*NET Development. (2026). *O*NET 31.0 database* [Data set]. U.S. Department of Labor, Employment and Training Administration. https://www.onetcenter.org/database.html +National Center for O*NET Development. (2026). *O*NET 31.0 Content Model +Reference* [Data set]. U.S. Department of Labor, Employment and Training +Administration. https://www.onetcenter.org/dl_files/database/db_31_0_json/content_model_reference.json + Peterson, N. G., Mumford, M. D., Borman, W. C., Jeanneret, P. R., Fleishman, E. A., Levin, K. Y., Campion, M. A., Mayfield, M. S., Morgeson, F. P., Pearlman, K., Gowing, M. K., Lancaster, A. R., Silver, M. B., & Dye, D. M. diff --git a/docs/product-requirements.md b/docs/product-requirements.md index 147c63c9f..63a6cbf14 100644 --- a/docs/product-requirements.md +++ b/docs/product-requirements.md @@ -86,8 +86,10 @@ continues to pass unchanged. Acceptance: SHACL rejects incomplete assertions; ontology tests prohibit FJA equivalence and require exact Post/evidence/PROV statement structure. ADR 0249 adds normalized, semantic-unit-bound persistence and an authorized Post-detail -projection. Search, graph navigation, extraction, and UI remain unavailable -until their separate ADR acceptance. +projection. ADR 0250 synchronizes all official O*NET cognitive-ability, +work-style, and work-activity Content Model elements into that versioned +registry without importing ratings. Search, graph navigation, extraction, and +UI remain unavailable until their separate ADR acceptance. ### PRD-FR-3 — Bounded ontology exploration diff --git a/lineageweave/occupational_construct_catalog.py b/lineageweave/occupational_construct_catalog.py new file mode 100644 index 000000000..154190da1 --- /dev/null +++ b/lineageweave/occupational_construct_catalog.py @@ -0,0 +1,192 @@ +"""Synchronize an evidence-safe subset of the official O*NET catalog.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from typing import Any + + +ONET_RELEASE = "31.0" +ONET_CONTENT_MODEL_URL = ( + "https://www.onetcenter.org/dl_files/database/" + "db_31_0_json/content_model_reference.json" +) +ONET_CONTENT_MODEL_CANONICAL_SHA256 = ( + "cb25e83a25c355dba035afdfc6b23ed8706a939d5f5021ed772d554ea49afb06" +) +ONET_VOCABULARY_IRI = "https://www.onetcenter.org/database.html" +ONET_LICENSE_IRI = "https://creativecommons.org/licenses/by/4.0/" +ONET_ATTRIBUTION = ( + "This product includes information from the O*NET 31.0 Database by " + "the U.S. Department of Labor, Employment and Training Administration " + "(USDOL/ETA). Used under the CC BY 4.0 license. O*NET® is a trademark " + "of USDOL/ETA." +) +_ELEMENT_ID = re.compile(r"^[0-9]+(?:\.[A-Za-z0-9]+)*$") +_FAMILY_ROOTS = ( + ("1.A.1", "cognitive_ability"), + ("1.D", "work_style"), + ("4.A", "work_activity"), +) + + +@dataclass(frozen=True) +class CatalogConstruct: + """One exact O*NET Content Model element admitted by ADR 0250.""" + + construct_iri: str + family_code: str + preferred_label: str + description: str | None + + +def catalog_content_sha256(payload: dict[str, Any]) -> str: + """Hash the deterministic canonical JSON representation of one release.""" + canonical = json.dumps( + payload, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +def parse_onet_construct_catalog(payload: dict[str, Any]) -> tuple[CatalogConstruct, ...]: + """Parse only published cognitive, work-style, and work-activity roots.""" + if payload.get("table_id") != "content_model_reference": + raise ValueError("O*NET payload is not the Content Model Reference") + rows = payload.get("row") + if not isinstance(rows, list): + raise ValueError("O*NET Content Model Reference rows must be an array") + + constructs: dict[str, CatalogConstruct] = {} + for row in rows: + if not isinstance(row, dict): + raise ValueError("O*NET Content Model Reference row must be an object") + element_id = row.get("element_id") + label = row.get("element_name") + if not isinstance(element_id, str) or not _ELEMENT_ID.fullmatch(element_id): + raise ValueError("O*NET element_id is malformed") + if not isinstance(label, str) or not label.strip(): + raise ValueError("O*NET element_name must be non-empty") + if label != label.strip(): + raise ValueError("O*NET element_name must not contain outer whitespace") + family = next( + ( + family_code + for root, family_code in _FAMILY_ROOTS + if element_id == root or element_id.startswith(f"{root}.") + ), + None, + ) + if family is None: + continue + description_value = row.get("description") + if description_value is not None and not isinstance(description_value, str): + raise ValueError("O*NET description must be text or null") + description = (description_value or "").strip() or None + iri = f"https://data.onetcenter.org/element/{element_id}" + if iri in constructs: + raise ValueError(f"duplicate O*NET construct IRI: {iri}") + constructs[iri] = CatalogConstruct(iri, family, label, description) + if not constructs: + raise ValueError("O*NET catalog contains no governed construct roots") + return tuple(constructs[iri] for iri in sorted(constructs)) + + +async def sync_onet_construct_catalog( + conn: Any, + payload: dict[str, Any], + *, + expected_source_sha256: str = ONET_CONTENT_MODEL_CANONICAL_SHA256, +) -> int: + """Atomically synchronize one immutable O*NET release and verify it exactly.""" + source_sha256 = catalog_content_sha256(payload) + if source_sha256 != expected_source_sha256: + raise ValueError("O*NET 31.0 source digest differs from the reviewed release") + constructs = parse_onet_construct_catalog(payload) + async with conn.transaction(): + vocabulary_id = await conn.fetchval( + """ + insert into occupational_construct_vocabulary + (vocabulary_iri, version_label, license_iri, attribution_text, + source_content_sha256) + values ($1, $2, $3, $4, $5) + on conflict (vocabulary_iri, version_label) do update set + source_content_sha256 = coalesce( + occupational_construct_vocabulary.source_content_sha256, + excluded.source_content_sha256 + ) + where occupational_construct_vocabulary.license_iri = excluded.license_iri + and occupational_construct_vocabulary.attribution_text = excluded.attribution_text + and ( + occupational_construct_vocabulary.source_content_sha256 is null + or occupational_construct_vocabulary.source_content_sha256 = excluded.source_content_sha256 + ) + returning vocabulary_id + """, + ONET_VOCABULARY_IRI, + ONET_RELEASE, + ONET_LICENSE_IRI, + ONET_ATTRIBUTION, + source_sha256, + ) + if vocabulary_id is None: + raise ValueError("O*NET release metadata conflicts with the stored catalog") + await conn.executemany( + """ + insert into occupational_construct + (vocabulary_id, construct_iri, construct_family_code, + preferred_label, construct_description) + values ($1, $2, $3, $4, $5) + on conflict (vocabulary_id, construct_iri) do update set + construct_description = coalesce( + occupational_construct.construct_description, + excluded.construct_description + ) + where occupational_construct.construct_family_code = excluded.construct_family_code + and occupational_construct.preferred_label = excluded.preferred_label + and ( + occupational_construct.construct_description is null + or occupational_construct.construct_description = excluded.construct_description + ) + """, + [ + ( + vocabulary_id, + construct.construct_iri, + construct.family_code, + construct.preferred_label, + construct.description, + ) + for construct in constructs + ], + ) + rows = await conn.fetch( + """ + select construct_iri, construct_family_code, preferred_label, + construct_description + from occupational_construct + where vocabulary_id = $1 + """, + vocabulary_id, + ) + stored = { + str(row["construct_iri"]): ( + str(row["construct_family_code"]), + str(row["preferred_label"]), + row["construct_description"], + ) + for row in rows + } + expected = { + construct.construct_iri: ( + construct.family_code, + construct.preferred_label, + construct.description, + ) + for construct in constructs + } + if stored != expected: + raise ValueError("stored O*NET catalog differs from the official release") + return len(constructs) diff --git a/migrations/0239_occupational_construct_catalog.sql b/migrations/0239_occupational_construct_catalog.sql new file mode 100644 index 000000000..25e01cbfd --- /dev/null +++ b/migrations/0239_occupational_construct_catalog.sql @@ -0,0 +1,25 @@ +-- ADR 0250: preserve the official release document and construct descriptions. + +alter table occupational_construct_vocabulary + add column if not exists source_content_sha256 text; + +alter table occupational_construct_vocabulary + drop constraint if exists occupational_construct_vocabulary_source_content_sha256_check; +alter table occupational_construct_vocabulary + add constraint occupational_construct_vocabulary_source_content_sha256_check + check ( + source_content_sha256 is null + or source_content_sha256 ~ '^[0-9a-f]{64}$' + ); + +alter table occupational_construct + add column if not exists construct_description text; + +alter table occupational_construct + drop constraint if exists occupational_construct_description_nonblank_check; +alter table occupational_construct + add constraint occupational_construct_description_nonblank_check + check ( + construct_description is null + or btrim(construct_description) <> '' + ); diff --git a/scripts/sync_occupational_construct_catalog.py b/scripts/sync_occupational_construct_catalog.py new file mode 100644 index 000000000..412d8d484 --- /dev/null +++ b/scripts/sync_occupational_construct_catalog.py @@ -0,0 +1,59 @@ +"""Synchronize the fixed official O*NET construct catalog into PostgreSQL.""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +import asyncpg + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from backend.app.config import load_settings +from lineageweave.http_client import get_json +from lineageweave.occupational_construct_catalog import ( + ONET_CONTENT_MODEL_URL, + sync_onet_construct_catalog, +) + + +def _parser() -> argparse.ArgumentParser: + """Build the operator-only catalog synchronization parser.""" + parser = argparse.ArgumentParser( + description="Synchronize the governed O*NET occupational construct catalog." + ) + parser.add_argument("--target-dsn") + return parser + + +async def synchronize_catalog(target_dsn: str) -> int: + """Download the fixed release and persist it without exposing credentials.""" + payload = await asyncio.to_thread( + get_json, + ONET_CONTENT_MODEL_URL, + timeout=30.0, + service_peer_name="onet-resource-center", + maximum_response_bytes=8 * 1024 * 1024, + expected_response_media_type="application/json", + ) + conn = await asyncpg.connect(target_dsn) + try: + return await sync_onet_construct_catalog(conn, payload) + finally: + await conn.close() + + +def main() -> None: + """Parse configuration, synchronize the catalog, and print only its count.""" + args = _parser().parse_args() + settings = load_settings() + count = asyncio.run(synchronize_catalog(args.target_dsn or settings.database_url)) + print({"release": "31.0", "construct_count": count}) + + +if __name__ == "__main__": + main() diff --git a/tests/test_occupational_construct_catalog.py b/tests/test_occupational_construct_catalog.py new file mode 100644 index 000000000..61ac937ae --- /dev/null +++ b/tests/test_occupational_construct_catalog.py @@ -0,0 +1,173 @@ +"""Contracts for the official O*NET occupational construct catalog.""" + +from __future__ import annotations + +import asyncio +from contextlib import AbstractAsyncContextManager + +import pytest + +from lineageweave.occupational_construct_catalog import ( + ONET_ATTRIBUTION, + ONET_CONTENT_MODEL_CANONICAL_SHA256, + catalog_content_sha256, + parse_onet_construct_catalog, + sync_onet_construct_catalog, +) + + +def _payload() -> dict[str, object]: + return { + "table_id": "content_model_reference", + "row": [ + { + "element_id": "1.A.1.a.1", + "element_name": "Oral Comprehension", + "description": "Understand spoken words.", + }, + { + "element_id": "1.D.1", + "element_name": "Achievement Orientation", + "description": " ", + }, + { + "element_id": "4.A.1.a.1", + "element_name": "Getting Information", + "description": None, + }, + { + "element_id": "2.C.1", + "element_name": "Education", + "description": "Outside the governed roots.", + }, + ], + } + + +def test_parser_admits_only_the_three_published_hierarchy_roots() -> None: + """Published element positions, not label guesses, determine each family.""" + constructs = parse_onet_construct_catalog(_payload()) + assert [construct.family_code for construct in constructs] == [ + "cognitive_ability", + "work_style", + "work_activity", + ] + assert constructs[0].construct_iri.endswith("/1.A.1.a.1") + assert constructs[1].description is None + + +def test_parser_rejects_malformed_or_duplicate_source_rows() -> None: + """A malformed official document cannot become a partial local catalog.""" + with pytest.raises(ValueError, match="Content Model Reference"): + parse_onet_construct_catalog({"table_id": "other", "row": []}) + payload = _payload() + rows = payload["row"] + assert isinstance(rows, list) + payload["row"] = [rows[0], rows[0]] + with pytest.raises(ValueError, match="duplicate"): + parse_onet_construct_catalog(payload) + + +def test_parser_preserves_labels_and_keeps_blank_description_unavailable() -> None: + """Official labels are exact while whitespace-only descriptions stay absent.""" + payload = _payload() + constructs = parse_onet_construct_catalog(payload) + assert constructs[1].preferred_label == "Achievement Orientation" + assert constructs[1].description is None + rows = payload["row"] + assert isinstance(rows, list) + rows[0]["element_name"] = " Oral Comprehension" + with pytest.raises(ValueError, match="outer whitespace"): + parse_onet_construct_catalog(payload) + + +def test_catalog_hash_is_key_order_independent() -> None: + """Equivalent decoded JSON produces one reproducible release digest.""" + assert catalog_content_sha256({"a": 1, "b": 2}) == catalog_content_sha256( + {"b": 2, "a": 1} + ) + + +class _Transaction(AbstractAsyncContextManager[None]): + async def __aenter__(self) -> None: + return None + + async def __aexit__(self, *_args: object) -> None: + return None + + +class _RecordingConnection: + def __init__(self) -> None: + self.batch: list[tuple[object, ...]] = [] + + def transaction(self) -> _Transaction: + return _Transaction() + + async def fetchval(self, query: str, *args: object) -> str: + assert "source_content_sha256" in query + assert args[3] == ONET_ATTRIBUTION + return "vocabulary-id" + + async def executemany( + self, query: str, args: list[tuple[object, ...]] + ) -> None: + assert "construct_description" in query + self.batch = args + + async def fetch(self, _query: str, *_args: object) -> list[dict[str, object]]: + return [ + { + "construct_iri": row[1], + "construct_family_code": row[2], + "preferred_label": row[3], + "construct_description": row[4], + } + for row in self.batch + ] + + +def test_sync_uses_one_transaction_and_verifies_exact_stored_metadata() -> None: + """The operator sync persists and verifies the whole admitted catalog.""" + conn = _RecordingConnection() + payload = _payload() + assert ( + asyncio.run( + sync_onet_construct_catalog( + conn, + payload, + expected_source_sha256=catalog_content_sha256(payload), + ) + ) + == 3 + ) + assert len(conn.batch) == 3 + + +def test_sync_rejects_conflicting_stored_construct_metadata() -> None: + """A same-version label conflict aborts instead of rewriting history.""" + class ConflictingConnection(_RecordingConnection): + async def fetch( + self, query: str, *args: object + ) -> list[dict[str, object]]: + rows = await super().fetch(query, *args) + rows[0]["preferred_label"] = "Conflicting label" + return rows + + with pytest.raises(ValueError, match="differs from the official release"): + payload = _payload() + asyncio.run( + sync_onet_construct_catalog( + ConflictingConnection(), + payload, + expected_source_sha256=catalog_content_sha256(payload), + ) + ) + + +def test_sync_rejects_unreviewed_release_before_opening_a_transaction() -> None: + """A same-URL document change cannot initialize a new release silently.""" + conn = _RecordingConnection() + with pytest.raises(ValueError, match="digest differs"): + asyncio.run(sync_onet_construct_catalog(conn, _payload())) + assert conn.batch == [] + assert len(ONET_CONTENT_MODEL_CANONICAL_SHA256) == 64 diff --git a/tests/test_occupational_construct_catalog_schema.py b/tests/test_occupational_construct_catalog_schema.py new file mode 100644 index 000000000..f7f16f3fc --- /dev/null +++ b/tests/test_occupational_construct_catalog_schema.py @@ -0,0 +1,17 @@ +"""Replay contracts for the official occupational construct catalog schema.""" + +from pathlib import Path + + +MIGRATION = Path("migrations/0239_occupational_construct_catalog.sql") + + +def test_catalog_migration_is_replay_safe_and_preserves_source_integrity() -> None: + """Existing volumes can replay the catalog metadata extension safely.""" + sql = MIGRATION.read_text(encoding="utf-8").casefold() + assert "add column if not exists source_content_sha256" in sql + assert "source_content_sha256 ~ '^[0-9a-f]{64}$'" in sql + assert "add column if not exists construct_description" in sql + assert "construct_description is null" in sql + assert "btrim(construct_description) <> ''" in sql + assert "drop constraint if exists" in sql diff --git a/tests/test_schema.py b/tests/test_schema.py index 49caf6f8c..e4b91c80e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -26,6 +26,10 @@ import pytest from backend.app.post_chat_ingestion import gather_global_chat_sources +from lineageweave.occupational_construct_catalog import ( + catalog_content_sha256, + sync_onet_construct_catalog, +) _ADMIN_DSN = os.environ.get( "LINEAGEWEAVE_TEST_POSTGRES_ADMIN_DSN", "postgresql://localhost/postgres" @@ -40,6 +44,19 @@ _POST_CONTENT_MIGRATION = ( Path(__file__).resolve().parents[1] / "migrations" / "0026_post_content_artifacts.sql" ) +_ONTOLOGY_TRUTH_STATUS_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations" / "0175_ontology_truth_status.sql" +) +_OCCUPATIONAL_CONSTRUCT_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0238_occupational_construct_assertion.sql" +) +_OCCUPATIONAL_CATALOG_MIGRATION = ( + Path(__file__).resolve().parents[1] + / "migrations" + / "0239_occupational_construct_catalog.sql" +) _SOURCE_STATE_MIGRATION = ( Path(__file__).resolve().parents[1] / "migrations" / "0033_source_state_provenance.sql" ) @@ -153,6 +170,9 @@ def schema_db(): with conn.cursor() as cur: cur.execute(_MIGRATION_PATH.read_text()) cur.execute(_POST_CONTENT_MIGRATION.read_text()) + cur.execute(_ONTOLOGY_TRUTH_STATUS_MIGRATION.read_text()) + cur.execute(_OCCUPATIONAL_CONSTRUCT_MIGRATION.read_text()) + cur.execute(_OCCUPATIONAL_CATALOG_MIGRATION.read_text()) cur.execute(_PROJECT_MENTION_MIGRATION.read_text()) cur.execute(_SOURCE_STATE_MIGRATION.read_text()) cur.execute(_SOURCE_CONTEXT_MIGRATION.read_text()) @@ -235,10 +255,87 @@ def test_migration_applies_cleanly(schema_db) -> None: "post_summary_action", "post_chat_result", "post_chat_citation", + "occupational_construct_vocabulary", + "occupational_construct", + "post_occupational_construct_assertion", } assert expected <= tables +def test_occupational_catalog_metadata_columns_exist(schema_db) -> None: + """The real schema preserves catalog descriptions and release integrity.""" + with schema_db.cursor() as cur: + cur.execute( + """ + select table_name, column_name + from information_schema.columns + where (table_name, column_name) in ( + ('occupational_construct_vocabulary', 'source_content_sha256'), + ('occupational_construct', 'construct_description') + ) + """ + ) + columns = set(cur.fetchall()) + assert columns == { + ("occupational_construct_vocabulary", "source_content_sha256"), + ("occupational_construct", "construct_description"), + } + + +def test_occupational_catalog_sync_persists_exact_rows(schema_db) -> None: + """The real PostgreSQL path atomically stores the governed catalog subset.""" + payload = { + "table_id": "content_model_reference", + "row": [ + { + "element_id": "1.A.1.a.1", + "element_name": "Synthetic cognitive ability", + "description": "Synthetic description.", + }, + { + "element_id": "1.D.1", + "element_name": "Synthetic work style", + "description": "", + }, + { + "element_id": "4.A.1", + "element_name": "Synthetic work activity", + "description": None, + }, + ], + } + + async def synchronize() -> int: + parsed_admin_dsn = urlsplit(_ADMIN_DSN) + db_dsn = urlunsplit( + parsed_admin_dsn._replace(path=f"/{schema_db.info.dbname}") + ) + conn = await asyncpg.connect(db_dsn) + try: + return await sync_onet_construct_catalog( + conn, + payload, + expected_source_sha256=catalog_content_sha256(payload), + ) + finally: + await conn.close() + + assert asyncio.run(synchronize()) == 3 + with schema_db.cursor() as cur: + cur.execute( + """ + select construct_family_code, preferred_label, construct_description + from occupational_construct + order by construct_family_code + """ + ) + assert cur.fetchall() == [ + ("cognitive_ability", "Synthetic cognitive ability", "Synthetic description."), + ("work_activity", "Synthetic work activity", None), + ("work_style", "Synthetic work style", None), + ] + + def test_global_ask_evidence_search_indexes_exist_on_normalized_tables(schema_db) -> None: """The real PostgreSQL schema owns all nine evidence-search indexes.""" with schema_db.cursor() as cur: