From 9bc9e93d20fc10191705613356b7c8ca101c5db7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 04:54:57 +0900 Subject: [PATCH 1/2] feat(ontology): expose occupation rating evidence --- ARCHITECTURE.md | 1 + CHANGELOG.md | 5 + backend/app/main.py | 29 +++- backend/app/occupation_rating_ingestion.py | 127 +++++++++++++++ docs/adr/0258-occupation-rating-read-api.md | 49 ++++++ docs/adr/README.md | 1 + docs/product-requirements.md | 13 ++ docs/product-technical-gap-baseline.md | 2 +- tests/test_occupation_rating_ingestion.py | 162 ++++++++++++++++++++ tests/test_schema.py | 20 +++ 10 files changed, 407 insertions(+), 2 deletions(-) create mode 100644 backend/app/occupation_rating_ingestion.py create mode 100644 docs/adr/0258-occupation-rating-read-api.md create mode 100644 tests/test_occupation_rating_ingestion.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 55e00b78d..6a24bce13 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -80,6 +80,7 @@ flowchart LR | `temporal_expressions.py` | Pure Korean relative-time resolver for Global Ask (ADR 0150) | | `ask_time_axis.py` | Event-time vs ingestion-time clock choice for that window (ADR 0202) | | `ontology.py` | Loads the governed Turtle source tree (`lineageweave-kg.ttl` plus generated fragments), the formal OWL 2/RDFS/SKOS vocabulary for the Knowledge Graph's node/edge types, source taxonomies, and published O*NET linkages (ADR 0004, ADR 0252, ADR 0255, ADR 0256) | +| `backend/app/occupation_rating_ingestion.py` | Projects authenticated, bounded occupation-rating source evidence with exact decimal and artifact provenance semantics (ADR 0258) | | `ontology_neighborhood.py` | Bounded typed ontology/provenance neighborhood (ADR 0184); PostgreSQL stays authoritative, OWL subclass is not an instance edge | | `ontology_source_cursor.py` | Opaque HMAC source-window continuation (ADR 0124); keyset pagination, never OFFSET | | `period_report.py` | Fit GRM/GPCM on persisted IRT rows, FIPC-select, EAP-score a period (ADR 0003 slice 3; Bock & Mislevy, 1982) | diff --git a/CHANGELOG.md b/CHANGELOG.md index 79869185c..15193b4e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ All notable changes to this project are documented here. Format follows ### Added +- Authenticated occupation profiles can now read one exact imported + release/source with rating and scale artifact provenance, exact decimal + strings, uncertainty, suppression/relevance flags, and explicit unavailable + versus empty outcomes. The endpoint derives no ranking or recommendation + (ADR 0258). - O*NET occupation-rating source evidence now has a replay-safe PostgreSQL contract with normalized release, source-table, scale, occupation, element, and observation tables. Exact release/source LIST partitions fail closed; diff --git a/backend/app/main.py b/backend/app/main.py index 6457bbde1..60dd9e1a5 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -30,7 +30,7 @@ import asyncpg import redis.asyncio as redis -from fastapi import Depends, FastAPI, HTTPException, Query, status +from fastapi import Depends, FastAPI, HTTPException, Path, Query, status from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel @@ -98,6 +98,7 @@ upsert_commitment_ticket, ) from backend.app.operations_dashboard import fetch_operations_dashboard +from backend.app.occupation_rating_ingestion import fetch_occupation_ratings from backend.app.keyman_ingestion import ingest_post_keymen from backend.app.knowledge_graph import ( corporate_entity_exists, @@ -2275,6 +2276,32 @@ async def read_ontology_neighborhood( return payload +@app.get("/api/occupations/{onetsoc_code}/ratings") +async def read_occupation_ratings( + onetsoc_code: str = Path(..., pattern=r"^[0-9]{2}-[0-9]{4}\.[0-9]{2}$"), + data_release_code: str = Query( + ..., min_length=1, max_length=63, pattern=r"^[a-z0-9][a-z0-9.-]*$" + ), + source_table_code: str = Query( + ..., min_length=1, max_length=63, pattern=r"^[a-z][a-z0-9_]*$" + ), + limit: int = Query(100, ge=1, le=500), + offset: int = Query(0, ge=0, le=10000), + _account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, object]: + """Return one authenticated, provenance-bearing occupation source profile.""" + async with pool.acquire() as conn: + return await fetch_occupation_ratings( + conn, + data_release_code=data_release_code, + source_table_code=source_table_code, + onetsoc_code=onetsoc_code, + limit=limit, + offset=offset, + ) + + @app.get("/api/posts/{post_id}/counterparties") async def read_post_counterparties( post_id: str, diff --git a/backend/app/occupation_rating_ingestion.py b/backend/app/occupation_rating_ingestion.py new file mode 100644 index 000000000..320fe248d --- /dev/null +++ b/backend/app/occupation_rating_ingestion.py @@ -0,0 +1,127 @@ +"""Read exact imported occupation ratings without deriving a score or weight.""" + +from __future__ import annotations + +from decimal import Decimal +from typing import Any, Protocol + + +class RatingReadConnection(Protocol): + """Small asyncpg-compatible surface used by the rating read projection.""" + + async def fetchrow(self, query: str, *args: object) -> Any: + """Return one row or ``None``.""" + + async def fetch(self, query: str, *args: object) -> list[Any]: + """Return ordered rows.""" + + +def _decimal_text(value: Decimal | None) -> str | None: + """Return the exact database decimal representation or honest absence.""" + return str(value) if value is not None else None + + +async def fetch_occupation_ratings( + conn: RatingReadConnection, + *, + data_release_code: str, + source_table_code: str, + onetsoc_code: str, + limit: int, + offset: int, +) -> dict[str, object]: + """Return one bounded source profile and explicit artifact availability.""" + source = await conn.fetchrow( + """select rating_source.source_table_name, + rating_source.source_artifact_url, + rating_source.source_artifact_sha256, + rating_source.source_row_count, + scale_source.source_artifact_url as scale_artifact_url, + scale_source.source_artifact_sha256 as scale_artifact_sha256, + scale_source.source_row_count as scale_source_row_count + from occupational_source_table rating_source + left join occupational_source_table scale_source + on scale_source.data_release_code = rating_source.data_release_code + and scale_source.source_table_code = 'scales_reference' + where rating_source.data_release_code = $1 + and rating_source.source_table_code = $2""", + data_release_code, + source_table_code, + ) + if source is None: + return { + "data_release_code": data_release_code, + "source_table_code": source_table_code, + "onetsoc_code": onetsoc_code, + "source_available": False, + "source": None, + "items": [], + "next_offset": None, + } + rows = await conn.fetch( + """select observation.element_id, element.element_name, + observation.scale_id, scale.scale_name, + scale.minimum_value, scale.maximum_value, + observation.category_value, observation.data_value, + observation.sample_size, observation.standard_error, + observation.lower_ci_bound, observation.upper_ci_bound, + observation.recommend_suppress, observation.not_relevant, + observation.source_updated_month, observation.domain_source_code + from occupational_rating_observation observation + join occupational_element_definition element + on element.data_release_code = observation.data_release_code + and element.element_id = observation.element_id + join occupational_scale_definition scale + on scale.data_release_code = observation.data_release_code + and scale.scale_id = observation.scale_id + where observation.data_release_code = $1 + and observation.source_table_code = $2 + and observation.onetsoc_code = $3 + order by observation.element_id, observation.scale_id, + observation.category_value nulls first + limit $4 offset $5""", + data_release_code, + source_table_code, + onetsoc_code, + limit + 1, + offset, + ) + page = rows[:limit] + items = [ + { + "element_id": row["element_id"], + "element_name": row["element_name"], + "scale_id": row["scale_id"], + "scale_name": row["scale_name"], + "minimum_value": _decimal_text(row["minimum_value"]), + "maximum_value": _decimal_text(row["maximum_value"]), + "category_value": row["category_value"], + "data_value": _decimal_text(row["data_value"]), + "sample_size": row["sample_size"], + "standard_error": _decimal_text(row["standard_error"]), + "lower_ci_bound": _decimal_text(row["lower_ci_bound"]), + "upper_ci_bound": _decimal_text(row["upper_ci_bound"]), + "recommend_suppress": row["recommend_suppress"], + "not_relevant": row["not_relevant"], + "source_updated_month": row["source_updated_month"], + "domain_source_code": row["domain_source_code"], + } + for row in page + ] + return { + "data_release_code": data_release_code, + "source_table_code": source_table_code, + "onetsoc_code": onetsoc_code, + "source_available": True, + "source": { + "source_table_name": source["source_table_name"], + "source_artifact_url": source["source_artifact_url"], + "source_artifact_sha256": source["source_artifact_sha256"], + "source_row_count": source["source_row_count"], + "scale_artifact_url": source["scale_artifact_url"], + "scale_artifact_sha256": source["scale_artifact_sha256"], + "scale_source_row_count": source["scale_source_row_count"], + }, + "items": items, + "next_offset": offset + limit if len(rows) > limit else None, + } diff --git a/docs/adr/0258-occupation-rating-read-api.md b/docs/adr/0258-occupation-rating-read-api.md new file mode 100644 index 000000000..a22f5bab1 --- /dev/null +++ b/docs/adr/0258-occupation-rating-read-api.md @@ -0,0 +1,49 @@ +# ADR 0258: Authenticated occupation-rating source read API + +**Status:** Accepted +**Date:** 2026-08-27 +**Extends:** ADR 0120, ADR 0184, ADR 0257 + +## Context + +ADR 0257 preserves released occupation-to-element observations, but a database +import alone does not let a product user inspect what a job profile says. A +read contract must distinguish an unimported source from an imported source +with no row for one occupation, preserve low-precision and not-relevant flags, +and avoid presenting a published rating as a local weight or recommendation. + +## Decision + +1. Add an authenticated, read-only occupation-rating endpoint. O*NET source + observations are licensed public reference data and are not tenant records; + any authenticated LineageWeave account may read an imported artifact. +2. Require exact release, source-table, and O*NET-SOC codes. Return + `source_available=false` when that pinned artifact is not imported; return + `source_available=true` with an empty item list when it is imported but has + no observation for the requested occupation. +3. Return the rating and Scales Reference artifact URLs, SHA-256 values, and + row counts. Every observation retains element/scale identity, declared + bounds, optional category, exact decimal strings, sample/error/interval, + suppression, relevance, source month, and domain source. +4. Order by element, scale, and category and use bounded offset pagination. + Per-occupation source partitions bound this projection; a cursor needs a + later decision only if measured production latency requires it. +5. Do not aggregate, rank, normalize, infer person traits, or recommend an + occupation. Suppressed values remain visible with the suppression flag so a + user can audit the source without mistaking low precision for absence. +6. API and frontend copy describe the evidence and the user's next action, + never importer, partition, model-provider, or orchestration internals. + +## Consequences + +The semantic layer gains an honest product read boundary without duplicating +psychometric arithmetic. An accessible UI and its Storybook states remain a +separate delivery step after this API has authenticated runtime evidence. + +## References + +National Center for O*NET Development. (2026). *O*NET 31.0 database* [Data +set]. https://www.onetcenter.org/database.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: +Queries—limit and offset*. https://www.postgresql.org/docs/current/queries-limit.html diff --git a/docs/adr/README.md b/docs/adr/README.md index 74e74592c..1a36fa1a8 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -34,6 +34,7 @@ decision from them. | [`SOC_2018_HIERARCHY_REFERENCES.md`](../doctoring/SOC_2018_HIERARCHY_REFERENCES.md) | [0252](0252-complete-2018-soc-hierarchy.md) | | [`ONET_31_LINKAGE_REFERENCES.md`](../doctoring/ONET_31_LINKAGE_REFERENCES.md) | [0256](0256-onet-content-model-published-linkages.md) | | [`ONET_RATING_STORE_REFERENCES.md`](../doctoring/ONET_RATING_STORE_REFERENCES.md) | [0257](0257-onet-occupation-rating-observation-store.md) | +| Occupation-rating authenticated read projection | [0258](0258-occupation-rating-read-api.md) | [0011](0011-prov-o-standard-relations.md) and [0065](0065-prov-o-provenance-boundary.md) cite the dated W3C PROV-O and PROV-DM Recommendations (https://www.w3.org/TR/2013/REC-prov-o-20130430/ and https://www.w3.org/TR/2013/REC-prov-dm-20130430/). diff --git a/docs/product-requirements.md b/docs/product-requirements.md index 7e3e116e5..d84b284b0 100644 --- a/docs/product-requirements.md +++ b/docs/product-requirements.md @@ -148,6 +148,19 @@ repeated null-category UPSERT is idempotent. API, UI, and derived modeling remain unavailable until separate accepted delivery records. +### PRD-FR-2E — Occupation-rating evidence read + +- Let an authenticated user open one exact release/source/occupation profile + with both rating and scale artifact provenance (ADR 0258). +- Distinguish an unavailable imported source from an available source with no + observation for the occupation. +- Preserve exact decimal text, uncertainty, suppression, relevance, source + month, domain source, and declared bounds; derive no ranking or recommendation. + +Acceptance: invalid identifiers and unbounded pages are rejected; an unavailable +source never appears as a negative profile; pagination is deterministic; and a +suppressed observation retains its value and warning flag together. + ### PRD-FR-3 — Bounded ontology exploration - Apply RBAC/ABAC, source eligibility, and knowledge cutoff before graph diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 51638152f..bd7e5a884 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -29,7 +29,7 @@ explicit unavailable state, not a reason to infer mappings from labels. | Occupation-to-construct relations | ADR 0257 defines a candidate 3NF, release/source-partitioned immutable observation store and deterministic pinned-CSV importer preserving value, optional category, sample/error/CI, suppression, relevance, exact `MM/YYYY` source update month, source digest, and domain provenance. The official O*NET 31.0 Abilities file (94,640 rows, 910 occupations, 52 elements; SHA-256 `7e9cd79791ce6014e1d26d0a449ae5b1e7aa7ef52d39b3934c3bb8d438104b88`) and all 33 Scales Reference rows (SHA-256 `bcba23858ce21ecaacbde303a8993e35d46724b4afb8c9ec2b10e04f42adcfc9`) imported into a throwaway local PostgreSQL database with all 94,640 observations, 55 suppression flags, 7,572 not-relevant flags, and source months from `12/2004` through `08/2026`; every scale retained `scales_reference` artifact provenance, the database was dropped afterward, and no corpus is committed or claimed deployed | Pass exact-head review/checks and protected merge; validate and import every selected official rating artifact through an authorized runtime, returning only aggregate evidence; never invent or locally normalize a weight | | Job-family and job-series semantics | No authoritative employer-specific job architecture is present | Define an organization-neutral import contract that preserves the authorized source hierarchy and distinguishes standard occupation codes from employer job families/series; no label-based binding | | Temporal and multilevel interpretation | Static vocabulary only; no person-level inference is asserted | Version valid and transaction time, preserve occupation/organization/unit nesting and multiple membership, and require TEPP or the owning Rust psychometric service before any calibrated temporal or multilevel result | -| Product consumption | The read model has no persisted semantic-layer consumer or authenticated UI evidence | Add a provenance-bearing API and accessible ontology exploration flow, then verify synthetic Storybook edge states plus authenticated aggregate runtime evidence without exposing identifying records | +| Product consumption | ADR 0258 defines a candidate authenticated occupation-rating API that distinguishes unavailable source from empty profile and returns exact decimal/uncertainty/warning/artifact provenance without ranking; accessible UI and authenticated runtime evidence remain absent | Pass exact-head review/checks and protected merge; add the accessible exploration UI, synthetic Storybook unavailable/empty/suppressed/paginated states, screenshots, and authenticated aggregate runtime evidence without exposing identifying records | ### Current exact-head PR queue diff --git a/tests/test_occupation_rating_ingestion.py b/tests/test_occupation_rating_ingestion.py new file mode 100644 index 000000000..3f2244d7f --- /dev/null +++ b/tests/test_occupation_rating_ingestion.py @@ -0,0 +1,162 @@ +"""Tests for the provenance-bearing occupation-rating read projection.""" + +import asyncio +from decimal import Decimal + +from backend.app.main import read_occupation_ratings +from backend.app.occupation_rating_ingestion import fetch_occupation_ratings + + +class FakeConnection: + """Minimal ordered asyncpg stand-in for one projection query.""" + + def __init__(self, source, rows=()): + self.source = source + self.rows = list(rows) + self.fetch_called = False + + async def fetchrow(self, _query: str, *_args: object): + """Return configured source metadata.""" + return self.source + + async def fetch(self, _query: str, *_args: object): + """Return configured observation rows.""" + self.fetch_called = True + return self.rows + + +class FakeAcquire: + """Async pool-acquire context for route wiring.""" + + def __init__(self, conn: FakeConnection): + self.conn = conn + + async def __aenter__(self) -> FakeConnection: + """Return the configured connection.""" + return self.conn + + async def __aexit__(self, *_args: object) -> None: + """Release without external state.""" + + +class FakePool: + """Minimal pool exposing one acquisition context.""" + + def __init__(self, conn: FakeConnection): + self.conn = conn + + def acquire(self) -> FakeAcquire: + """Return one deterministic acquisition context.""" + return FakeAcquire(self.conn) + + +def test_unimported_source_is_not_an_empty_observed_profile() -> None: + conn = FakeConnection(None) + + result = asyncio.run( + fetch_occupation_ratings( + conn, + data_release_code="onet-31.0", + source_table_code="abilities", + onetsoc_code="15-1252.00", + limit=100, + offset=0, + ) + ) + + assert result["source_available"] is False + assert result["items"] == [] + assert conn.fetch_called is False + + +def test_rating_projection_preserves_exact_decimal_and_warning_flags() -> None: + source = { + "source_table_name": "Abilities", + "source_artifact_url": "https://example.test/abilities.csv", + "source_artifact_sha256": "a" * 64, + "source_row_count": 2, + "scale_artifact_url": "https://example.test/scales.csv", + "scale_artifact_sha256": "b" * 64, + "scale_source_row_count": 33, + } + row = { + "element_id": "1.A.1.a.1", + "element_name": "Oral Comprehension", + "scale_id": "IM", + "scale_name": "Importance", + "minimum_value": Decimal("1.00"), + "maximum_value": Decimal("5.00"), + "category_value": None, + "data_value": Decimal("4.10"), + "sample_size": 8, + "standard_error": Decimal("0.1830"), + "lower_ci_bound": Decimal("3.7414"), + "upper_ci_bound": Decimal("4.4586"), + "recommend_suppress": True, + "not_relevant": None, + "source_updated_month": "08/2026", + "domain_source_code": "Analyst", + } + conn = FakeConnection(source, (row, row)) + + result = asyncio.run( + fetch_occupation_ratings( + conn, + data_release_code="onet-31.0", + source_table_code="abilities", + onetsoc_code="15-1252.00", + limit=1, + offset=0, + ) + ) + + item = result["items"][0] + assert item["data_value"] == "4.10" + assert item["standard_error"] == "0.1830" + assert item["recommend_suppress"] is True + assert item["not_relevant"] is None + assert result["source"]["scale_artifact_sha256"] == "b" * 64 + assert result["next_offset"] == 1 + + +def test_empty_profile_keeps_imported_scale_provenance() -> None: + source = { + "source_table_name": "Abilities", + "source_artifact_url": "https://example.test/abilities.csv", + "source_artifact_sha256": "a" * 64, + "source_row_count": 2, + "scale_artifact_url": "https://example.test/scales.csv", + "scale_artifact_sha256": "b" * 64, + "scale_source_row_count": 33, + } + + result = asyncio.run( + fetch_occupation_ratings( + FakeConnection(source), + data_release_code="onet-31.0", + source_table_code="abilities", + onetsoc_code="15-9999.99", + limit=100, + offset=0, + ) + ) + + assert result["source_available"] is True + assert result["items"] == [] + assert result["source"]["scale_artifact_sha256"] == "b" * 64 + + +def test_authenticated_route_delegates_to_bounded_projection() -> None: + result = asyncio.run( + read_occupation_ratings( + onetsoc_code="15-1252.00", + data_release_code="onet-31.0", + source_table_code="abilities", + limit=100, + offset=0, + _account=object(), + pool=FakePool(FakeConnection(None)), + ) + ) + + assert result["source_available"] is False diff --git a/tests/test_schema.py b/tests/test_schema.py index d6cf67712..49dacb2a2 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -28,6 +28,7 @@ import psycopg2.errors import pytest +from backend.app.occupation_rating_ingestion import fetch_occupation_ratings from backend.app.post_chat_ingestion import gather_global_chat_sources from scripts.import_onet_ratings import import_ratings @@ -463,6 +464,25 @@ def test_onet_rating_importer_is_idempotent_against_postgresql( ) assert cur.fetchone() == (1, Decimal("4.10"), True) + async def read_imported_profile() -> dict[str, object]: + conn = await asyncpg.connect(args.target_dsn) + try: + return await fetch_occupation_ratings( + conn, + data_release_code=args.release_code, + source_table_code=args.source_table_code, + onetsoc_code="15-1252.00", + limit=100, + offset=0, + ) + finally: + await conn.close() + + profile = asyncio.run(read_imported_profile()) + assert profile["source_available"] is True + assert profile["items"][0]["data_value"] == "4.10" + assert profile["source"]["scale_artifact_sha256"] == args.scales_sha256 + def test_global_ask_evidence_search_indexes_exist_on_normalized_tables(schema_db) -> None: """The real PostgreSQL schema owns all nine evidence-search indexes.""" From e6a1d6217fb9491c6c15882573e94090c2ede071 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 27 Aug 2026 06:00:13 +0900 Subject: [PATCH 2/2] docs: keep occupation rating ADR lint-clean --- docs/adr/0258-occupation-rating-read-api.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/adr/0258-occupation-rating-read-api.md b/docs/adr/0258-occupation-rating-read-api.md index a22f5bab1..7af31c817 100644 --- a/docs/adr/0258-occupation-rating-read-api.md +++ b/docs/adr/0258-occupation-rating-read-api.md @@ -1,7 +1,8 @@ # ADR 0258: Authenticated occupation-rating source read API -**Status:** Accepted -**Date:** 2026-08-27 +**Status:** Accepted + +**Date:** 2026-08-27 **Extends:** ADR 0120, ADR 0184, ADR 0257 ## Context