diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7cbe3697f..75683aa63 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -86,7 +86,9 @@ flowchart LR | `commitment_extraction.py` | Pluggable LLM derivation of a customer commitment (promise + deadline) from a post; `Null` default, `ContextualOrchestrator` real impl | | `temporal_expressions.py` | Pure Korean relative-time resolver for Global Ask (ADR 0150) | | `ask_time_axis.py` | Event-time vs ingestion-time clock choice for that window (ADR 0202) | -| `ontology.py` | Loads `docs/ontology/lineageweave-kg.ttl`, the formal OWL 2/RDFS/SKOS vocabulary for the Knowledge Graph's node/edge types (ADR 0004) | +| `ontology.py` | Loads the governed Turtle source tree (`lineageweave-kg.ttl` plus generated fragments), the formal OWL 2/RDFS/SKOS vocabulary for the Knowledge Graph's node/edge types, source taxonomies, and published O*NET linkages (ADR 0004, ADR 0252, ADR 0255, ADR 0256) | +| `backend/app/occupation_rating_ingestion.py` | Projects authenticated occupation-rating evidence plus persisted source and represented-occupation catalogs (ADR 0258, ADR 0260, ADR 0261) | +| `frontend/src/components/OccupationRatingProfile.tsx` | Selects an imported source, filters stored occupation titles without ranking, and reads exact Dashboard evidence while preserving absence, uncertainty, and warning semantics (ADR 0259–0262) | | `ontology_neighborhood.py` | Bounded typed ontology/provenance neighborhood (ADR 0184); PostgreSQL stays authoritative, OWL subclass is not an instance edge | | `ontology_source_cursor.py` | Opaque HMAC source-window continuation (ADR 0124); keyset pagination, never OFFSET | | `period_report.py` | Fit GRM/GPCM on persisted IRT rows, FIPC-select, EAP-score a period (ADR 0003 slice 3; Bock & Mislevy, 1982) | @@ -679,7 +681,8 @@ vocabulary (`node_type`, `edge_type`, `entity_relationship_type`, `person_side`, `corporate_entity_level`) actually matches what the Ontology/Semantic-Layer claim implies. -`docs/ontology/lineageweave-kg.ttl` is a real OWL 2 / RDFS / SKOS +`docs/ontology/lineageweave-kg.ttl` and its deterministic governed fragments +are a real OWL 2 / RDFS / SKOS ontology in Turtle syntax: classes for `Post`/`Person`/`CorporateEntity` (with `OurSidePerson`/`CounterpartyPerson` subclasses), object properties for each `edge_type_code` and `entity_relationship_type` @@ -693,7 +696,7 @@ specification over it, in the same sense W3C's own stack uses "semantic layer" (RDFS/OWL as the governed conceptual layer over raw data), not a separate BI-metrics product and not a parallel triple store. -`lineageweave/ontology.py` parses the Turtle file once with `rdflib` +`lineageweave/ontology.py` parses the Turtle source tree once with `rdflib` (pure Python, no Rust toolchain, unlike `fast-mlsirm`) and exposes the vocabulary as importable IRI constants, so application code has one canonical name per class/property instead of re-typing lookup codes as @@ -710,6 +713,19 @@ enforcement mechanism: a future PR that adds a new `edge_type` or `entity_relationship_type` code without updating the ontology fails this test, not just a docstring's word. +### Authorized job architecture snapshots + +The public SOC/O*NET vocabulary and an employer's job architecture remain +different graphs. ADR 0263 adds an organization-scoped PostgreSQL source +boundary for private job-family/job-series snapshots: immutable source +metadata owns normalized nodes, source-declared broader/narrower edges, and +optional explicit bindings to a versioned external occupation scheme. An edge +table preserves multiple-family membership; the importer rejects cycles and +never derives a parent or binding from a label or code pattern. The snapshot +is source evidence only. It does not create a person, post, organizational +unit, competency, score, weight, or ontology assertion, and runtime rows never +enter repository artifacts. + ## Phase 6c: post content normalization before any LLM/embedding call The brief's latest revision calls out, explicitly, that a post body mixing diff --git a/CHANGELOG.d/2.21.0-occupation-catalog-filter.md b/CHANGELOG.d/2.21.0-occupation-catalog-filter.md new file mode 100644 index 000000000..bf62b2c40 --- /dev/null +++ b/CHANGELOG.d/2.21.0-occupation-catalog-filter.md @@ -0,0 +1,5 @@ +### Added + +- Occupation evidence now filters the imported occupation catalog by published + title or retained code without ranking or typed SOC fallback, and fails closed + when the filter matches nothing (ADR 0262). diff --git a/backend/app/main.py b/backend/app/main.py index 12036e8f5..6481d2378 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -30,7 +30,7 @@ import asyncpg import redis.asyncio as redis -from fastapi import Depends, FastAPI, HTTPException, Query, status +from fastapi import Depends, FastAPI, HTTPException, Path, Query, status from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel @@ -154,6 +154,11 @@ fetch_post_evaluation, ingest_post_evaluation, ) +from backend.app.occupation_rating_ingestion import ( + fetch_occupation_rating_sources, + fetch_occupation_ratings, + fetch_rating_source_occupations, +) from backend.app.post_summary_ingestion import ( fetch_persisted_summary, persist_post_summary, @@ -2484,6 +2489,62 @@ async def read_worker_function_construct_catalog( return construct_catalog_payload() +@app.get("/api/occupations/{onetsoc_code}/ratings") +async def read_occupation_ratings( + onetsoc_code: str = Path(..., pattern=r"^[0-9]{2}-[0-9]{4}\.[0-9]{2}$"), + data_release_code: str = Query( + ..., min_length=1, max_length=63, pattern=r"^[a-z0-9][a-z0-9.-]*$" + ), + source_table_code: str = Query( + ..., min_length=1, max_length=63, pattern=r"^[a-z][a-z0-9_]*$" + ), + limit: int = Query(100, ge=1, le=500), + offset: int = Query(0, ge=0, le=10000), + _account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, object]: + """Return one authenticated, provenance-bearing occupation source profile.""" + async with pool.acquire() as conn: + return await fetch_occupation_ratings( + conn, + data_release_code=data_release_code, + source_table_code=source_table_code, + onetsoc_code=onetsoc_code, + limit=limit, + offset=offset, + ) + + +@app.get("/api/occupation-rating-sources") +async def read_occupation_rating_sources( + _account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, list[dict[str, object]]]: + """Return the authenticated catalog of imported occupation-rating sources.""" + async with pool.acquire() as conn: + return await fetch_occupation_rating_sources(conn) + + +@app.get("/api/occupation-rating-occupations") +async def read_rating_source_occupations( + data_release_code: str = Query( + ..., min_length=1, max_length=63, pattern=r"^[a-z0-9][a-z0-9.-]*$" + ), + source_table_code: str = Query( + ..., min_length=1, max_length=63, pattern=r"^[a-z][a-z0-9_]*$" + ), + _account: CurrentAccount = Depends(get_current_account), + pool: asyncpg.Pool = Depends(get_pool), +) -> dict[str, object]: + """Return occupations represented in one imported rating source.""" + async with pool.acquire() as conn: + return await fetch_rating_source_occupations( + conn, + data_release_code=data_release_code, + source_table_code=source_table_code, + ) + + @app.get("/api/posts/{post_id}/counterparties") async def read_post_counterparties( post_id: str, diff --git a/backend/app/occupation_rating_ingestion.py b/backend/app/occupation_rating_ingestion.py new file mode 100644 index 000000000..f7d2d10a1 --- /dev/null +++ b/backend/app/occupation_rating_ingestion.py @@ -0,0 +1,198 @@ +"""Read exact imported occupation ratings without deriving a score or weight.""" + +from __future__ import annotations + +from decimal import Decimal +from typing import Any, Protocol + + +class RatingReadConnection(Protocol): + """Small asyncpg-compatible surface used by the rating read projection.""" + + async def fetchrow(self, query: str, *args: object) -> Any: + """Return one row or ``None``.""" + + async def fetch(self, query: str, *args: object) -> list[Any]: + """Return ordered rows.""" + + +def _decimal_text(value: Decimal | None) -> str | None: + """Return the exact database decimal representation or honest absence.""" + return str(value) if value is not None else None + + +async def fetch_occupation_ratings( + conn: RatingReadConnection, + *, + data_release_code: str, + source_table_code: str, + onetsoc_code: str, + limit: int, + offset: int, +) -> dict[str, object]: + """Return one bounded source profile and explicit artifact availability.""" + source = await conn.fetchrow( + """select rating_source.source_table_name, + rating_source.source_artifact_url, + rating_source.source_artifact_sha256, + rating_source.source_row_count, + scale_source.source_artifact_url as scale_artifact_url, + scale_source.source_artifact_sha256 as scale_artifact_sha256, + scale_source.source_row_count as scale_source_row_count + from occupational_source_table rating_source + left join occupational_source_table scale_source + on scale_source.data_release_code = rating_source.data_release_code + and scale_source.source_table_code = 'scales_reference' + where rating_source.data_release_code = $1 + and rating_source.source_table_code = $2""", + data_release_code, + source_table_code, + ) + if source is None: + return { + "data_release_code": data_release_code, + "source_table_code": source_table_code, + "onetsoc_code": onetsoc_code, + "source_available": False, + "source": None, + "items": [], + "next_offset": None, + } + rows = await conn.fetch( + """select observation.element_id, element.element_name, + observation.scale_id, scale.scale_name, + scale.minimum_value, scale.maximum_value, + observation.category_value, observation.data_value, + observation.sample_size, observation.standard_error, + observation.lower_ci_bound, observation.upper_ci_bound, + observation.recommend_suppress, observation.not_relevant, + observation.source_updated_month, observation.domain_source_code + from occupational_rating_observation observation + join occupational_element_definition element + on element.data_release_code = observation.data_release_code + and element.element_id = observation.element_id + join occupational_scale_definition scale + on scale.data_release_code = observation.data_release_code + and scale.scale_id = observation.scale_id + where observation.data_release_code = $1 + and observation.source_table_code = $2 + and observation.onetsoc_code = $3 + order by observation.element_id, observation.scale_id, + observation.category_value nulls first + limit $4 offset $5""", + data_release_code, + source_table_code, + onetsoc_code, + limit + 1, + offset, + ) + page = rows[:limit] + items = [ + { + "element_id": row["element_id"], + "element_name": row["element_name"], + "scale_id": row["scale_id"], + "scale_name": row["scale_name"], + "minimum_value": _decimal_text(row["minimum_value"]), + "maximum_value": _decimal_text(row["maximum_value"]), + "category_value": row["category_value"], + "data_value": _decimal_text(row["data_value"]), + "sample_size": row["sample_size"], + "standard_error": _decimal_text(row["standard_error"]), + "lower_ci_bound": _decimal_text(row["lower_ci_bound"]), + "upper_ci_bound": _decimal_text(row["upper_ci_bound"]), + "recommend_suppress": row["recommend_suppress"], + "not_relevant": row["not_relevant"], + "source_updated_month": row["source_updated_month"], + "domain_source_code": row["domain_source_code"], + } + for row in page + ] + return { + "data_release_code": data_release_code, + "source_table_code": source_table_code, + "onetsoc_code": onetsoc_code, + "source_available": True, + "source": { + "source_table_name": source["source_table_name"], + "source_artifact_url": source["source_artifact_url"], + "source_artifact_sha256": source["source_artifact_sha256"], + "source_row_count": source["source_row_count"], + "scale_artifact_url": source["scale_artifact_url"], + "scale_artifact_sha256": source["scale_artifact_sha256"], + "scale_source_row_count": source["scale_source_row_count"], + }, + "items": items, + "next_offset": offset + limit if len(rows) > limit else None, + } + + +async def fetch_occupation_rating_sources( + conn: RatingReadConnection, +) -> dict[str, list[dict[str, object]]]: + """Return imported rating artifacts that contain at least one observation.""" + rows = await conn.fetch( + """select source.data_release_code, release.release_version, + release.source_publisher_name, release.source_license_url, + source.source_table_code, source.source_table_name, + source.source_artifact_url, source.source_artifact_sha256, + source.source_row_count + from occupational_source_table source + join occupational_data_release release + on release.data_release_code = source.data_release_code + where source.source_table_code <> 'scales_reference' + and exists ( + select 1 + from occupational_rating_observation observation + where observation.data_release_code = source.data_release_code + and observation.source_table_code = source.source_table_code + ) + order by release.imported_at desc, source.data_release_code, + source.source_table_name, source.source_table_code""" + ) + return {"sources": [dict(row) for row in rows]} + + +async def fetch_rating_source_occupations( + conn: RatingReadConnection, + *, + data_release_code: str, + source_table_code: str, +) -> dict[str, object]: + """Return occupations with observations in one exact imported source.""" + source = await conn.fetchrow( + """select 1 + from occupational_source_table + where data_release_code = $1 and source_table_code = $2""", + data_release_code, + source_table_code, + ) + if source is None: + return { + "data_release_code": data_release_code, + "source_table_code": source_table_code, + "source_available": False, + "occupations": [], + } + rows = await conn.fetch( + """select classification.onetsoc_code, classification.occupation_title + from occupational_classification_entry classification + where classification.data_release_code = $1 + and exists ( + select 1 + from occupational_rating_observation observation + where observation.data_release_code = classification.data_release_code + and observation.source_table_code = $2 + and observation.onetsoc_code = classification.onetsoc_code + ) + order by classification.occupation_title, + classification.onetsoc_code""", + data_release_code, + source_table_code, + ) + return { + "data_release_code": data_release_code, + "source_table_code": source_table_code, + "source_available": True, + "occupations": [dict(row) for row in rows], + } diff --git a/docs/adr/0159-published-ontology-pages.md b/docs/adr/0159-published-ontology-pages.md index 30caec60b..543e32b37 100644 --- a/docs/adr/0159-published-ontology-pages.md +++ b/docs/adr/0159-published-ontology-pages.md @@ -40,12 +40,17 @@ and consumer migration plan. 3. Publish equivalent machine-readable artifacts beside the HTML: `ontology.ttl`, `ontology.jsonld`, `ontology.nt`, the PROV-O support profile, and a source-digest manifest. -4. Preserve `lineageweave-kg.ttl` byte-for-byte as the published Turtle - artifact. JSON-LD and N-Triples are generated from a canonicalized RDF graph - and are tested for semantic isomorphism with the source. +4. For a single governed Turtle source, preserve `lineageweave-kg.ttl` + byte-for-byte as the published Turtle artifact. When a later accepted ADR + adds governed fragments, publish the merged graph as deterministic canonical + RDF that parses as Turtle; test every machine format for semantic + isomorphism with the complete source graph. Never concatenate independent + Turtle documents because their prefix and base declarations have + document-local scope. 5. Do not add a build timestamp. The same source tree must produce the same - artifact bytes. The manifest records the source SHA-256 and the complete - published ontology-directory inventory instead. + artifact bytes. The manifest records every governed source path and SHA-256, + the ordered source-tree SHA-256, and the complete published + ontology-directory inventory instead. 6. Run publication through `scripts/publish_ontology_site.py`, a fail-closed boundary that rejects duplicate HTML fragments, non-HTTP(S) linked IRIs, symlink outputs, source-overlapping outputs, and replacement of directories diff --git a/docs/adr/0245-io-occupational-taxonomy-in-the-published-ontology.md b/docs/adr/0245-io-occupational-taxonomy-in-the-published-ontology.md index 69c58e968..a8476f33d 100644 --- a/docs/adr/0245-io-occupational-taxonomy-in-the-published-ontology.md +++ b/docs/adr/0245-io-occupational-taxonomy-in-the-published-ontology.md @@ -3,6 +3,10 @@ **Status:** Accepted **Date:** 2026-08-26 **Extends:** [ADR 0004](0004-knowledge-graph-ontology.md), [ADR 0145](0145-psychometric-channel-weight-estimation.md), [ADR 0207](0207-repository-case-ontology-namespace-canonical.md), [ADR 0232](0232-worker-function-taxonomy-in-the-published-ontology.md) +<<<<<<< HEAD +======= +**Superseded in part by:** [ADR 0252](0252-complete-2018-soc-hierarchy.md), which expands the major-group-only scheme into the complete 2018 SOC hierarchy. +>>>>>>> origin/feat/onet-rating-occupation-filter ## Context diff --git a/docs/adr/0257-onet-occupation-rating-observation-store.md b/docs/adr/0257-onet-occupation-rating-observation-store.md new file mode 100644 index 000000000..8e00e13a8 --- /dev/null +++ b/docs/adr/0257-onet-occupation-rating-observation-store.md @@ -0,0 +1,89 @@ +# ADR 0257: O*NET occupation-rating observation store + +**Status:** Accepted +**Date:** 2026-08-27 +**Extends:** ADR 0166, ADR 0255, ADR 0256 + +## Context + +O*NET 31.0 publishes occupation-specific ratings for content-model abilities, essential and +transferable skills, knowledge, education, training and experience, interests, +work styles, work activities, work context, and adjacent content-model +domains. A rating is not merely an edge: its meaning depends on the release, +source table, occupation, element, scale, optional response category, sample +size, standard error, 95% confidence interval, precision-suppression flag, +relevance flag, source update month, and domain source. O*NET publishes that +field as a seven-character `MM/YYYY` value; coercing it to a database date +would invent a day (National Center for O*NET Development, 2026b). + +Flattening those attributes into ontology predicates would erase measurement +and provenance boundaries. Loading them into Python mathematical code would +also violate LineageWeave's externalized-compute boundary. + +## Decision + +1. Store source releases, source tables, scales, occupations, content-model + elements, and rating observations in separate third-normal-form tables. +2. Preserve published numeric values exactly as decimal observations. They are + source ratings, never locally estimated weights, person scores, causal + effects, or calibrated psychometric parameters. +3. An observation key is release + source table + occupation + element + scale + + optional category. PostgreSQL `UNIQUE NULLS NOT DISTINCT` keeps an absent + category an honest single absence rather than replacing it with a sentinel. +4. Partition observations first by release and then by source-table code. + These are authoritative lifecycle/query boundaries and require no invented + hash modulus. An importer must create both exact LIST partitions before + inserting; without them PostgreSQL rejects the row. +5. Every insert uses the owning release/table artifact digest and idempotent + `ON CONFLICT DO NOTHING`. An exact duplicate is idempotent, while a row with + the same identity and different source values fails closed. Endpoint names + and scale names must match their normalized reference + rows; each scale definition names its owning source-table artifact, and the + importer rejects disagreement rather than overwriting identity. +6. Preserve `recommend_suppress` and `not_relevant` independently. A suppressed + value remains stored with its warning; a not-relevant value is not converted + to zero. Missing `n`, error, or interval values remain null. +7. Range and uncertainty constraints reject negative sample/error values, + inverted confidence intervals, malformed or future source update months, malformed source + digests, and values outside their declared scale bounds before persistence. Scale bounds + govern the published `Data Value`, not the optional response-category code. In particular, + O*NET's `CXP` rows store a category in `Category` while `Data Value` is the percentage that + endorsed it, so the authoritative `CXP` bounds are 0 through 100 (National Center for + O*NET Development, 2026b). +8. This content-model-element store excludes Task Ratings, whose integer Task + IDs and task-statement identity require a separate normalized target table; + it does not reinterpret a Task ID as a content-model element. +9. This store is immutable source evidence. Row mutation and whole-store + truncation fail closed. Any later aggregation, comparison, + temporal model, multilevel model, or occupational recommendation belongs to + TEPP/fast-mlsirm or another owning Rust service and must cite these rows. +10. `scripts/import_onet_ratings.py` accepts one caller-pinned official CSV and + Scales Reference file. It verifies both artifact SHA-256 values and row + counts, exact scale names and bounds, reference-name consistency, finite + decimals, Y/N/blank flags, optional Category and Not Relevant columns, + exact `MM/YYYY` source months, and observation-key uniqueness before opening the + target connection. A transaction-scoped advisory lock serializes one + release's partition DDL. + +## Consequences + +LineageWeave can import the governed public O*NET content-model rating corpus without +manufacturing semantics or embedding large production datasets in git. +Release/source partitions localize hot imports and permit exact detach/archive +operations. The same pinned artifact is idempotent; a reused release, source, +scale, occupation, or element identity with different source metadata fails +closed. A separate API/UI ADR is still required before exposing ratings. + +## References + +National Center for O*NET Development. (2026a). *O*NET 31.0 database* [Data +set]. https://www.onetcenter.org/database.html + +National Center for O*NET Development. (2026b). *Work context: O*NET 31.0 data +dictionary*. https://www.onetcenter.org/dictionary/31.0/excel/work_context.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: +Table partitioning*. https://www.postgresql.org/docs/current/ddl-partitioning.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: +CREATE TABLE*. https://www.postgresql.org/docs/current/sql-createtable.html diff --git a/docs/adr/0258-occupation-rating-read-api.md b/docs/adr/0258-occupation-rating-read-api.md new file mode 100644 index 000000000..7af31c817 --- /dev/null +++ b/docs/adr/0258-occupation-rating-read-api.md @@ -0,0 +1,50 @@ +# ADR 0258: Authenticated occupation-rating source read API + +**Status:** Accepted + +**Date:** 2026-08-27 +**Extends:** ADR 0120, ADR 0184, ADR 0257 + +## Context + +ADR 0257 preserves released occupation-to-element observations, but a database +import alone does not let a product user inspect what a job profile says. A +read contract must distinguish an unimported source from an imported source +with no row for one occupation, preserve low-precision and not-relevant flags, +and avoid presenting a published rating as a local weight or recommendation. + +## Decision + +1. Add an authenticated, read-only occupation-rating endpoint. O*NET source + observations are licensed public reference data and are not tenant records; + any authenticated LineageWeave account may read an imported artifact. +2. Require exact release, source-table, and O*NET-SOC codes. Return + `source_available=false` when that pinned artifact is not imported; return + `source_available=true` with an empty item list when it is imported but has + no observation for the requested occupation. +3. Return the rating and Scales Reference artifact URLs, SHA-256 values, and + row counts. Every observation retains element/scale identity, declared + bounds, optional category, exact decimal strings, sample/error/interval, + suppression, relevance, source month, and domain source. +4. Order by element, scale, and category and use bounded offset pagination. + Per-occupation source partitions bound this projection; a cursor needs a + later decision only if measured production latency requires it. +5. Do not aggregate, rank, normalize, infer person traits, or recommend an + occupation. Suppressed values remain visible with the suppression flag so a + user can audit the source without mistaking low precision for absence. +6. API and frontend copy describe the evidence and the user's next action, + never importer, partition, model-provider, or orchestration internals. + +## Consequences + +The semantic layer gains an honest product read boundary without duplicating +psychometric arithmetic. An accessible UI and its Storybook states remain a +separate delivery step after this API has authenticated runtime evidence. + +## References + +National Center for O*NET Development. (2026). *O*NET 31.0 database* [Data +set]. https://www.onetcenter.org/database.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: +Queries—limit and offset*. https://www.postgresql.org/docs/current/queries-limit.html diff --git a/docs/adr/0259-occupation-rating-evidence-ui.md b/docs/adr/0259-occupation-rating-evidence-ui.md new file mode 100644 index 000000000..efcf08726 --- /dev/null +++ b/docs/adr/0259-occupation-rating-evidence-ui.md @@ -0,0 +1,52 @@ +# ADR 0259: Occupation-rating evidence in the existing Dashboard + +- Status: Accepted +- Date: 2026-08-27 +- Extends: ADR 0183, ADR 0206, ADR 0258 +- Figma file ID: `1Su3lDRmiZdcUs47t1QwIX` + +## Context + +ADR 0258 makes an exact imported occupation profile readable, but an API does +not let an authenticated user find a published work characteristic or notice +that a value has low precision. ADR 0183 fixes the analyst GNB and prohibits a +new destination for every evidence type. The existing Dashboard is the place +for evidence-oriented next actions and already owns responsive table and form +tokens under ADR 0206. + +## Decision + +1. Add the occupation profile below the existing operations evidence on the + Dashboard. Do not add or rename a GNB destination. +2. Require the user to submit an exact O*NET-SOC code, data release, and source + table. Native form validation rejects malformed occupation codes before a + request; the API remains the trust-boundary validator. +3. Show each exact published value beside its declared scale bounds, optional + category, sample size, standard error, confidence interval, source month, + domain source, suppression warning, and not-relevant flag. Do not calculate + a score, rank, weight, trait estimate, or recommendation. +4. Keep `source unavailable` distinct from `occupation has no observations`. + Both states give a next action instead of displaying zero or a blank table. +5. Link the rating artifact and scale definition. The API carries their + digests and row counts for provenance; a later disclosure control may show + those identifiers when user research demonstrates that it aids the task. +6. Reuse the existing Dashboard Figma file, design tokens, native controls, + responsive overflow, focus behavior, and reduced-motion baseline. The table + has a named keyboard-focusable region and every warning is text, not color. +7. Storybook records populated, narrow, source-unavailable, and empty-profile + scenes using synthetic records only. Runtime screenshot review covers the + populated desktop and narrow scenes. + +## Consequences + +Users can inspect source evidence without confusing absence, low precision, or +not-relevant responses with a negative occupational conclusion. The interface +does not introduce a local psychometric or inference implementation. + +## References + +National Center for O*NET Development. (2026). *O*NET 31.0 database* [Data +set]. https://www.onetcenter.org/database.html + +World Wide Web Consortium. (2024). *Web Content Accessibility Guidelines +(WCAG) 2.2* (W3C Recommendation). https://www.w3.org/TR/WCAG22/ diff --git a/docs/adr/0260-occupation-rating-source-catalog.md b/docs/adr/0260-occupation-rating-source-catalog.md new file mode 100644 index 000000000..93f80ce54 --- /dev/null +++ b/docs/adr/0260-occupation-rating-source-catalog.md @@ -0,0 +1,41 @@ +# ADR 0260: Imported occupation-rating source catalog + +- Status: Accepted +- Date: 2026-08-27 +- Extends: ADR 0257, ADR 0258, ADR 0259 + +## Context + +ADR 0259 initially requires users to type internal release and source-table +codes. That makes a valid product action depend on repository knowledge and +allows a user to request a source that was never imported. The normalized +rating store already owns the exact imported artifact catalog and therefore is +the only authoritative selector source. + +## Decision + +1. Add an authenticated read endpoint that lists rating artifacts containing + at least one persisted occupation observation. Exclude the Scales Reference + support artifact from selectable rating sources. +2. Return release code/version, publisher and license, source code/name, URL, + SHA-256, and declared row count. Order releases by persisted import time and + sources by stored name/code; do not infer recency from a version string. +3. The Dashboard selects only an entry returned by this endpoint. If the + catalog is loading, empty, or unavailable, disable profile submission and + give the user a next action. Do not retain a hidden hand-written fallback. +4. Authentication matches ADR 0258: imported O*NET artifacts are public + reference data, while the catalog still requires a valid workspace account. +5. The catalog does not claim that all official O*NET artifacts are imported. + It describes only current database state with immutable artifact provenance. + +## Consequences + +The occupation evidence workflow no longer asks users to know storage codes, +and an unavailable artifact cannot masquerade as a selectable source. Adding +an official artifact remains an importer operation with digest and row-count +validation rather than a UI-created catalog row. + +## References + +National Center for O*NET Development. (2026). *O*NET 31.0 database* [Data +set]. https://www.onetcenter.org/database.html diff --git a/docs/adr/0261-rating-source-occupation-selector.md b/docs/adr/0261-rating-source-occupation-selector.md new file mode 100644 index 000000000..8a3c73ddf --- /dev/null +++ b/docs/adr/0261-rating-source-occupation-selector.md @@ -0,0 +1,45 @@ +# ADR 0261: Occupations represented by an imported rating source + +- Status: Accepted +- Date: 2026-08-27 +- Extends: ADR 0257, ADR 0258, ADR 0260 + +## Context + +The source catalog removes internal release/source entry, but ADR 0260 still +leaves users to type an O*NET-SOC code. The normalized store already preserves +the source occupation title and code. A release may contain occupations that +are absent from one rating artifact, so the release classification alone is +not sufficient evidence that a profile exists for the selected source. + +## Decision + +1. Add an authenticated read endpoint returning stored O*NET-SOC code/title + pairs that have at least one observation in one exact imported rating + source. Keep unavailable source distinct from an available empty source. +2. Join by normalized release/code identity and an observation-existence + predicate. Do not bind occupations by title similarity, keyword inference, + external search, or a locally reconstructed classification. +3. Order by the stored occupation title and then code. Return the complete + represented set because the official imported classification is the + authoritative finite selector domain; do not introduce an arbitrary result + cutoff that makes valid occupations disappear. +4. Replace free-text occupation-code entry with a native select whose visible + label begins with the stored title and retains the exact code. Changing the + rating source clears both occupation selection and displayed evidence; + changing the occupation clears displayed evidence. +5. While the occupation catalog is loading, empty, or unavailable, disable + profile submission and state the next action. Pagination remains bound to + the identifiers returned by the loaded profile under ADR 0259. + +## Consequences + +Users choose an occupation by its authoritative title without knowing an +internal code, while API requests continue to carry exact stable identifiers. +Employer job families and series remain outside this selector until their +separate authorized import contract exists. + +## References + +National Center for O*NET Development. (2026). *O*NET 31.0 database* [Data +set]. https://www.onetcenter.org/database.html diff --git a/docs/adr/0262-occupation-catalog-title-filter.md b/docs/adr/0262-occupation-catalog-title-filter.md new file mode 100644 index 000000000..57b2d8f65 --- /dev/null +++ b/docs/adr/0262-occupation-catalog-title-filter.md @@ -0,0 +1,41 @@ +# ADR 0262: Occupation catalog title filter + +- Status: Accepted +- Date: 2026-08-27 +- Extends: ADR 0259, ADR 0260, ADR 0261 + +## Context + +ADR 0261 replaced typed O*NET-SOC entry with a native select of occupations +that have observations in the chosen source. An official rating artifact can +cover hundreds of occupations, so a user still cannot find a published title +without scanning the full catalog. A free-typed code would reintroduce the +gap ADR 0261 closed. + +## Decision + +1. Keep the occupation control as a native select populated only from the + imported occupation catalog for the selected source. +2. Add a native search field that filters that catalog by case-insensitive + substring of the published title or retained O*NET-SOC code. Do not rank, + boost, or infer similarity. +3. If the filter matches no catalog row, disable profile submission and give + a next action. If the current selection leaves the filtered set, move to + the first remaining catalog identity or clear the selection. +4. Reset the filter when the source or occupation catalog reloads. Never + submit a value that is not in the loaded catalog. +5. Authentication, provenance, and fail-closed unavailable/empty catalog + states remain ADR 0261. + +## Consequences + +A user can find a published occupation by title without typing an internal +code and without treating filter order as a recommendation. + +## References + +National Center for O*NET Development. (2026). *O*NET 31.0 database* [Data +set]. https://www.onetcenter.org/database.html + +World Wide Web Consortium. (2024). *Web Content Accessibility Guidelines +(WCAG) 2.2* (W3C Recommendation). https://www.w3.org/TR/WCAG22/ diff --git a/docs/adr/0263-authorized-job-architecture-import.md b/docs/adr/0263-authorized-job-architecture-import.md new file mode 100644 index 000000000..9802ed1b4 --- /dev/null +++ b/docs/adr/0263-authorized-job-architecture-import.md @@ -0,0 +1,71 @@ +# ADR 0263: Authorized job-family and job-series snapshot import + +**Status:** Accepted +**Date:** 2026-08-27 +**Extends:** ADR 0001, ADR 0065, ADR 0248, ADR 0252 + +## Context + +The official SOC and O*NET classifications describe occupations; ADR 0252 +therefore prohibits treating them as an employer's job family, job series, or +position. The product nevertheless needs to preserve an authorized +organization's own job architecture without deriving a crosswalk from codes or +labels and without committing its records. + +W3C ORG separates a role taxonomy from a person, membership, organization, and +post, and recommends SKOS for taxonomic role structures. PROV-O separates an +entity's generation/invalidation history from domain validity. OPM's handbook +also distinguishes occupational groups, series, positions, and job-family +classification standards. These authorities support separate identities and +source assertions; none authorizes a universal employer crosswalk. + +## Decision + +1. Import only a caller-authorized, SHA-256-pinned source snapshot into four + third-normal-form tables: source, node, hierarchy edge, and explicit + occupation binding. Runtime records stay outside git; repository tests use + synthetic rows only. +2. A node is exactly `job_family` or `job_series`. It is never an SOC/O*NET + occupation, organizational unit, person, position, competency, or measured + trait. The source code, label, description, and optional validity dates are + preserved without normalization. +3. Hierarchy is an edge table, not a parent column. This preserves a + source-declared series in multiple families and rejects missing endpoints, + self-links, and cycles. No label, code shape, lexical similarity, embedding, + or LLM may create an edge. +4. An occupation binding exists only when the source supplies the scheme IRI, + scheme version, occupation code, and relation code together. A title that + resembles an occupation code remains unbound. +5. Snapshots are immutable system-time evidence. A changed source requires a + new snapshot code; divergent reuse of a snapshot/node/edge/binding identity + fails through immutable-update triggers. Optional `valid_from`/`valid_to` + record source validity and never invent missing dates. +6. The corporate entity must already exist. Imports neither create an + organization nor infer authorization. Entity-first indexes bound the + organization-scoped read path; physical partitioning is deferred until + observed cardinality or lock evidence justifies a non-arbitrary boundary. +7. This contract publishes no person assignment, recommendation, competency + score, importance weight, psychometric estimate, or causal claim. Those need + their owning authorization and measurement decisions. + +## Consequences + +LineageWeave can represent employer-specific family/series structure without +polluting the public occupational vocabulary or leaking runtime data. Multiple +membership and temporal validity remain source evidence. API, UI, RDF +projection, and person/post binding remain unavailable until separate accepted +decisions define authorization and customer actions. + +## Verification + +- `tests/test_import_job_architecture.py` proves multiple membership, explicit + binding, no label binding, cycle rejection, and incomplete-source rejection. +- `tests/test_job_architecture_schema.py` pins normalized identities, + immutability, kind separation, and occupation-scheme separation. +- PostgreSQL integration must prove replay-safe migration, idempotent identical + import, and divergent snapshot rejection before protected delivery. + +## References + +See +[`docs/doctoring/JOB_ARCHITECTURE_REFERENCES.md`](../doctoring/JOB_ARCHITECTURE_REFERENCES.md). diff --git a/docs/doctoring/JOB_ARCHITECTURE_REFERENCES.md b/docs/doctoring/JOB_ARCHITECTURE_REFERENCES.md new file mode 100644 index 000000000..de04d47a7 --- /dev/null +++ b/docs/doctoring/JOB_ARCHITECTURE_REFERENCES.md @@ -0,0 +1,22 @@ +# Job architecture source-boundary references + +## References (APA 7th) + +U.S. Office of Personnel Management. (2018). *Handbook of occupational groups +and families*. https://www.opm.gov/policy-data-oversight/classification-qualifications/classifying-general-schedule-positions/occupationalhandbook.pdf + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology*. +https://www.w3.org/TR/2013/REC-prov-o-20130430/ + +World Wide Web Consortium. (2014). *The organization ontology*. +https://www.w3.org/TR/2014/REC-vocab-org-20140116/ + +## Adoption boundary + +W3C ORG supplies the separation between role taxonomies, posts, memberships, +people, and organizations and recommends SKOS for role taxonomies. PROV-O +supplies source-generation and invalidation provenance. OPM demonstrates that +occupational groups, series, positions, and job-family standards are distinct +classification objects. None of these sources supplies an employer-to-SOC or +employer-to-O*NET mapping; therefore only caller-supplied explicit bindings are +persisted. diff --git a/docs/doctoring/ONET_RATING_STORE_REFERENCES.md b/docs/doctoring/ONET_RATING_STORE_REFERENCES.md new file mode 100644 index 000000000..6258ed2b2 --- /dev/null +++ b/docs/doctoring/ONET_RATING_STORE_REFERENCES.md @@ -0,0 +1,26 @@ +# O*NET occupation-rating store evidence + +This supporting note records the source and database capabilities governed by +ADR 0257. It introduces no independent architecture decision. + +O*NET 31.0 occupation data tables publish occupation and element identifiers, +scale identifiers and names, decimal values, optional category values, sample +sizes, standard errors, confidence bounds, suppression/relevance flags, update +dates, and domain sources. These fields remain source observations; they are +not locally estimated psychometric weights. + +PostgreSQL declarative partitioning provides exact LIST boundaries and +partition pruning. A unique constraint on a partitioned table includes its +partition key; `UNIQUE NULLS NOT DISTINCT` makes a missing category one stable +identity without a sentinel value. + +## APA 7 references + +National Center for O*NET Development. (2026). *O*NET 31.0 database* [Data +set]. https://www.onetcenter.org/database.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: +Table partitioning*. https://www.postgresql.org/docs/current/ddl-partitioning.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: +CREATE TABLE*. https://www.postgresql.org/docs/current/sql-createtable.html diff --git a/docs/product-requirements.md b/docs/product-requirements.md index df2d15bce..177836d50 100644 --- a/docs/product-requirements.md +++ b/docs/product-requirements.md @@ -163,6 +163,180 @@ canonical namespace, and lookup round-trip isolation are enforced by `tests/test_io_taxonomy.py`; `tests/test_ontology.py` continues to pass unchanged. +### PRD-FR-2A — Worker-function taxonomy + +- Publish the DOT/FJA Data/People/Things worker functions (24 concepts, + official definitions verbatim) in the canonical ontology namespace + (ADR 0232), each with its definitional ordinal rank. Do not infer a + DOT-to-O*NET or Fleishman crosswalk that the authorities do not publish. +- Expose the taxonomy through a deterministic application read model with + fail-closed lookups; an absent function is an honest unknown. +- Carry no numeric weight from the taxonomy: ranks are scale positions, + never calibrated weights. + +Acceptance: completeness, full verbatim definitions, deterministic ordering, +and lookup round-trip isolation are enforced by +`tests/test_worker_function_taxonomy.py`; `tests/test_ontology.py` +continues to pass unchanged. + +### PRD-FR-2B — Occupational classification and worker-characteristic taxonomy + +- Publish all four levels of the 2018 Standard Occupational Classification: + 23 major groups, 98 minor groups, 459 broad occupations, and 867 detailed + occupations with exact source parents, titles, and codes (ADR 0252), plus + the four O*NET 31.0 job-zone categories with + published names and source values 2 through 5 (ADR 0245). +- Publish the worker-characteristic families that work-related + cognition, affect, and behavior resolve into: Fleishman's four ability + domains, Holland's six RIASEC interest types with the published + hexagonal adjacency relation, the six explicitly legacy O*NET work-value + clusters, and + the seven higher-order dimensions of the revised O*NET Work Styles + structure. +- Publish all 3,006 O*NET 31.0 Content Model Reference elements with exact + identifiers, names, descriptions, and source-defined outline parents + (ADR 0264). Treat the six roots and 18 second-level branches as navigation + classes, never occupation ratings, person traits, scores, or weights. +- Declare typed derivation properties from classifications to + characteristics but assert no instance binding; binding requires a + versioned released source profile imported with provenance in its own + decision. +- Expose everything through a deterministic application read model with + fail-closed lookups; carry no numeric importance or level rating from + any occupational profile. + +Acceptance: completeness counts, verbatim titles, closed RIASEC +vocabulary, exact published adjacency pairs, deterministic ordering, +canonical namespace, and lookup round-trip isolation are enforced by +`tests/test_io_taxonomy.py`, `tests/test_soc_2018_hierarchy.py`, and +`tests/test_onet_content_model.py`; +`tests/test_ontology.py` continues to pass unchanged. +### PRD-FR-2C — Evidence-bound occupational constructs + +- Keep cognitive abilities, work styles, work activities, affective + reactions, and performance behaviors as non-equivalent construct classes + (ADR 0248). FJA worker functions remain separate. +- Reuse official external identifiers and source-published relationships; + never infer a DPT-to-psychology crosswalk or relabel work style as affect. +- Publish the eight O*NET 31.0 Ability, Essential Skill, Transferable Skill, + and Work Style link tables to Work Activities and Work Context as 1,417 + directed, assertion-level provenance-bearing relations (ADR 0256). Treat + relevance as neither a causal effect nor a numeric weight. +- Bind a construct to record content only through a provenance-bearing, + evidence-cited assertion. Do not promote record evidence to a person trait, + score, causal effect, or job requirement. + +Acceptance: SHACL rejects incomplete record assertions; ontology tests +prohibit FJA equivalence, require exact Post/evidence/PROV statement structure, +and reproduce every pinned O*NET linkage with its exact source table. Runtime +persistence and UI remain unavailable until their separate ADR acceptance. + +### PRD-FR-2D — Occupation-rating source observations + +- Persist released occupation-to-element ratings as source observations, not + ontology weights: release, source table, occupation, element, scale, + optional category, value, sample/error/interval, suppression, relevance, + exact source update month, and domain source remain independently auditable + (ADR 0257); the product must not invent a day for O*NET's `MM/YYYY` field. +- Keep normalized reference identities in third normal form and partition the + observation store by exact release then source table. An unknown partition + fails closed instead of entering a catch-all table. +- Preserve decimals and missingness exactly. No local aggregation, + normalization, person inference, or psychometric estimation is permitted. +- Reject divergent duplicate identities and owner-level truncation. Task + Ratings remain unavailable until their integer Task IDs and statements have + a separate normalized source-target contract. + +Acceptance: the replay-safe migration creates the normalized store; the pinned +CSV importer validates both rating and scale-reference digests and row counts, +reference identity, source scale, uncertainty, flags, and dates before +persistence; PostgreSQL integration proves missing partitions fail closed and +repeated null-category UPSERT is idempotent. +API, UI, and derived modeling remain unavailable until separate accepted +delivery records. + +### PRD-FR-2E — Occupation-rating evidence read + +- Let an authenticated user open one exact release/source/occupation profile + with both rating and scale artifact provenance (ADR 0258). +- Distinguish an unavailable imported source from an available source with no + observation for the occupation. +- Preserve exact decimal text, uncertainty, suppression, relevance, source + month, domain source, and declared bounds; derive no ranking or recommendation. + +Acceptance: invalid identifiers and unbounded pages are rejected; an unavailable +source never appears as a negative profile; pagination is deterministic; and a +suppressed observation retains its value and warning flag together. + +### PRD-FR-2F — Occupation-rating evidence view + +- Let an authenticated user submit an exact O*NET-SOC code, release, and source + from the existing Dashboard without changing the governed GNB (ADR 0259). +- Display published values beside bounds, sample/error/interval evidence, + source time, and text warnings; link both source artifacts. +- Give different next actions for unavailable source, empty occupation, + transport failure, and additional pages. + +Acceptance: keyboard users can operate the form and named horizontally +scrollable table; narrow layouts retain complete values; suppression remains +visible beside its value; and Storybook covers populated, narrow, unavailable, +and empty states using synthetic data. + +### PRD-FR-2G — Imported rating-source catalog + +- Populate the occupation evidence selector only from imported artifacts that + contain observations, preserving release and artifact provenance (ADR 0260). +- Exclude the scale-definition support artifact from the rating-source selector. +- Disable profile submission and state the next action while the catalog is + loading, empty, or unavailable. + +Acceptance: a user never types an internal release/source code; the selector +order follows persisted import time rather than parsed version heuristics; and +the real PostgreSQL integration test proves an imported synthetic artifact is +listed while its supporting scale artifact is not. + +### PRD-FR-2H — Occupations represented in a rating source + +- Populate the occupation selector with exact stored code/title pairs that + have observations in the selected imported source (ADR 0261). +- Clear the current occupation and profile when the source changes, and clear + the profile when the occupation changes; never mix continuation rows across + occupations or sources. +- Keep unavailable source, available-empty source, loading, and transport + failure distinct and actionable. + +Acceptance: a user selects a stored title rather than typing an internal code; +the PostgreSQL integration test proves the source membership predicate; and +component tests prove selector changes clear prior evidence and pagination +stays bound to the loaded profile identifiers. + +### PRD-FR-2I — Occupation catalog title filter + +- Let an authenticated user filter the imported occupation catalog by + published title or retained code without ranking or typed-code fallback + (ADR 0262). +- Reset the filter when the source changes. +- Disable profile submission and state the next action when the filter + matches no catalog occupation. + +Acceptance: submitting still sends only a catalog identity; a non-matching +filter never creates a request; and Storybook covers a no-match state. + +### PRD-FR-2J — Authorized job-family and job-series snapshots + +- Import one authorized, pinned organization-specific source snapshot without + committing runtime rows or creating an organization (ADR 0263). +- Keep job families, job series, standard occupations, organizational units, + positions, people, and psychological constructs as distinct identities. +- Preserve source-declared multiple-family membership and validity dates; infer + no parent or occupation binding from a code, label, similarity, or model. +- Persist a standard-occupation binding only when scheme IRI, version, code, + and source relation are all explicitly supplied. + +Acceptance: synthetic tests reproduce a series with two source-declared family +parents, reject cycles and partial bindings, leave an occupation-looking label +unbound, and prove the normalized snapshot store is immutable. + ### PRD-FR-3 — Bounded ontology exploration - Apply RBAC/ABAC, source eligibility, and knowledge cutoff before graph diff --git a/frontend/src/App.css b/frontend/src/App.css index 294beb1e0..b9a7b8df8 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -1488,8 +1488,44 @@ .dashboard-case-card dd { margin: 0; font-weight: 600; } .dashboard-case-card button { margin-top: auto; } +.occupation-rating-profile { + max-width: 1440px; + margin: 0 auto 2rem; + padding: 2rem; + color: var(--color-text-heading); +} + +.occupation-rating-form { + display: grid; + grid-template-columns: minmax(16rem, 1fr) minmax(18rem, 1.4fr) auto; + align-items: end; + gap: var(--space-control-gap); + margin: 1rem 0; +} + +.occupation-rating-form label, +.occupation-rating-occupation-select, +.occupation-rating-source { + display: grid; + gap: var(--space-control-gap); +} + +.occupation-rating-form input, +.occupation-rating-form select { min-height: var(--size-control-min); } +.occupation-rating-source { grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr)); margin: 1rem 0; } +.occupation-rating-scroll-hint { display: none; } +.occupation-rating-table { overflow-x: auto; border: 1px solid var(--color-border); } +.occupation-rating-table table { width: 100%; min-width: 64rem; border-collapse: collapse; } +.occupation-rating-table caption { padding: 0.75rem; text-align: left; } +.occupation-rating-table th, +.occupation-rating-table td { padding: 0.75rem; border-bottom: 1px solid var(--color-border); text-align: left; vertical-align: top; } +.occupation-rating-table small { display: block; color: var(--color-text); } + @media (max-width: 900px) { .operations-dashboard { padding: 1rem; } + .occupation-rating-profile { padding: 1rem; } + .occupation-rating-form { grid-template-columns: 1fr; } + .occupation-rating-scroll-hint { display: block; } .operations-dashboard-heading { align-items: start; flex-direction: column; } .dashboard-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); } .dashboard-case-grid { grid-template-columns: 1fr; } diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b80b9fbed..3194ef14b 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -105,6 +105,7 @@ import { TeppAcceptedReceipt } from "./components/TeppAcceptedReceipt"; import { chatEvidenceKindLabel } from "./evidenceKindLabels"; import { WorkspaceNav, type WorkspaceDestination } from "./components/WorkspaceNav"; import { OperationsDashboard } from "./components/OperationsDashboard"; +import { OccupationRatingProfile } from "./components/OccupationRatingProfile"; import { initialWorkspaceDestination } from "./gnbChrome"; import { LineageDag } from "./LineageDag"; import { PostBody } from "./PostBody"; @@ -5317,13 +5318,16 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean />
{destination === "dashboard" ? ( - { - setPostToOpen(postId); - setDestination("board"); - }} - /> + <> + { + setPostToOpen(postId); + setDestination("board"); + }} + /> + + ) : null} {destination === "board" ? ( { vi.unstubAllGlobals(); @@ -19,6 +27,53 @@ describe("backendFetch provider-error boundary", () => { ); }); + it("encodes an exact occupation rating source request", async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ source_available: false, items: [] }), { + headers: { "Content-Type": "application/json" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + + await fetchOccupationRatings("access-token", { + onetsocCode: "15-1252.00", + dataReleaseCode: "onet-31.0", + sourceTableCode: "abilities", + }); + + expect(fetchMock.mock.calls[0][0]).toContain( + "/api/occupations/15-1252.00/ratings?data_release_code=onet-31.0&source_table_code=abilities&limit=100&offset=0", + ); + }); + + it("reads the authenticated occupation source catalog", async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ sources: [] }), { + headers: { "Content-Type": "application/json" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + + await fetchOccupationRatingSources("access-token"); + + expect(fetchMock.mock.calls[0][0]).toContain("/api/occupation-rating-sources"); + }); + + it("reads occupations for one exact imported source", async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ occupations: [] }), { + headers: { "Content-Type": "application/json" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + + await fetchRatingSourceOccupations("access-token", "onet-31.0", "abilities"); + + expect(fetchMock.mock.calls[0][0]).toContain( + "/api/occupation-rating-occupations?data_release_code=onet-31.0&source_table_code=abilities", + ); + }); + it("does not expose provider details from server failures", async () => { vi.stubGlobal( "fetch", diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 0b7d340f9..92893276e 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -1382,6 +1382,105 @@ export function updateTicketStatus( }); } +export interface OccupationRatingItem { + element_id: string; + element_name: string; + scale_id: string; + scale_name: string; + minimum_value: string; + maximum_value: string; + category_value: number | null; + data_value: string; + sample_size: number | null; + standard_error: string | null; + lower_ci_bound: string | null; + upper_ci_bound: string | null; + recommend_suppress: boolean | null; + not_relevant: boolean | null; + source_updated_month: string | null; + domain_source_code: string | null; +} + +export interface OccupationRatingProfile { + data_release_code: string; + source_table_code: string; + onetsoc_code: string; + source_available: boolean; + source: { + source_table_name: string; + source_artifact_url: string; + source_artifact_sha256: string; + source_row_count: number; + scale_artifact_url: string | null; + scale_artifact_sha256: string | null; + scale_source_row_count: number | null; + } | null; + items: OccupationRatingItem[]; + next_offset: number | null; +} + +export interface OccupationRatingSource { + data_release_code: string; + release_version: string; + source_publisher_name: string; + source_license_url: string; + source_table_code: string; + source_table_name: string; + source_artifact_url: string; + source_artifact_sha256: string; + source_row_count: number; +} + +export function fetchOccupationRatingSources( + accessToken: string, +): Promise<{ sources: OccupationRatingSource[] }> { + return backendFetch("/api/occupation-rating-sources", accessToken); +} + +export interface RatingSourceOccupation { + onetsoc_code: string; + occupation_title: string; +} + +export function fetchRatingSourceOccupations( + accessToken: string, + dataReleaseCode: string, + sourceTableCode: string, +): Promise<{ + data_release_code: string; + source_table_code: string; + source_available: boolean; + occupations: RatingSourceOccupation[]; +}> { + const params = new URLSearchParams({ + data_release_code: dataReleaseCode, + source_table_code: sourceTableCode, + }); + return backendFetch(`/api/occupation-rating-occupations?${params.toString()}`, accessToken); +} + +export function fetchOccupationRatings( + accessToken: string, + query: { + onetsocCode: string; + dataReleaseCode: string; + sourceTableCode: string; + limit?: number; + offset?: number; + }, +): Promise { + const params = new URLSearchParams({ + data_release_code: query.dataReleaseCode, + source_table_code: query.sourceTableCode, + limit: String(query.limit ?? 100), + offset: String(query.offset ?? 0), + }); + return backendFetch( + `/api/occupations/${encodeURIComponent(query.onetsocCode)}/ratings?${params.toString()}`, + accessToken, + ); +} + export function fetchPostActivity( accessToken: string, postId: string, diff --git a/frontend/src/components/OccupationRatingProfile.stories.tsx b/frontend/src/components/OccupationRatingProfile.stories.tsx new file mode 100644 index 000000000..0ec7ddf4e --- /dev/null +++ b/frontend/src/components/OccupationRatingProfile.stories.tsx @@ -0,0 +1,123 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect, userEvent, within } from "storybook/test"; +import { OccupationRatingProfile, OccupationRatingProfileView } from "./OccupationRatingProfile"; +import "../App.css"; + +const ready = { + data_release_code: "onet-31.0", source_table_code: "abilities", onetsoc_code: "15-1252.00", source_available: true, + source: { source_table_name: "Abilities", source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), source_row_count: 94640, scale_artifact_url: "https://example.test/scales.csv", scale_artifact_sha256: "b".repeat(64), scale_source_row_count: 33 }, + items: [ + { element_id: "1.A.1.a.1", element_name: "Oral Comprehension", scale_id: "IM", scale_name: "Importance", minimum_value: "1.00", maximum_value: "5.00", category_value: null, data_value: "4.10", sample_size: 120, standard_error: "0.0800", lower_ci_bound: "3.9432", upper_ci_bound: "4.2568", recommend_suppress: true, not_relevant: null, source_updated_month: "08/2026", domain_source_code: "Analyst" }, + { element_id: "1.A.1.a.2", element_name: "Written Comprehension", scale_id: "LV", scale_name: "Level", minimum_value: "0.00", maximum_value: "7.00", category_value: null, data_value: "5.25", sample_size: 118, standard_error: "0.1100", lower_ci_bound: "5.0344", upper_ci_bound: "5.4656", recommend_suppress: false, not_relevant: false, source_updated_month: "08/2026", domain_source_code: "Analyst" }, + ], + next_offset: null, +}; + +const meta = { title: "Ontology/OccupationRatingProfile", component: OccupationRatingProfileView, parameters: { layout: "fullscreen" }, args: { profile: ready } } satisfies Meta; +export default meta; +type Story = StoryObj; + +export const EvidenceReady: Story = { + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("4.10")).toBeVisible(); + await expect(canvas.getByText(/정밀도가 낮아/)).toBeVisible(); + }, +}; + +export const InteractiveEvidenceReady: Story = { + render: () => , + beforeEach: () => { + const previousFetch = globalThis.fetch; + globalThis.fetch = async (input) => new Response(JSON.stringify( + String(input).includes("occupation-rating-sources") + ? { sources: [{ + data_release_code: "onet-31.0", release_version: "31.0", + source_publisher_name: "Synthetic publisher", source_license_url: "https://example.test/license", + source_table_code: "abilities", source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), + source_row_count: 94640, + }] } + : String(input).includes("occupation-rating-occupations") + ? { + data_release_code: "onet-31.0", source_table_code: "abilities", + source_available: true, + occupations: [ + { onetsoc_code: "11-1011.00", occupation_title: "Chief Executives" }, + { onetsoc_code: "15-1252.00", occupation_title: "Software Developers" }, + ], + } + : ready, + ), { headers: { "Content-Type": "application/json" } }); + return () => { globalThis.fetch = previousFetch; }; + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const occupation = await canvas.findByLabelText("직업"); + await canvas.findByRole("option", { name: "Software Developers · 15-1252.00" }); + await userEvent.selectOptions(occupation, "15-1252.00"); + await userEvent.click(canvas.getByRole("button", { name: "직업 근거 열기" })); + await expect(canvas.findByText("4.10")).resolves.toBeVisible(); + }, +}; + +export const NarrowViewport: Story = { + ...InteractiveEvidenceReady, + parameters: { viewport: { defaultViewport: "mobile1" } }, +}; +export const CatalogEmpty: Story = { + render: () => , + beforeEach: () => { + const previousFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(JSON.stringify({ sources: [] }), { + headers: { "Content-Type": "application/json" }, + }); + return () => { globalThis.fetch = previousFetch; }; + }, + play: async ({ canvasElement }) => { + await expect(within(canvasElement).findByText(/가져온 직업 근거 표가 없습니다/)).resolves.toBeVisible(); + }, +}; +export const CatalogUnavailable: Story = { + render: () => , + beforeEach: () => { + const previousFetch = globalThis.fetch; + globalThis.fetch = async () => { throw new Error("synthetic catalog failure"); }; + return () => { globalThis.fetch = previousFetch; }; + }, + play: async ({ canvasElement }) => { + await expect(within(canvasElement).findByRole("alert")).resolves.toHaveTextContent("잠시 후 다시 열어 보세요"); + }, +}; +export const OccupationsEmpty: Story = { + render: () => , + beforeEach: () => { + const previousFetch = globalThis.fetch; + globalThis.fetch = async (input) => new Response(JSON.stringify( + String(input).includes("occupation-rating-sources") + ? { sources: [{ + data_release_code: "onet-31.0", release_version: "31.0", + source_publisher_name: "Synthetic publisher", source_license_url: "https://example.test/license", + source_table_code: "abilities", source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), + source_row_count: 2, + }] } + : { data_release_code: "onet-31.0", source_table_code: "abilities", source_available: true, occupations: [] }, + ), { headers: { "Content-Type": "application/json" } }); + return () => { globalThis.fetch = previousFetch; }; + }, + play: async ({ canvasElement }) => { + await expect(within(canvasElement).findByText(/선택할 수 있는 직업이 없습니다/)).resolves.toBeVisible(); + }, +}; +export const OccupationFilterEmpty: Story = { + ...InteractiveEvidenceReady, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await canvas.findByRole("option", { name: "Software Developers · 15-1252.00" }); + await userEvent.type(canvas.getByLabelText("직업 찾기"), "unknown-occupation"); + await expect(canvas.findByText(/입력한 조건에 맞는 직업이 없습니다/)).resolves.toBeVisible(); + }, +}; +export const SourceUnavailable: Story = { args: { profile: { ...ready, source_available: false, source: null, items: [] } } }; +export const EmptyOccupation: Story = { args: { profile: { ...ready, items: [] } } }; diff --git a/frontend/src/components/OccupationRatingProfile.test.tsx b/frontend/src/components/OccupationRatingProfile.test.tsx new file mode 100644 index 000000000..7ad16c6d3 --- /dev/null +++ b/frontend/src/components/OccupationRatingProfile.test.tsx @@ -0,0 +1,307 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + fetchOccupationRatingSources, + fetchOccupationRatings, + fetchRatingSourceOccupations, + type OccupationRatingProfile as Payload, +} from "../api"; +import { OccupationRatingProfile, OccupationRatingProfileView } from "./OccupationRatingProfile"; + +vi.mock("../api", async (importOriginal) => ({ + ...(await importOriginal()), + fetchOccupationRatingSources: vi.fn(), + fetchOccupationRatings: vi.fn(), + fetchRatingSourceOccupations: vi.fn(), +})); + +const ready: Payload = { + data_release_code: "onet-31.0", + source_table_code: "abilities", + onetsoc_code: "15-1252.00", + source_available: true, + source: { + source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", + source_artifact_sha256: "a".repeat(64), + source_row_count: 2, + scale_artifact_url: "https://example.test/scales.csv", + scale_artifact_sha256: "b".repeat(64), + scale_source_row_count: 33, + }, + items: [{ + element_id: "1.A.1.a.1", element_name: "Oral Comprehension", + scale_id: "IM", scale_name: "Importance", minimum_value: "1.00", maximum_value: "5.00", + category_value: null, data_value: "4.10", sample_size: 120, standard_error: "0.0800", + lower_ci_bound: "3.9432", upper_ci_bound: "4.2568", recommend_suppress: true, + not_relevant: true, source_updated_month: "08/2026", domain_source_code: "Analyst", + }], + next_offset: null, +}; + +beforeEach(() => { + vi.mocked(fetchOccupationRatings).mockClear(); + vi.mocked(fetchOccupationRatingSources).mockResolvedValue({ + sources: [{ + data_release_code: "onet-31.0", release_version: "31.0", + source_publisher_name: "Synthetic publisher", source_license_url: "https://example.test/license", + source_table_code: "abilities", source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), + source_row_count: 2, + }], + }); + vi.mocked(fetchRatingSourceOccupations).mockResolvedValue({ + data_release_code: "onet-31.0", + source_table_code: "abilities", + source_available: true, + occupations: [ + { onetsoc_code: "11-1011.00", occupation_title: "Chief Executives" }, + { onetsoc_code: "15-1252.00", occupation_title: "Software Developers" }, + ], + }); +}); + +describe("OccupationRatingProfile", () => { + it("submits exact identifiers and renders warnings beside the retained value", async () => { + vi.mocked(fetchOccupationRatingSources).mockResolvedValue({ + sources: [{ + data_release_code: "onet-31.0", release_version: "31.0", + source_publisher_name: "Synthetic publisher", source_license_url: "https://example.test/license", + source_table_code: "abilities", source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), + source_row_count: 2, + }], + }); + vi.mocked(fetchOccupationRatings).mockResolvedValue(ready); + render(); + expect(await screen.findByRole("option", { name: "31.0 · Abilities" })).toBeInTheDocument(); + await screen.findByRole("option", { name: "Software Developers · 15-1252.00" }); + await userEvent.selectOptions( + await screen.findByLabelText("직업"), + "15-1252.00", + ); + await userEvent.click(screen.getByRole("button", { name: "직업 근거 열기" })); + expect(fetchOccupationRatings).toHaveBeenCalledWith("synthetic-token", { + onetsocCode: "15-1252.00", dataReleaseCode: "onet-31.0", sourceTableCode: "abilities", offset: 0, + }); + expect(await screen.findByText("4.10")).toBeInTheDocument(); + expect(screen.getByText(/정밀도가 낮아/)).toBeInTheDocument(); + expect(screen.getByText(/해당 없음 응답이 포함됩니다/)).toBeInTheDocument(); + expect(screen.getByText(/표를 가로로 밀어/)).toBeInTheDocument(); + }); + + it("fails closed when no imported rating source exists", async () => { + vi.mocked(fetchOccupationRatingSources).mockResolvedValue({ sources: [] }); + render(); + + expect(await screen.findByText(/가져온 직업 근거 표가 없습니다/)).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "직업 근거 열기" })).toBeDisabled(); + }); + + it("fails closed when an imported source has no selectable occupation", async () => { + vi.mocked(fetchOccupationRatingSources).mockResolvedValue({ + sources: [{ + data_release_code: "onet-31.0", release_version: "31.0", + source_publisher_name: "Synthetic publisher", source_license_url: "https://example.test/license", + source_table_code: "abilities", source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), + source_row_count: 2, + }], + }); + vi.mocked(fetchRatingSourceOccupations).mockResolvedValue({ + data_release_code: "onet-31.0", source_table_code: "abilities", + source_available: true, occupations: [], + }); + render(); + + expect(await screen.findByText(/선택할 수 있는 직업이 없습니다/)).toBeInTheDocument(); + expect(screen.getByLabelText("직업")).toBeDisabled(); + }); + + it("distinguishes an unavailable occupation catalog from an empty one", async () => { + vi.mocked(fetchRatingSourceOccupations).mockResolvedValue({ + data_release_code: "onet-31.0", source_table_code: "abilities", + source_available: false, occupations: [], + }); + render(); + + expect(await screen.findByText(/직업 목록이 아직 준비되지 않았습니다/)).toHaveAttribute("role", "status"); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(screen.queryByText(/선택할 수 있는 직업이 없습니다/)).not.toBeInTheDocument(); + }); + + it("reports a transport failure separately from an unavailable occupation catalog", async () => { + vi.mocked(fetchRatingSourceOccupations).mockRejectedValue(new Error("synthetic transport failure")); + render(); + + expect(await screen.findByRole("alert")).toHaveTextContent("직업 목록을 확인하지 못했습니다"); + expect(screen.queryByText(/직업 목록이 아직 준비되지 않았습니다/)).not.toBeInTheDocument(); + }); + + it("clears a stale catalog error when authentication changes", async () => { + vi.mocked(fetchOccupationRatingSources) + .mockRejectedValueOnce(new Error("synthetic catalog failure")) + .mockResolvedValueOnce({ sources: [{ + data_release_code: "onet-31.0", release_version: "31.0", + source_publisher_name: "Synthetic publisher", source_license_url: "https://example.test/license", + source_table_code: "abilities", source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), + source_row_count: 2, + }] }); + const { rerender } = render(); + expect(await screen.findByRole("alert")).toHaveTextContent("근거 표를 확인하지 못했습니다"); + + rerender(); + + expect(await screen.findByRole("option", { name: "31.0 · Abilities" })).toBeInTheDocument(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + }); + + it("filters stored titles and submits only the selected catalog identity", async () => { + vi.mocked(fetchOccupationRatings).mockResolvedValue(ready); + render(); + await screen.findByRole("option", { name: "Software Developers · 15-1252.00" }); + + await userEvent.type(screen.getByLabelText("직업 찾기"), "15-1252"); + expect(screen.queryByRole("option", { name: "Chief Executives · 11-1011.00" })).not.toBeInTheDocument(); + await userEvent.selectOptions(screen.getByLabelText("직업"), "15-1252.00"); + await userEvent.click(screen.getByRole("button", { name: "직업 근거 열기" })); + + expect(fetchOccupationRatings).toHaveBeenCalledWith("synthetic-token", { + onetsocCode: "15-1252.00", dataReleaseCode: "onet-31.0", sourceTableCode: "abilities", offset: 0, + }); + }); + + it("fails closed when the title filter matches no catalog occupation", async () => { + render(); + await screen.findByRole("option", { name: "Software Developers · 15-1252.00" }); + + await userEvent.type(screen.getByLabelText("직업 찾기"), "unknown-occupation"); + + expect(await screen.findByText(/입력한 조건에 맞는 직업이 없습니다/)).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "직업 근거 열기" })).toBeDisabled(); + expect(fetchOccupationRatings).not.toHaveBeenCalled(); + }); + + it("clears evidence and ignores an in-flight response when authentication changes", async () => { + let finishExpired: ((profile: Payload) => void) | undefined; + vi.mocked(fetchOccupationRatingSources).mockResolvedValue({ sources: [{ + data_release_code: "onet-31.0", release_version: "31.0", + source_publisher_name: "Synthetic publisher", source_license_url: "https://example.test/license", + source_table_code: "abilities", source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), + source_row_count: 2, + }] }); + vi.mocked(fetchOccupationRatings).mockImplementation( + () => new Promise((resolve) => { finishExpired = resolve; }), + ); + const { rerender } = render(); + await screen.findByRole("option", { name: "31.0 · Abilities" }); + await screen.findByRole("option", { name: "Software Developers · 15-1252.00" }); + await userEvent.selectOptions(screen.getByLabelText("직업"), "15-1252.00"); + await userEvent.click(screen.getByRole("button", { name: "직업 근거 열기" })); + + rerender(); + finishExpired?.(ready); + + expect(await screen.findByRole("option", { name: "31.0 · Abilities" })).toBeInTheDocument(); + expect(screen.queryByText("4.10")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "직업 근거 열기" })).toBeDisabled(); + }); + + it("clears loaded evidence when the occupation selection changes", async () => { + vi.mocked(fetchOccupationRatingSources).mockResolvedValue({ + sources: [{ + data_release_code: "onet-31.0", release_version: "31.0", + source_publisher_name: "Synthetic publisher", source_license_url: "https://example.test/license", + source_table_code: "abilities", source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), + source_row_count: 2, + }], + }); + vi.mocked(fetchOccupationRatings).mockResolvedValueOnce({ ...ready, next_offset: 100 }); + render(); + const occupation = await screen.findByLabelText("직업"); + await screen.findByRole("option", { name: "Software Developers · 15-1252.00" }); + await userEvent.selectOptions(occupation, "15-1252.00"); + await userEvent.click(screen.getByRole("button", { name: "직업 근거 열기" })); + await screen.findByText("4.10"); + + await userEvent.selectOptions(occupation, "11-1011.00"); + + expect(screen.queryByText("4.10")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "다음 관측값 불러오기" })).not.toBeInTheDocument(); + }); + + it("removes stale evidence while a fresh occupation loads", async () => { + vi.mocked(fetchOccupationRatingSources).mockResolvedValue({ + sources: [{ + data_release_code: "onet-31.0", release_version: "31.0", + source_publisher_name: "Synthetic publisher", source_license_url: "https://example.test/license", + source_table_code: "abilities", source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), + source_row_count: 2, + }], + }); + vi.mocked(fetchOccupationRatings) + .mockResolvedValueOnce(ready) + .mockImplementationOnce(() => new Promise(() => undefined)); + render(); + const occupation = await screen.findByLabelText("직업"); + await screen.findByRole("option", { name: "Software Developers · 15-1252.00" }); + await userEvent.selectOptions(occupation, "15-1252.00"); + await userEvent.click(screen.getByRole("button", { name: "직업 근거 열기" })); + await screen.findByText("4.10"); + + await userEvent.selectOptions(occupation, "11-1011.00"); + await userEvent.click(screen.getByRole("button", { name: "직업 근거 열기" })); + + expect(screen.queryByText("4.10")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "근거를 불러오는 중" })).toBeDisabled(); + }); + + it("ignores a superseded occupation response that finishes last", async () => { + let finishFirst: ((profile: Payload) => void) | undefined; + vi.mocked(fetchOccupationRatingSources).mockResolvedValue({ sources: [{ + data_release_code: "onet-31.0", release_version: "31.0", + source_publisher_name: "Synthetic publisher", source_license_url: "https://example.test/license", + source_table_code: "abilities", source_table_name: "Abilities", + source_artifact_url: "https://example.test/abilities.csv", source_artifact_sha256: "a".repeat(64), + source_row_count: 2, + }] }); + vi.mocked(fetchOccupationRatings) + .mockImplementationOnce(() => new Promise((resolve) => { finishFirst = resolve; })) + .mockResolvedValueOnce({ ...ready, onetsoc_code: "11-1011.00", items: [{ ...ready.items[0], data_value: "3.20" }] }); + render(); + const occupation = await screen.findByLabelText("직업"); + await screen.findByRole("option", { name: "Software Developers · 15-1252.00" }); + await userEvent.selectOptions(occupation, "15-1252.00"); + await userEvent.click(screen.getByRole("button", { name: "직업 근거 열기" })); + + await userEvent.selectOptions(occupation, "11-1011.00"); + fireEvent.submit(occupation.closest("form")!); + expect(await screen.findByText("3.20")).toBeInTheDocument(); + + finishFirst?.(ready); + expect(screen.queryByText("4.10")).not.toBeInTheDocument(); + expect(screen.getByText("3.20")).toBeInTheDocument(); + }); + + it("distinguishes an unavailable artifact from an empty occupation profile", () => { + const { rerender } = render(); + expect(screen.getByRole("status")).toHaveTextContent("아직 준비되지 않았습니다"); + rerender(); + expect(screen.getByRole("status")).toHaveTextContent("직업이나 근거 표를 바꿔"); + }); + + it("does not turn a non-http artifact value into a customer link", () => { + render(); + + expect(screen.queryByRole("link", { name: "평정 원문 열기" })).not.toBeInTheDocument(); + expect(screen.getByText(/데이터 담당자에게 출처 확인을 요청하세요/)).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/components/OccupationRatingProfile.tsx b/frontend/src/components/OccupationRatingProfile.tsx new file mode 100644 index 000000000..70b2f1a5c --- /dev/null +++ b/frontend/src/components/OccupationRatingProfile.tsx @@ -0,0 +1,321 @@ +import { useEffect, useRef, useState } from "react"; +import { + fetchOccupationRatingSources, + fetchOccupationRatings, + fetchRatingSourceOccupations, + type OccupationRatingProfile as OccupationRatingProfilePayload, + type OccupationRatingSource, + type RatingSourceOccupation, +} from "../api"; + +type Props = { accessToken: string }; + +function matchesOccupationCatalogQuery( + occupation: RatingSourceOccupation, + query: string, +): boolean { + const needle = query.trim().toLocaleLowerCase("en-US"); + return !needle || occupation.occupation_title.toLocaleLowerCase("en-US").includes(needle) + || occupation.onetsoc_code.toLocaleLowerCase("en-US").includes(needle); +} + +function safeHttpUrl(value: string | null | undefined): string | null { + if (!value) return null; + try { + const parsed = new URL(value); + return parsed.protocol === "http:" || parsed.protocol === "https:" ? parsed.toString() : null; + } catch { + return null; + } +} + +/** Lets an authenticated user inspect one exact imported occupation profile. */ +export function OccupationRatingProfile({ accessToken }: Props) { + const [onetsocCode, setOnetsocCode] = useState(""); + const [sources, setSources] = useState(null); + const [selectedSource, setSelectedSource] = useState(""); + const [sourceCatalogError, setSourceCatalogError] = useState(false); + const [occupations, setOccupations] = useState(null); + const [occupationQuery, setOccupationQuery] = useState(""); + const [occupationCatalogUnavailable, setOccupationCatalogUnavailable] = useState(false); + const [occupationCatalogError, setOccupationCatalogError] = useState(false); + const [profile, setProfile] = useState(null); + const [status, setStatus] = useState<"idle" | "loading" | "error">("idle"); + const requestSequence = useRef(0); + const selectedSourceRecord = sources?.find( + (item) => `${item.data_release_code}|${item.source_table_code}` === selectedSource, + ); + const profileMatchesForm = profile != null + && profile.onetsoc_code === onetsocCode + && profile.data_release_code === selectedSourceRecord?.data_release_code + && profile.source_table_code === selectedSourceRecord?.source_table_code; + + useEffect(() => { + let active = true; + requestSequence.current += 1; + setSourceCatalogError(false); + setSources(null); + setSelectedSource(""); + setProfile(null); + setStatus("idle"); + fetchOccupationRatingSources(accessToken) + .then(({ sources: loaded }) => { + if (!active) return; + setSources(loaded); + setSelectedSource( + loaded[0] ? `${loaded[0].data_release_code}|${loaded[0].source_table_code}` : "", + ); + }) + .catch(() => active && setSourceCatalogError(true)); + return () => { active = false; }; + }, [accessToken]); + + useEffect(() => { + requestSequence.current += 1; + setStatus("idle"); + const source = sources?.find( + (item) => `${item.data_release_code}|${item.source_table_code}` === selectedSource, + ); + setOnetsocCode(""); + setOccupationQuery(""); + setProfile(null); + setOccupationCatalogUnavailable(false); + setOccupationCatalogError(false); + if (!source) { + setOccupations(null); + return; + } + let active = true; + setOccupations(null); + fetchRatingSourceOccupations( + accessToken, + source.data_release_code, + source.source_table_code, + ) + .then((payload) => { + if (!active) return; + if (!payload.source_available) { + setOccupationCatalogUnavailable(true); + return; + } + setOccupations(payload.occupations); + }) + .catch(() => active && setOccupationCatalogError(true)); + return () => { active = false; }; + }, [accessToken, selectedSource, sources]); + + function load(offset: number | null = null) { + const requestId = requestSequence.current + 1; + requestSequence.current = requestId; + const source = selectedSourceRecord; + const request = offset == null && source + ? { + onetsocCode, + dataReleaseCode: source.data_release_code, + sourceTableCode: source.source_table_code, + } + : profile + ? { + onetsocCode: profile.onetsoc_code, + dataReleaseCode: profile.data_release_code, + sourceTableCode: profile.source_table_code, + } + : null; + if (!request) return; + if (offset == null) setProfile(null); + setStatus("loading"); + fetchOccupationRatings(accessToken, { + ...request, + offset: offset ?? 0, + }) + .then((payload) => { + if (requestSequence.current !== requestId) return; + setProfile((current) => + offset != null && current + ? { ...payload, items: [...current.items, ...payload.items] } + : payload, + ); + setStatus("idle"); + }) + .catch(() => { + if (requestSequence.current === requestId) setStatus("error"); + }); + } + + const visibleOccupations = (occupations ?? []).filter((occupation) => + matchesOccupationCatalogQuery(occupation, occupationQuery), + ); + + return ( +
+
+

공개 직업 근거

+

직업별 업무 특성 확인

+

직업과 근거 표를 선택해 관측값, 오차, 사용 주의사항을 함께 확인하세요.

+
+
{ + event.preventDefault(); + load(); + }} + > +
+ + +
+ + +
+ {sources === null && !sourceCatalogError ?

사용 가능한 근거 표를 확인하는 중입니다.

: null} + {sources?.length === 0 ?

가져온 직업 근거 표가 없습니다. 데이터 담당자에게 근거 가져오기를 요청하세요.

: null} + {sourceCatalogError ?

사용 가능한 근거 표를 확인하지 못했습니다. 잠시 후 다시 열어 보세요.

: null} + {selectedSource && occupations === null && !occupationCatalogUnavailable && !occupationCatalogError ?

이 근거 표의 직업 목록을 확인하는 중입니다.

: null} + {selectedSource && occupations?.length === 0 ?

이 근거 표에 선택할 수 있는 직업이 없습니다. 다른 근거 표를 선택하세요.

: null} + {occupations != null && occupations.length > 0 && visibleOccupations.length === 0 ? ( +

입력한 조건에 맞는 직업이 없습니다. 검색어를 바꾸거나 다른 근거 표를 선택하세요.

+ ) : null} + {occupationCatalogUnavailable ?

이 근거 표의 직업 목록이 아직 준비되지 않았습니다. 다른 근거 표를 선택하거나 데이터 담당자에게 가져오기를 요청하세요.

: null} + {occupationCatalogError ?

직업 목록을 확인하지 못했습니다. 잠시 후 다시 열어 보세요.

: null} + {status === "error" ? ( +

직업 근거를 불러오지 못했습니다. 선택 항목과 접근 권한을 확인한 뒤 다시 시도하세요.

+ ) : null} + {profile ? : null} + {profileMatchesForm && profile.next_offset != null ? ( + + ) : null} +
+ ); +} + +/** Renders an exact occupation profile for runtime and Storybook scenes. */ +export function OccupationRatingProfileView({ + profile, +}: { + profile: OccupationRatingProfilePayload; +}) { + if (!profile.source_available) { + return ( +

+ 선택한 릴리스와 근거 표가 아직 준비되지 않았습니다. 다른 근거 표를 선택하거나 데이터 담당자에게 가져오기를 요청하세요. +

+ ); + } + if (profile.items.length === 0) { + return ( +

+ 이 근거 표에는 선택한 직업의 관측값이 없습니다. 직업이나 근거 표를 바꿔 확인하세요. +

+ ); + } + const sourceArtifactUrl = safeHttpUrl(profile.source?.source_artifact_url); + const scaleArtifactUrl = safeHttpUrl(profile.source?.scale_artifact_url); + return ( + <> +
+ {profile.source?.source_table_name} + {profile.data_release_code} · {profile.onetsoc_code} + {sourceArtifactUrl ? ( + 평정 원문 열기 + ) : ( + 원문 링크를 사용할 수 없습니다. 데이터 담당자에게 출처 확인을 요청하세요. + )} + {scaleArtifactUrl ? ( + 척도 정의 열기 + ) : null} +
+

표를 가로로 밀어 오차와 사용 주의를 확인하세요.

+
+ + + + + + + + + {profile.items.map((item) => ( + + + + + + + + + ))} + +
값과 오차 및 사용 주의사항
업무 특성척도값표본·오차출처 시점사용 주의
{item.element_name}{item.element_id}{item.scale_name} ({item.minimum_value}–{item.maximum_value}){item.data_value}{item.category_value == null ? null : ` · 범주 ${item.category_value}`} + {item.sample_size == null ? "표본 수 없음" : `N ${item.sample_size}`} + {item.standard_error == null ? null : ` · SE ${item.standard_error}`} + {item.lower_ci_bound == null || item.upper_ci_bound == null ? null : ` · CI ${item.lower_ci_bound}–${item.upper_ci_bound}`} + {item.source_updated_month ?? "시점 없음"}{item.domain_source_code ? ` · ${item.domain_source_code}` : ""} + {[ + item.recommend_suppress ? "정밀도가 낮아 해석 전 원문을 확인하세요." : null, + item.not_relevant ? "해당 없음 응답이 포함됩니다." : null, + ].filter(Boolean).join(" ") || "공개 근거와 함께 해석하세요."} +
+
+ + ); +} diff --git a/migrations/0222_onet_rating_observation_store.sql b/migrations/0222_onet_rating_observation_store.sql new file mode 100644 index 000000000..b07a0461d --- /dev/null +++ b/migrations/0222_onet_rating_observation_store.sql @@ -0,0 +1,226 @@ +-- ADR 0257: normalized, immutable O*NET occupation-rating source evidence. +-- Release and source-table LIST partitions are created by the importer before +-- data insertion; no default partition may silently absorb an unknown source. + +begin; + +create table if not exists occupational_data_release ( + data_release_code text primary key, + release_version text not null, + source_publisher_name text not null, + source_license_url text not null, + imported_at timestamptz not null default now(), + constraint occupational_data_release_code_check + check (btrim(data_release_code) <> ''), + constraint occupational_release_version_check + check (btrim(release_version) <> '') +); + +create table if not exists occupational_source_table ( + data_release_code text not null, + source_table_code text not null, + source_table_name text not null, + source_artifact_url text not null, + source_artifact_sha256 text not null, + source_row_count bigint not null, + primary key (data_release_code, source_table_code), + constraint occupational_source_table_release_fkey + foreign key (data_release_code) + references occupational_data_release (data_release_code), + constraint occupational_source_table_code_check + check (btrim(source_table_code) <> ''), + constraint occupational_source_artifact_check + check (source_artifact_sha256 ~ '^[0-9a-f]{64}$'), + constraint occupational_source_row_count_check + check (source_row_count > 0) +); + +create table if not exists occupational_scale_definition ( + data_release_code text not null, + source_table_code text not null, + scale_id text not null, + scale_name text not null, + minimum_value numeric not null, + maximum_value numeric not null, + primary key (data_release_code, scale_id), + constraint occupational_scale_release_fkey + foreign key (data_release_code) + references occupational_data_release (data_release_code), + constraint occupational_scale_source_table_fkey + foreign key (data_release_code, source_table_code) + references occupational_source_table (data_release_code, source_table_code), + constraint occupational_scale_id_check check (btrim(scale_id) <> ''), + constraint occupational_scale_bounds_check + check (minimum_value <= maximum_value) +); + +create table if not exists occupational_classification_entry ( + data_release_code text not null, + onetsoc_code text not null, + occupation_title text not null, + primary key (data_release_code, onetsoc_code), + constraint occupational_classification_release_fkey + foreign key (data_release_code) + references occupational_data_release (data_release_code), + constraint occupational_onetsoc_code_check + check (onetsoc_code ~ '^[0-9]{2}-[0-9]{4}\.[0-9]{2}$'), + constraint occupational_title_check check (btrim(occupation_title) <> '') +); + +create table if not exists occupational_element_definition ( + data_release_code text not null, + element_id text not null, + element_name text not null, + primary key (data_release_code, element_id), + constraint occupational_element_release_fkey + foreign key (data_release_code) + references occupational_data_release (data_release_code), + constraint occupational_element_id_check + check (element_id ~ '^[1-6](\.[A-Za-z0-9]+)*$'), + constraint occupational_element_name_check check (btrim(element_name) <> '') +); + +create table if not exists occupational_rating_observation ( + data_release_code text not null, + source_table_code text not null, + onetsoc_code text not null, + element_id text not null, + scale_id text not null, + category_value integer, + data_value numeric not null, + sample_size integer, + standard_error numeric, + lower_ci_bound numeric, + upper_ci_bound numeric, + recommend_suppress boolean, + not_relevant boolean, + source_updated_month text not null, + domain_source_code text not null, + constraint occupational_rating_source_table_fkey + foreign key (data_release_code, source_table_code) + references occupational_source_table (data_release_code, source_table_code), + constraint occupational_rating_classification_fkey + foreign key (data_release_code, onetsoc_code) + references occupational_classification_entry (data_release_code, onetsoc_code), + constraint occupational_rating_element_fkey + foreign key (data_release_code, element_id) + references occupational_element_definition (data_release_code, element_id), + constraint occupational_rating_scale_fkey + foreign key (data_release_code, scale_id) + references occupational_scale_definition (data_release_code, scale_id), + constraint occupational_rating_identity_key + unique nulls not distinct + (data_release_code, source_table_code, onetsoc_code, element_id, scale_id, category_value), + constraint occupational_rating_sample_size_check + check (sample_size is null or sample_size > 0), + constraint occupational_rating_standard_error_check + check (standard_error is null or standard_error >= 0), + constraint occupational_rating_interval_presence_check + check ((lower_ci_bound is null) = (upper_ci_bound is null)), + constraint occupational_rating_interval_order_check + check (lower_ci_bound is null or lower_ci_bound <= upper_ci_bound), + constraint occupational_rating_data_value_check + check (data_value::text not in ('NaN', 'Infinity', '-Infinity')), + constraint occupational_rating_domain_source_check + check (btrim(domain_source_code) <> ''), + constraint occupational_rating_source_updated_month_check + check (source_updated_month ~ '^(0[1-9]|1[0-2])/[0-9]{4}$') +) partition by list (data_release_code); + +create index if not exists occupational_rating_occupation_element_idx + on occupational_rating_observation + (data_release_code, onetsoc_code, element_id, scale_id); + +create index if not exists occupational_rating_element_occupation_idx + on occupational_rating_observation + (data_release_code, element_id, scale_id, onetsoc_code); + +create or replace function validate_occupational_rating_insert() +returns trigger +language plpgsql +as $$ +declare + declared_minimum numeric; + declared_maximum numeric; + existing_observation occupational_rating_observation%rowtype; +begin + if new.source_updated_month !~ '^(0[1-9]|1[0-2])/[0-9]{4}$' then + raise check_violation using message = 'source_updated_month must be MM/YYYY'; + end if; + if to_date('01/' || new.source_updated_month, 'DD/MM/YYYY') + > date_trunc('month', current_date)::date then + raise check_violation using message = 'source_updated_month must not be in the future'; + end if; + select minimum_value, maximum_value + into declared_minimum, declared_maximum + from occupational_scale_definition + where data_release_code = new.data_release_code + and scale_id = new.scale_id; + if declared_minimum is not null + and new.data_value not between declared_minimum and declared_maximum then + raise check_violation using message = 'data_value is outside the declared scale bounds'; + end if; + select observation.* + into existing_observation + from occupational_rating_observation observation + where observation.data_release_code = new.data_release_code + and observation.source_table_code = new.source_table_code + and observation.onetsoc_code = new.onetsoc_code + and observation.element_id = new.element_id + and observation.scale_id = new.scale_id + and observation.category_value is not distinct from new.category_value; + if found and row( + existing_observation.data_value, + existing_observation.sample_size, + existing_observation.standard_error, + existing_observation.lower_ci_bound, + existing_observation.upper_ci_bound, + existing_observation.recommend_suppress, + existing_observation.not_relevant, + existing_observation.source_updated_month, + existing_observation.domain_source_code + ) is distinct from row( + new.data_value, + new.sample_size, + new.standard_error, + new.lower_ci_bound, + new.upper_ci_bound, + new.recommend_suppress, + new.not_relevant, + new.source_updated_month, + new.domain_source_code + ) then + raise check_violation using message = 'occupational rating identity conflicts with immutable evidence'; + end if; + return new; +end; +$$; + +drop trigger if exists occupational_rating_validate_insert + on occupational_rating_observation; +create trigger occupational_rating_validate_insert +before insert on occupational_rating_observation +for each row execute function validate_occupational_rating_insert(); + +create or replace function reject_occupational_rating_mutation() +returns trigger +language plpgsql +as $$ +begin + raise check_violation using message = 'occupational rating evidence is immutable'; +end; +$$; + +drop trigger if exists occupational_rating_reject_mutation + on occupational_rating_observation; +create trigger occupational_rating_reject_mutation +before update or delete on occupational_rating_observation +for each row execute function reject_occupational_rating_mutation(); + +drop trigger if exists occupational_rating_reject_truncate + on occupational_rating_observation; +create trigger occupational_rating_reject_truncate +before truncate on occupational_rating_observation +for each statement execute function reject_occupational_rating_mutation(); + +commit; diff --git a/migrations/0223_authorized_job_architecture.sql b/migrations/0223_authorized_job_architecture.sql new file mode 100644 index 000000000..193730103 --- /dev/null +++ b/migrations/0223_authorized_job_architecture.sql @@ -0,0 +1,159 @@ +-- ADR 0263: authorized, source-preserving job-family/job-series snapshots. + +begin; + +create table if not exists job_architecture_source ( + corporate_entity_id uuid not null references corporate_entity (corporate_entity_id), + source_system_code text not null, + source_snapshot_code text not null, + source_name text not null, + source_artifact_url text not null, + source_artifact_sha256 text not null, + source_row_count bigint not null, + imported_at timestamptz not null default now(), + primary key (corporate_entity_id, source_system_code, source_snapshot_code), + constraint job_architecture_source_system_check + check (source_system_code ~ '^[a-z][a-z0-9_]{0,62}$'), + constraint job_architecture_snapshot_check check (btrim(source_snapshot_code) <> ''), + constraint job_architecture_source_name_check check (btrim(source_name) <> ''), + constraint job_architecture_source_digest_check + check (source_artifact_sha256 ~ '^[0-9a-f]{64}$'), + constraint job_architecture_source_rows_check check (source_row_count > 0) +); + +create table if not exists job_architecture_node ( + corporate_entity_id uuid not null, + source_system_code text not null, + source_snapshot_code text not null, + job_architecture_code text not null, + job_architecture_kind_code text not null, + job_architecture_name text not null, + job_architecture_description text, + valid_from date, + valid_to date, + primary key ( + corporate_entity_id, source_system_code, source_snapshot_code, + job_architecture_code + ), + constraint job_architecture_node_source_fkey + foreign key (corporate_entity_id, source_system_code, source_snapshot_code) + references job_architecture_source + (corporate_entity_id, source_system_code, source_snapshot_code), + constraint job_architecture_node_code_check check (btrim(job_architecture_code) <> ''), + constraint job_architecture_node_kind_check + check (job_architecture_kind_code in ('job_family', 'job_series')), + constraint job_architecture_node_name_check check (btrim(job_architecture_name) <> ''), + constraint job_architecture_node_validity_check + check (valid_from is null or valid_to is null or valid_from <= valid_to) +); + +create table if not exists job_architecture_hierarchy_edge ( + corporate_entity_id uuid not null, + source_system_code text not null, + source_snapshot_code text not null, + broader_job_architecture_code text not null, + narrower_job_architecture_code text not null, + source_relation_code text not null, + primary key ( + corporate_entity_id, source_system_code, source_snapshot_code, + broader_job_architecture_code, narrower_job_architecture_code + ), + constraint job_architecture_hierarchy_source_fkey + foreign key (corporate_entity_id, source_system_code, source_snapshot_code) + references job_architecture_source + (corporate_entity_id, source_system_code, source_snapshot_code), + constraint job_architecture_hierarchy_broader_fkey + foreign key ( + corporate_entity_id, source_system_code, source_snapshot_code, + broader_job_architecture_code + ) references job_architecture_node ( + corporate_entity_id, source_system_code, source_snapshot_code, + job_architecture_code + ), + constraint job_architecture_hierarchy_narrower_fkey + foreign key ( + corporate_entity_id, source_system_code, source_snapshot_code, + narrower_job_architecture_code + ) references job_architecture_node ( + corporate_entity_id, source_system_code, source_snapshot_code, + job_architecture_code + ), + constraint job_architecture_hierarchy_distinct_check + check (broader_job_architecture_code <> narrower_job_architecture_code), + constraint job_architecture_hierarchy_relation_check + check (btrim(source_relation_code) <> '') +); + +create table if not exists job_architecture_occupation_binding ( + corporate_entity_id uuid not null, + source_system_code text not null, + source_snapshot_code text not null, + job_architecture_code text not null, + occupation_scheme_iri text not null, + occupation_scheme_version text not null, + occupation_code text not null, + source_relation_code text not null, + primary key ( + corporate_entity_id, source_system_code, source_snapshot_code, + job_architecture_code, occupation_scheme_iri, + occupation_scheme_version, occupation_code + ), + constraint job_architecture_binding_node_fkey + foreign key ( + corporate_entity_id, source_system_code, source_snapshot_code, + job_architecture_code + ) references job_architecture_node ( + corporate_entity_id, source_system_code, source_snapshot_code, + job_architecture_code + ), + constraint job_architecture_binding_scheme_check + check (occupation_scheme_iri ~ '^https?://'), + constraint job_architecture_binding_version_check + check (btrim(occupation_scheme_version) <> ''), + constraint job_architecture_binding_code_check check (btrim(occupation_code) <> ''), + constraint job_architecture_binding_relation_check + check (btrim(source_relation_code) <> '') +); + +create index if not exists job_architecture_node_lookup_idx + on job_architecture_node + (corporate_entity_id, job_architecture_kind_code, job_architecture_name); + +create index if not exists job_architecture_binding_occupation_idx + on job_architecture_occupation_binding + (occupation_scheme_iri, occupation_scheme_version, occupation_code); + +create or replace function reject_job_architecture_mutation() +returns trigger +language plpgsql +as $$ +begin + raise check_violation using message = 'job architecture source evidence is immutable'; +end; +$$; + +do $$ +declare + table_name text; +begin + foreach table_name in array array[ + 'job_architecture_source', + 'job_architecture_node', + 'job_architecture_hierarchy_edge', + 'job_architecture_occupation_binding' + ] loop + execute format('drop trigger if exists job_architecture_reject_mutation on %I', table_name); + execute format( + 'create trigger job_architecture_reject_mutation before update or delete on %I for each row execute function reject_job_architecture_mutation()', + table_name + ); + execute format('drop trigger if exists job_architecture_reject_truncate on %I', table_name); + execute format( + 'create trigger job_architecture_reject_truncate before truncate on %I for each statement execute function reject_job_architecture_mutation()', + table_name + ); + end loop; +end; +$$; + +commit; diff --git a/scripts/import_job_architecture.py b/scripts/import_job_architecture.py new file mode 100644 index 000000000..5fb436dfd --- /dev/null +++ b/scripts/import_job_architecture.py @@ -0,0 +1,341 @@ +"""Validate and import one authorized job-family/job-series snapshot.""" + +from __future__ import annotations + +import argparse +import asyncio +import csv +import hashlib +import json +import re +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from urllib.parse import urlsplit + +import asyncpg + +_FIELDS = { + "Node Code", + "Node Kind", + "Node Name", + "Parent Code", + "Hierarchy Relation", + "Valid From", + "Valid To", + "Occupation Scheme IRI", + "Occupation Scheme Version", + "Occupation Code", + "Occupation Relation", +} +_KINDS = {"job_family", "job_series"} +_SOURCE_CODE = re.compile(r"^[a-z][a-z0-9_]{0,62}$") +_SHA256 = re.compile(r"^[0-9a-fA-F]{64}$") + + +@dataclass(frozen=True) +class JobArchitectureNode: + """One exact node from an authorized source snapshot.""" + + code: str + kind: str + name: str + description: str | None + valid_from: date | None + valid_to: date | None + + +@dataclass(frozen=True) +class JobArchitectureEdge: + """One source-declared broader-to-narrower relationship.""" + + broader_code: str + narrower_code: str + source_relation_code: str + + +@dataclass(frozen=True) +class OccupationBinding: + """One explicit source binding to an external occupation code.""" + + node_code: str + scheme_iri: str + scheme_version: str + occupation_code: str + source_relation_code: str + + +def _optional_date(value: str, field: str) -> date | None: + """Parse an optional ISO date without inventing a missing instant.""" + text = value.strip() + if not text: + return None + try: + return date.fromisoformat(text) + except ValueError as exc: + raise ValueError(f"invalid {field}: {value!r}") from exc + + +def _https_url(value: str, field: str) -> str: + """Validate an HTTPS URL with no embedded credentials.""" + parsed = urlsplit(value) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + ): + raise ValueError(f"{field} must be an HTTPS URL without userinfo") + return value + + +def read_job_architecture( + path: Path, +) -> tuple[ + list[JobArchitectureNode], + list[JobArchitectureEdge], + list[OccupationBinding], + int, +]: + """Return exact nodes, hierarchy edges, and explicit occupation bindings.""" + nodes: dict[str, JobArchitectureNode] = {} + edges: dict[tuple[str, str], JobArchitectureEdge] = {} + bindings: dict[tuple[str, str, str, str], OccupationBinding] = {} + row_count = 0 + with path.open(encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle) + missing = sorted(_FIELDS - set(reader.fieldnames or ())) + if missing: + raise ValueError(f"missing CSV columns: {', '.join(missing)}") + for line_number, row in enumerate(reader, start=2): + row_count += 1 + if None in row or any(value is None for value in row.values()): + raise ValueError(f"malformed CSV row: {line_number}") + code = row["Node Code"].strip() + kind = row["Node Kind"].strip() + name = row["Node Name"].strip() + if not code or not name or kind not in _KINDS: + raise ValueError(f"invalid node identity at row {line_number}") + valid_from = _optional_date(row["Valid From"], "valid from") + valid_to = _optional_date(row["Valid To"], "valid to") + if valid_from and valid_to and valid_from > valid_to: + raise ValueError(f"inverted validity interval at row {line_number}") + node = JobArchitectureNode( + code, + kind, + name, + row.get("Description", "").strip() or None, + valid_from, + valid_to, + ) + if code in nodes and nodes[code] != node: + raise ValueError(f"conflicting node identity: {code}") + nodes[code] = node + parent = row["Parent Code"].strip() + hierarchy_relation = row["Hierarchy Relation"].strip() + if parent: + if not hierarchy_relation: + raise ValueError(f"missing hierarchy relation at row {line_number}") + edge = JobArchitectureEdge(parent, code, hierarchy_relation) + edge_key = (parent, code) + if edge_key in edges and edges[edge_key] != edge: + raise ValueError(f"conflicting hierarchy relation at row {line_number}") + edges[edge_key] = edge + scheme = row["Occupation Scheme IRI"].strip() + version = row["Occupation Scheme Version"].strip() + occupation = row["Occupation Code"].strip() + occupation_relation = row["Occupation Relation"].strip() + supplied = ( + bool(scheme), + bool(version), + bool(occupation), + bool(occupation_relation), + ) + if any(supplied) and not all(supplied): + raise ValueError(f"partial occupation binding at row {line_number}") + if all(supplied): + parsed = urlsplit(scheme) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + ): + raise ValueError(f"invalid occupation scheme IRI at row {line_number}") + if not occupation_relation: + raise ValueError(f"missing binding relation at row {line_number}") + binding = OccupationBinding( + code, + scheme, + version, + occupation, + occupation_relation, + ) + binding_key = (code, scheme, version, occupation) + if binding_key in bindings and bindings[binding_key] != binding: + raise ValueError(f"conflicting occupation relation at row {line_number}") + bindings[binding_key] = binding + if not nodes: + raise ValueError("job architecture file has no rows") + for edge in edges.values(): + if edge.broader_code not in nodes: + raise ValueError(f"unknown parent node: {edge.broader_code}") + if edge.broader_code == edge.narrower_code: + raise ValueError(f"self hierarchy edge: {edge.broader_code}") + children: dict[str, set[str]] = {code: set() for code in nodes} + incoming = dict.fromkeys(nodes, 0) + for edge in edges.values(): + children[edge.broader_code].add(edge.narrower_code) + incoming[edge.narrower_code] += 1 + ready = [code for code, count in incoming.items() if count == 0] + visited = 0 + while ready: + code = ready.pop() + visited += 1 + for child in children[code]: + incoming[child] -= 1 + if incoming[child] == 0: + ready.append(child) + if visited != len(nodes): + raise ValueError("cyclic job architecture hierarchy") + return ( + list(nodes.values()), + sorted(edges.values(), key=repr), + sorted(bindings.values(), key=repr), + row_count, + ) + + +def _parser() -> argparse.ArgumentParser: + """Build the explicit source-snapshot import contract.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--target-dsn", required=True) + parser.add_argument("--corporate-entity-code", required=True) + parser.add_argument("--source-system-code", required=True) + parser.add_argument("--source-snapshot-code", required=True) + parser.add_argument("--source-name", required=True) + parser.add_argument("--source-url", required=True) + parser.add_argument("--source-sha256", required=True) + parser.add_argument("--source-row-count", type=int, required=True) + parser.add_argument("--source-file", type=Path, required=True) + return parser + + +async def import_job_architecture(args: argparse.Namespace) -> dict[str, object]: + """Validate one pinned snapshot before transactionally persisting it.""" + if not _SOURCE_CODE.fullmatch(args.source_system_code): + raise ValueError("source system code must be lower snake case") + for field in ("corporate_entity_code", "source_snapshot_code", "source_name"): + if not str(getattr(args, field)).strip(): + raise ValueError(f"{field} must not be blank") + _https_url(args.source_url, "source URL") + if not _SHA256.fullmatch(args.source_sha256): + raise ValueError("source SHA-256 must be one digest") + if args.source_row_count <= 0 or not args.source_file.is_file(): + raise ValueError("source row count and file must be valid") + digest = hashlib.sha256(args.source_file.read_bytes()).hexdigest() + if digest != args.source_sha256.lower(): + raise ValueError("source artifact SHA-256 mismatch") + nodes, edges, bindings, row_count = read_job_architecture(args.source_file) + if row_count != args.source_row_count: + raise ValueError("source artifact row-count mismatch") + conn = await asyncpg.connect(args.target_dsn) + try: + async with conn.transaction(): + entity_id = await conn.fetchval( + "select corporate_entity_id from corporate_entity where corporate_entity_code = $1", + args.corporate_entity_code, + ) + if entity_id is None: + raise ValueError("corporate entity must already exist") + key = (entity_id, args.source_system_code, args.source_snapshot_code) + await conn.execute( + """insert into job_architecture_source + (corporate_entity_id, source_system_code, source_snapshot_code, + source_name, source_artifact_url, source_artifact_sha256, + source_row_count) + values ($1,$2,$3,$4,$5,$6,$7) + on conflict (corporate_entity_id, source_system_code, source_snapshot_code) + do update set source_name = excluded.source_name + where row(job_architecture_source.source_name, + job_architecture_source.source_artifact_url, + job_architecture_source.source_artifact_sha256, + job_architecture_source.source_row_count) + is distinct from row(excluded.source_name, + excluded.source_artifact_url, + excluded.source_artifact_sha256, + excluded.source_row_count)""", + *key, + args.source_name, + args.source_url, + digest, + row_count, + ) + await conn.executemany( + """insert into job_architecture_node + (corporate_entity_id, source_system_code, source_snapshot_code, + job_architecture_code, job_architecture_kind_code, + job_architecture_name, job_architecture_description, + valid_from, valid_to) + values ($1,$2,$3,$4,$5,$6,$7,$8,$9) + on conflict (corporate_entity_id, source_system_code, + source_snapshot_code, job_architecture_code) + do update set job_architecture_name = excluded.job_architecture_name + where row(job_architecture_node.job_architecture_kind_code, + job_architecture_node.job_architecture_name, + job_architecture_node.job_architecture_description, + job_architecture_node.valid_from, + job_architecture_node.valid_to) + is distinct from row(excluded.job_architecture_kind_code, + excluded.job_architecture_name, + excluded.job_architecture_description, + excluded.valid_from, excluded.valid_to)""", + [(*key, n.code, n.kind, n.name, n.description, n.valid_from, n.valid_to) for n in nodes], + ) + await conn.executemany( + """insert into job_architecture_hierarchy_edge + (corporate_entity_id, source_system_code, source_snapshot_code, + broader_job_architecture_code, narrower_job_architecture_code, + source_relation_code) + values ($1,$2,$3,$4,$5,$6) + on conflict (corporate_entity_id, source_system_code, + source_snapshot_code, broader_job_architecture_code, + narrower_job_architecture_code) + do update set source_relation_code = excluded.source_relation_code + where job_architecture_hierarchy_edge.source_relation_code + is distinct from excluded.source_relation_code""", + [(*key, e.broader_code, e.narrower_code, e.source_relation_code) for e in edges], + ) + await conn.executemany( + """insert into job_architecture_occupation_binding + (corporate_entity_id, source_system_code, source_snapshot_code, + job_architecture_code, occupation_scheme_iri, + occupation_scheme_version, occupation_code, source_relation_code) + values ($1,$2,$3,$4,$5,$6,$7,$8) + on conflict (corporate_entity_id, source_system_code, + source_snapshot_code, job_architecture_code, + occupation_scheme_iri, occupation_scheme_version, + occupation_code) + do update set source_relation_code = excluded.source_relation_code + where job_architecture_occupation_binding.source_relation_code + is distinct from excluded.source_relation_code""", + [(*key, b.node_code, b.scheme_iri, b.scheme_version, b.occupation_code, b.source_relation_code) for b in bindings], + ) + finally: + await conn.close() + return { + "source_snapshot_code": args.source_snapshot_code, + "imported_nodes": len(nodes), + "imported_hierarchy_edges": len(edges), + "imported_occupation_bindings": len(bindings), + "source_sha256": digest, + } + + +def main() -> None: + """Run the importer and print aggregate, non-identifying evidence.""" + print(json.dumps(asyncio.run(import_job_architecture(_parser().parse_args())), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/import_onet_ratings.py b/scripts/import_onet_ratings.py new file mode 100644 index 000000000..729a4d32e --- /dev/null +++ b/scripts/import_onet_ratings.py @@ -0,0 +1,560 @@ +"""Validate and import one official O*NET occupation-rating CSV artifact.""" + +from __future__ import annotations + +import argparse +import asyncio +import csv +import hashlib +import re +import json +import re +from dataclasses import dataclass +from datetime import UTC, date, datetime +from decimal import Decimal, InvalidOperation +from pathlib import Path +from urllib.parse import urlsplit + +import asyncpg + +_RATING_FIELDS = { + "O*NET-SOC Code", + "Title", + "Element ID", + "Element Name", + "Scale ID", + "Scale Name", + "Data Value", + "Date", + "Domain Source", +} +_SCALE_FIELDS = {"Scale ID", "Scale Name", "Minimum", "Maximum"} +_SOURCE_CODE = re.compile(r"^[a-z][a-z0-9_]{0,62}$") +_SCALES_SOURCE_CODE = "scales_reference" +_SHA256 = re.compile(r"^[0-9a-fA-F]{64}$") + + +@dataclass(frozen=True) +class ScaleDefinition: + """One exact O*NET scale identity and its declared numeric bounds.""" + + scale_id: str + scale_name: str + minimum_value: Decimal + maximum_value: Decimal + + +@dataclass(frozen=True) +class RatingObservation: + """One validated O*NET occupation-to-element source observation.""" + + onetsoc_code: str + occupation_title: str + element_id: str + element_name: str + scale_id: str + category_value: int | None + data_value: Decimal + sample_size: int | None + standard_error: Decimal | None + lower_ci_bound: Decimal | None + upper_ci_bound: Decimal | None + recommend_suppress: bool | None + not_relevant: bool | None + source_updated_month: str + domain_source_code: str + + +def _decimal(value: str, field: str, *, optional: bool = False) -> Decimal | None: + """Parse one finite source decimal while preserving an honest blank.""" + text = value.strip() + if optional and not text: + return None + try: + parsed = Decimal(text) + except InvalidOperation as exc: + raise ValueError(f"invalid {field}: {value!r}") from exc + if not parsed.is_finite(): + raise ValueError(f"invalid {field}: {value!r}") + return parsed + + +def _integer(value: str, field: str, *, optional: bool = False) -> int | None: + """Parse one source integer while preserving an honest blank.""" + text = value.strip() + if optional and not text: + return None + try: + return int(text) + except ValueError as exc: + raise ValueError(f"invalid {field}: {value!r}") from exc + + +def _flag(value: str, field: str) -> bool | None: + """Parse the official Y/N/blank tri-state flag vocabulary.""" + text = value.strip() + if not text: + return None + if text == "Y": + return True + if text == "N": + return False + raise ValueError(f"invalid {field} flag: {value!r}") + + +def _updated_month(value: str, today: date) -> str: + """Validate and preserve one exact O*NET ``MM/YYYY`` source month.""" + text = value.strip() + if not re.fullmatch(r"(0[1-9]|1[0-2])/[0-9]{4}", text): + raise ValueError(f"invalid source update date: {value!r}") + try: + month_text, year_text = text.split("/") + parsed = date(int(year_text), int(month_text), 1) + except (TypeError, ValueError) as exc: + raise ValueError(f"invalid source update date: {value!r}") from exc + if parsed > today.replace(day=1): + raise ValueError(f"future source update date: {value!r}") + return text + + +def _rows(path: Path, required: set[str]) -> list[dict[str, str]]: + """Read one UTF-8 CSV only when its authoritative columns are present.""" + with path.open(encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle) + fields = set(reader.fieldnames or ()) + missing = sorted(required - fields) + if missing: + raise ValueError(f"missing CSV columns: {', '.join(missing)}") + rows = [] + for line_number, row in enumerate(reader, start=2): + if None in row or any(value is None for value in row.values()): + raise ValueError(f"malformed CSV row: {line_number}") + rows.append(dict(row)) + return rows + + +def read_scale_file(path: Path) -> dict[str, ScaleDefinition]: + """Return exact scale definitions from the official Scales Reference CSV.""" + scales: dict[str, ScaleDefinition] = {} + for row in _rows(path, _SCALE_FIELDS): + scale_id = row["Scale ID"].strip() + definition = ScaleDefinition( + scale_id=scale_id, + scale_name=row["Scale Name"].strip(), + minimum_value=_decimal(row["Minimum"], "scale minimum"), # type: ignore[arg-type] + maximum_value=_decimal(row["Maximum"], "scale maximum"), # type: ignore[arg-type] + ) + if not scale_id or not definition.scale_name: + raise ValueError("empty scale identity") + if definition.minimum_value > definition.maximum_value: + raise ValueError(f"inverted scale bounds: {scale_id}") + if scale_id in scales and scales[scale_id] != definition: + raise ValueError(f"conflicting scale definition: {scale_id}") + scales[scale_id] = definition + if not scales: + raise ValueError("scale file has no rows") + return scales + + +def read_rating_file( + path: Path, + scales: dict[str, ScaleDefinition], + *, + today: date | None = None, +) -> list[RatingObservation]: + """Validate an official rating CSV and return exact source observations.""" + observed_today = today or datetime.now(UTC).date() + occupations: dict[str, str] = {} + elements: dict[str, str] = {} + observations: list[RatingObservation] = [] + identities: set[tuple[str, str, str, int | None]] = set() + for row in _rows(path, _RATING_FIELDS): + onetsoc_code = row["O*NET-SOC Code"].strip() + occupation_title = row["Title"].strip() + element_id = row["Element ID"].strip() + element_name = row["Element Name"].strip() + scale_id = row["Scale ID"].strip() + scale = scales.get(scale_id) + if scale is None or row["Scale Name"].strip() != scale.scale_name: + raise ValueError(f"unknown or conflicting scale identity: {scale_id}") + if ( + onetsoc_code in occupations + and occupations[onetsoc_code] != occupation_title + ): + raise ValueError(f"conflicting occupation title: {onetsoc_code}") + if element_id in elements and elements[element_id] != element_name: + raise ValueError(f"conflicting element name: {element_id}") + occupations[onetsoc_code] = occupation_title + elements[element_id] = element_name + data_value = _decimal(row["Data Value"], "data value") + if not scale.minimum_value <= data_value <= scale.maximum_value: # type: ignore[operator] + raise ValueError(f"data value outside scale {scale_id}") + sample_size = _integer(row.get("N", ""), "sample size", optional=True) + standard_error = _decimal( + row.get("Standard Error", ""), "standard error", optional=True + ) + lower = _decimal(row.get("Lower CI Bound", ""), "lower CI bound", optional=True) + upper = _decimal(row.get("Upper CI Bound", ""), "upper CI bound", optional=True) + if sample_size is not None and sample_size <= 0: + raise ValueError("sample size must be positive") + if standard_error is not None and standard_error < 0: + raise ValueError("standard error must be non-negative") + if (lower is None) != (upper is None) or (lower is not None and lower > upper): + raise ValueError("invalid confidence interval") + category = _integer(row.get("Category", ""), "category", optional=True) + identity = (onetsoc_code, element_id, scale_id, category) + if identity in identities: + raise ValueError(f"duplicate rating identity: {identity}") + identities.add(identity) + domain_source = row["Domain Source"].strip() + if ( + not onetsoc_code + or not occupation_title + or not element_id + or not element_name + or not domain_source + ): + raise ValueError("empty rating identity") + observations.append( + RatingObservation( + onetsoc_code=onetsoc_code, + occupation_title=occupation_title, + element_id=element_id, + element_name=element_name, + scale_id=scale_id, + category_value=category, + data_value=data_value, # type: ignore[arg-type] + sample_size=sample_size, + standard_error=standard_error, + lower_ci_bound=lower, + upper_ci_bound=upper, + recommend_suppress=_flag( + row.get("Recommend Suppress", ""), "recommend suppress" + ), + not_relevant=_flag(row.get("Not Relevant", ""), "not relevant"), + source_updated_month=_updated_month(row["Date"], observed_today), + domain_source_code=domain_source, + ) + ) + if not observations: + raise ValueError("rating file has no rows") + return observations + + +def _parser() -> argparse.ArgumentParser: + """Build the explicit, provenance-bearing importer command contract.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--target-dsn", required=True) + parser.add_argument("--release-code", required=True) + parser.add_argument("--release-version", required=True) + parser.add_argument("--source-table-code", required=True) + parser.add_argument("--source-table-name", required=True) + parser.add_argument("--source-url", required=True) + parser.add_argument("--source-sha256", required=True) + parser.add_argument("--source-row-count", type=int, required=True) + parser.add_argument("--publisher", default="National Center for O*NET Development") + parser.add_argument( + "--license-url", default="https://creativecommons.org/licenses/by/4.0/" + ) + parser.add_argument("--scales-file", type=Path, required=True) + parser.add_argument("--scales-url", required=True) + parser.add_argument("--scales-sha256", required=True) + parser.add_argument("--scales-row-count", type=int, required=True) + parser.add_argument("--ratings-file", type=Path, required=True) + return parser + + +def _validate_args(args: argparse.Namespace) -> None: + """Reject ambiguous provenance and unsafe source identities before I/O.""" + if ( + not _SOURCE_CODE.fullmatch(args.source_table_code) + or args.source_table_code == _SCALES_SOURCE_CODE + ): + raise ValueError("source table code must be non-reserved lower snake case") + for field in ("release_code", "release_version", "source_table_name", "publisher"): + if not str(getattr(args, field)).strip(): + raise ValueError(f"{field} must not be blank") + for field in ("source_sha256", "scales_sha256"): + if not _SHA256.fullmatch(str(getattr(args, field))): + raise ValueError(f"{field} must be one SHA-256 digest") + for field in ("source_row_count", "scales_row_count"): + if getattr(args, field) <= 0: + raise ValueError(f"{field} must be positive") + for field in ("source_url", "scales_url", "license_url"): + parsed = urlsplit(str(getattr(args, field))) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + ): + raise ValueError(f"{field} must be an HTTPS URL without userinfo") + for field in ("ratings_file", "scales_file"): + if not getattr(args, field).is_file(): + raise ValueError(f"{field} must be a regular file") + + +async def _reject_reference_conflicts( + conn: asyncpg.Connection, + args: argparse.Namespace, + scales: dict[str, ScaleDefinition], + observations: list[RatingObservation], +) -> None: + """Reject a reused source identity whose immutable source labels differ.""" + release = await conn.fetchrow( + """select release_version, source_publisher_name, source_license_url + from occupational_data_release where data_release_code = $1""", + args.release_code, + ) + if release is not None and tuple(release) != ( + args.release_version, + args.publisher, + args.license_url, + ): + raise ValueError("conflicting release identity") + sources = ( + ( + args.source_table_code, + args.source_table_name, + args.source_url, + args.source_sha256.lower(), + args.source_row_count, + ), + ( + _SCALES_SOURCE_CODE, + "Scales Reference", + args.scales_url, + args.scales_sha256.lower(), + args.scales_row_count, + ), + ) + for source_code, source_name, source_url, source_digest, source_rows in sources: + source = await conn.fetchrow( + """select source_table_name, source_artifact_url, + source_artifact_sha256, source_row_count + from occupational_source_table + where data_release_code = $1 and source_table_code = $2""", + args.release_code, + source_code, + ) + if source is not None and tuple(source) != ( + source_name, + source_url, + source_digest, + source_rows, + ): + raise ValueError(f"conflicting source-table identity: {source_code}") + expected = { + "scale": { + item.scale_id: ( + _SCALES_SOURCE_CODE, + item.scale_name, + item.minimum_value, + item.maximum_value, + ) + for item in scales.values() + }, + "occupation": { + item.onetsoc_code: (item.occupation_title,) for item in observations + }, + "element": {item.element_id: (item.element_name,) for item in observations}, + } + queries = { + "scale": "select scale_id, source_table_code, scale_name, minimum_value, maximum_value from occupational_scale_definition where data_release_code = $1", + "occupation": "select onetsoc_code, occupation_title from occupational_classification_entry where data_release_code = $1", + "element": "select element_id, element_name from occupational_element_definition where data_release_code = $1", + } + for kind, query in queries.items(): + for row in await conn.fetch(query, args.release_code): + source_id, *values = tuple(row) + if ( + source_id in expected[kind] + and tuple(values) != expected[kind][source_id] + ): + raise ValueError(f"conflicting {kind} identity: {source_id}") + + +async def import_ratings(args: argparse.Namespace) -> dict[str, object]: + """Validate one pinned source artifact, then transactionally UPSERT its rows.""" + _validate_args(args) + digest = hashlib.sha256(args.ratings_file.read_bytes()).hexdigest() + if digest != args.source_sha256.lower(): + raise ValueError("rating artifact SHA-256 mismatch") + scales_digest = hashlib.sha256(args.scales_file.read_bytes()).hexdigest() + if scales_digest != args.scales_sha256.lower(): + raise ValueError("scales artifact SHA-256 mismatch") + scales = read_scale_file(args.scales_file) + if len(scales) != args.scales_row_count: + raise ValueError("scales artifact row-count mismatch") + observations = read_rating_file(args.ratings_file, scales) + if len(observations) != args.source_row_count: + raise ValueError("rating artifact row-count mismatch") + conn = await asyncpg.connect(args.target_dsn) + release_partition = f"occupational_rating_release_{hashlib.sha256(args.release_code.encode()).hexdigest()[:16]}" + source_partition = f"occupational_rating_source_{hashlib.sha256(f'{args.release_code}\0{args.source_table_code}'.encode()).hexdigest()[:16]}" + try: + async with conn.transaction(): + await conn.execute( + "select pg_advisory_xact_lock(hashtextextended($1, 0))", + args.release_code, + ) + await _reject_reference_conflicts(conn, args, scales, observations) + if not re.fullmatch(r"[a-z][a-z0-9_]*", release_partition): + raise ValueError("invalid release partition identifier") + if not re.fullmatch(r"[a-z][a-z0-9_]*", source_partition): + raise ValueError("invalid source partition identifier") + await conn.execute( + """insert into occupational_data_release + (data_release_code, release_version, source_publisher_name, source_license_url) + values ($1, $2, $3, $4) + on conflict (data_release_code) do nothing""", + args.release_code, + args.release_version, + args.publisher, + args.license_url, + ) + await conn.execute( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + """ + do $$ + begin + execute format( + 'create table if not exists %I partition of occupational_rating_observation ' + 'for values in (%L) partition by list (source_table_code)', + $1, $2 + ); + end + $$; + """, + release_partition, + args.release_code, + ) + await conn.execute( # nosemgrep: python.lang.security.audit.sqli.asyncpg-sqli.asyncpg-sqli + """ + do $$ + begin + execute format( + 'create table if not exists %I partition of %I for values in (%L)', + $1, $2, $3 + ); + end + $$; + """, + source_partition, + release_partition, + args.source_table_code, + ) + await conn.execute( + """insert into occupational_source_table + (data_release_code, source_table_code, source_table_name, + source_artifact_url, source_artifact_sha256, source_row_count) + values ($1, $2, $3, $4, $5, $6) + on conflict (data_release_code, source_table_code) do nothing""", + args.release_code, + args.source_table_code, + args.source_table_name, + args.source_url, + digest, + len(observations), + ) + await conn.execute( + """insert into occupational_source_table + (data_release_code, source_table_code, source_table_name, + source_artifact_url, source_artifact_sha256, source_row_count) + values ($1, $2, 'Scales Reference', $3, $4, $5) + on conflict (data_release_code, source_table_code) do nothing""", + args.release_code, + _SCALES_SOURCE_CODE, + args.scales_url, + scales_digest, + len(scales), + ) + await conn.executemany( + """insert into occupational_scale_definition + (data_release_code, source_table_code, scale_id, scale_name, + minimum_value, maximum_value) + values ($1, $2, $3, $4, $5, $6) + on conflict (data_release_code, scale_id) do nothing""", + [ + ( + args.release_code, + _SCALES_SOURCE_CODE, + item.scale_id, + item.scale_name, + item.minimum_value, + item.maximum_value, + ) + for item in scales.values() + ], + ) + await conn.executemany( + """insert into occupational_classification_entry + (data_release_code, onetsoc_code, occupation_title) + values ($1, $2, $3) on conflict (data_release_code, onetsoc_code) do nothing""", + [ + (args.release_code, code, title) + for code, title in { + item.onetsoc_code: item.occupation_title + for item in observations + }.items() + ], + ) + await conn.executemany( + """insert into occupational_element_definition + (data_release_code, element_id, element_name) + values ($1, $2, $3) on conflict (data_release_code, element_id) do nothing""", + [ + (args.release_code, code, name) + for code, name in { + item.element_id: item.element_name for item in observations + }.items() + ], + ) + await conn.executemany( + """insert into occupational_rating_observation + (data_release_code, source_table_code, onetsoc_code, element_id, + scale_id, category_value, data_value, sample_size, standard_error, + lower_ci_bound, upper_ci_bound, recommend_suppress, not_relevant, + source_updated_month, domain_source_code) + values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) + on conflict on constraint occupational_rating_identity_key do nothing""", + [ + ( + args.release_code, + args.source_table_code, + item.onetsoc_code, + item.element_id, + item.scale_id, + item.category_value, + item.data_value, + item.sample_size, + item.standard_error, + item.lower_ci_bound, + item.upper_ci_bound, + item.recommend_suppress, + item.not_relevant, + item.source_updated_month, + item.domain_source_code, + ) + for item in observations + ], + ) + finally: + await conn.close() + return { + "release_code": args.release_code, + "source_table_code": args.source_table_code, + "imported_rows": len(observations), + "source_sha256": digest, + "scales_sha256": scales_digest, + } + + +def main() -> None: + """Run the command-line importer and print aggregate, non-identifying evidence.""" + print( + json.dumps(asyncio.run(import_ratings(_parser().parse_args())), sort_keys=True) + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_import_job_architecture.py b/tests/test_import_job_architecture.py new file mode 100644 index 000000000..965783cbf --- /dev/null +++ b/tests/test_import_job_architecture.py @@ -0,0 +1,127 @@ +"""Contracts for authorized job-family/job-series snapshot imports.""" + +import csv +from pathlib import Path + +import pytest + +from scripts.import_job_architecture import read_job_architecture + + +_FIELDS = [ + "Node Code", + "Node Kind", + "Node Name", + "Description", + "Parent Code", + "Hierarchy Relation", + "Valid From", + "Valid To", + "Occupation Scheme IRI", + "Occupation Scheme Version", + "Occupation Code", + "Occupation Relation", +] + + +def _write(path: Path, rows: list[dict[str, str]]) -> Path: + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=_FIELDS) + writer.writeheader() + writer.writerows(rows) + return path + + +def _row(code: str, kind: str, name: str, **values: str) -> dict[str, str]: + row = dict.fromkeys(_FIELDS, "") + row.update({"Node Code": code, "Node Kind": kind, "Node Name": name}, **values) + return row + + +def test_snapshot_preserves_multiple_membership_and_explicit_binding(tmp_path: Path) -> None: + path = _write( + tmp_path / "architecture.csv", + [ + _row("F-A", "job_family", "Synthetic family A"), + _row("F-B", "job_family", "Synthetic family B"), + _row( + "S-1", + "job_series", + "Synthetic series", + **{ + "Parent Code": "F-A", + "Hierarchy Relation": "source_broader", + "Valid From": "2026-01-01", + "Occupation Scheme IRI": "https://example.test/occupation-scheme", + "Occupation Scheme Version": "2026", + "Occupation Code": "SYN-1", + "Occupation Relation": "source_classification", + }, + ), + _row( + "S-1", + "job_series", + "Synthetic series", + **{ + "Parent Code": "F-B", + "Hierarchy Relation": "source_broader", + "Valid From": "2026-01-01", + "Occupation Scheme IRI": "https://example.test/occupation-scheme", + "Occupation Scheme Version": "2026", + "Occupation Code": "SYN-1", + "Occupation Relation": "source_classification", + }, + ), + ], + ) + + nodes, edges, bindings, row_count = read_job_architecture(path) + + assert row_count == 4 + assert len(nodes) == 3 + assert {(edge.broader_code, edge.narrower_code) for edge in edges} == { + ("F-A", "S-1"), + ("F-B", "S-1"), + } + assert len(bindings) == 1 + assert bindings[0].occupation_code == "SYN-1" + + +def test_label_never_creates_an_occupation_binding(tmp_path: Path) -> None: + path = _write( + tmp_path / "unbound.csv", + [_row("S-1", "job_series", "15-1252 Software developers")], + ) + + _, _, bindings, _ = read_job_architecture(path) + + assert bindings == [] + + +@pytest.mark.parametrize( + ("rows", "message"), + [ + ( + [ + _row("F-A", "job_family", "Family", **{"Parent Code": "S-1", "Hierarchy Relation": "broader"}), + _row("S-1", "job_series", "Series", **{"Parent Code": "F-A", "Hierarchy Relation": "broader"}), + ], + "cyclic", + ), + ( + [_row("S-1", "job_series", "Series", **{"Occupation Scheme IRI": "https://example.test/scheme"})], + "partial occupation binding", + ), + ( + [_row("S-1", "job_series", "Series", **{"Parent Code": "missing", "Hierarchy Relation": "broader"})], + "unknown parent", + ), + ], +) +def test_invalid_source_relationships_fail_closed( + tmp_path: Path, + rows: list[dict[str, str]], + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + read_job_architecture(_write(tmp_path / "invalid.csv", rows)) diff --git a/tests/test_import_onet_ratings.py b/tests/test_import_onet_ratings.py new file mode 100644 index 000000000..74fdc509e --- /dev/null +++ b/tests/test_import_onet_ratings.py @@ -0,0 +1,233 @@ +"""Contracts for the official O*NET occupation-rating CSV importer.""" + +import asyncio +import hashlib +from datetime import date +from decimal import Decimal +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from scripts.import_onet_ratings import ( + import_ratings, + read_rating_file, + read_scale_file, +) + + +def _write(path: Path, text: str) -> Path: + path.write_text(text, encoding="utf-8", newline="") + return path + + +def test_official_csv_preserves_decimal_missingness_and_uncertainty( + tmp_path: Path, +) -> None: + scales = read_scale_file( + _write( + tmp_path / "scales.csv", + "Scale ID,Scale Name,Minimum,Maximum\nIM,Importance,1,5\n", + ) + ) + ratings = read_rating_file( + _write( + tmp_path / "abilities.csv", + "O*NET-SOC Code,Title,Element ID,Element Name,Scale ID,Scale Name,Data Value,N,Standard Error,Lower CI Bound,Upper CI Bound,Recommend Suppress,Not Relevant,Date,Domain Source\n" + "15-1252.00,Synthetic occupation,1.A.1.a.1,Oral Comprehension,IM,Importance,4.10,120,0.08,3.94,4.26,N,,08/2026,Analyst\n", + ), + scales, + today=date(2026, 8, 27), + ) + + assert len(ratings) == 1 + assert ratings[0].data_value == Decimal("4.10") + assert ratings[0].data_value.as_tuple().exponent == -2 + assert ratings[0].standard_error == Decimal("0.08") + assert ratings[0].lower_ci_bound == Decimal("3.94") + assert ratings[0].upper_ci_bound == Decimal("4.26") + assert ratings[0].not_relevant is None + assert ratings[0].source_updated_month == "08/2026" + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("Data Value", "5.01", "outside scale"), + ("Standard Error", "-0.01", "standard error"), + ("Date", "09/2026", "future"), + ("Date", "8/2026", "invalid source update date"), + ("Recommend Suppress", "maybe", "flag"), + ], +) +def test_invalid_source_measurement_fails_before_persistence( + tmp_path: Path, + field: str, + value: str, + message: str, +) -> None: + scales = read_scale_file( + _write( + tmp_path / "scales.csv", + "Scale ID,Scale Name,Minimum,Maximum\nIM,Importance,1,5\n", + ) + ) + row = { + "O*NET-SOC Code": "15-1252.00", + "Title": "Synthetic occupation", + "Element ID": "1.A.1.a.1", + "Element Name": "Oral Comprehension", + "Scale ID": "IM", + "Scale Name": "Importance", + "Data Value": "4.10", + "N": "120", + "Standard Error": "0.08", + "Lower CI Bound": "3.94", + "Upper CI Bound": "4.26", + "Recommend Suppress": "N", + "Not Relevant": "", + "Date": "08/2026", + "Domain Source": "Analyst", + } + row[field] = value + path = tmp_path / "invalid.csv" + import csv + + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=row) + writer.writeheader() + writer.writerow(row) + + with pytest.raises(ValueError, match=message): + read_rating_file(path, scales, today=date(2026, 8, 27)) + + +def test_conflicting_reference_name_fails_closed(tmp_path: Path) -> None: + scales = read_scale_file( + _write( + tmp_path / "scales.csv", + "Scale ID,Scale Name,Minimum,Maximum\nIM,Importance,1,5\n", + ) + ) + path = _write( + tmp_path / "conflict.csv", + "O*NET-SOC Code,Title,Element ID,Element Name,Scale ID,Scale Name,Data Value,N,Standard Error,Lower CI Bound,Upper CI Bound,Recommend Suppress,Not Relevant,Date,Domain Source\n" + "15-1252.00,Synthetic occupation,1.A.1.a.1,Oral Comprehension,IM,Importance,4.10,,,,,N,,08/2026,Analyst\n" + "15-1252.00,Conflicting title,1.A.1.a.1,Oral Comprehension,IM,Importance,4.20,,,,,N,,08/2026,Analyst\n", + ) + + with pytest.raises(ValueError, match="conflicting occupation title"): + read_rating_file(path, scales, today=date(2026, 8, 27)) + + +def test_short_csv_row_fails_with_import_error(tmp_path: Path) -> None: + scales = read_scale_file( + _write( + tmp_path / "scales.csv", + "Scale ID,Scale Name,Minimum,Maximum\nIM,Importance,1,5\n", + ) + ) + path = _write( + tmp_path / "short.csv", + "O*NET-SOC Code,Title,Element ID,Element Name,Scale ID,Scale Name,Data Value,Date,Domain Source\n" + "15-1252.00,Synthetic occupation,1.A.1.a.1\n", + ) + + with pytest.raises(ValueError, match="malformed CSV row: 2"): + read_rating_file(path, scales, today=date(2026, 8, 27)) + + +def test_category_table_may_omit_not_relevant_without_inventing_false( + tmp_path: Path, +) -> None: + scales = read_scale_file( + _write( + tmp_path / "scales.csv", + "Scale ID,Scale Name,Minimum,Maximum\nPT,Percent,0,100\n", + ) + ) + rows = read_rating_file( + _write( + tmp_path / "education.csv", + "O*NET-SOC Code,Title,Element ID,Element Name,Scale ID,Scale Name,Category,Data Value,N,Standard Error,Lower CI Bound,Upper CI Bound,Recommend Suppress,Date,Domain Source\n" + "15-1252.00,Synthetic occupation,2.D.1,Education,PT,Percent,6,42.50,100,1.2,40.1,44.9,N,08/2026,Incumbent\n", + ), + scales, + today=date(2026, 8, 27), + ) + + assert rows[0].category_value == 6 + assert rows[0].not_relevant is None + + +def test_machine_generated_profile_keeps_unpublished_uncertainty_missing( + tmp_path: Path, +) -> None: + scales = read_scale_file( + _write( + tmp_path / "scales.csv", + "Scale ID,Scale Name,Minimum,Maximum\nDR,Distinctiveness Rank,0,7\n", + ) + ) + rows = read_rating_file( + _write( + tmp_path / "work_styles.csv", + "O*NET-SOC Code,Title,Element ID,Element Name,Scale ID,Scale Name,Data Value,Date,Domain Source\n" + "15-1252.00,Synthetic occupation,1.D.1.a,Innovation,DR,Distinctiveness Rank,7.00,08/2026,AI/Expert\n", + ), + scales, + today=date(2026, 8, 27), + ) + + assert rows[0].sample_size is None + assert rows[0].standard_error is None + assert rows[0].recommend_suppress is None + + +def test_scales_digest_fails_before_database_connection( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + scales = _write( + tmp_path / "scales.csv", + "Scale ID,Scale Name,Minimum,Maximum\nIM,Importance,1,5\n", + ) + ratings = _write( + tmp_path / "abilities.csv", + "O*NET-SOC Code,Title,Element ID,Element Name,Scale ID,Scale Name,Data Value,N,Standard Error,Lower CI Bound,Upper CI Bound,Recommend Suppress,Not Relevant,Date,Domain Source\n" + "15-1252.00,Synthetic occupation,1.A.1.a.1,Oral Comprehension,IM,Importance,4.10,120,0.08,3.94,4.26,N,,08/2026,Analyst\n", + ) + connected = False + + async def fake_connect(_dsn: str) -> None: + nonlocal connected + connected = True + + monkeypatch.setattr("scripts.import_onet_ratings.asyncpg.connect", fake_connect) + args = SimpleNamespace( + target_dsn="postgresql://unused", + release_code="onet-31.0-synthetic", + release_version="31.0-synthetic", + source_table_code="abilities", + source_table_name="Abilities", + source_url="https://example.test/abilities.csv", + source_sha256=hashlib.sha256(ratings.read_bytes()).hexdigest(), + source_row_count=1, + publisher="Synthetic publisher", + license_url="https://example.test/license", + scales_file=scales, + scales_url="https://example.test/scales.csv", + scales_sha256="0" * 64, + scales_row_count=1, + ratings_file=ratings, + ) + + args.source_url = "https://:secret@example.test/abilities.csv" + with pytest.raises(ValueError, match="without userinfo"): + asyncio.run(import_ratings(args)) + assert connected is False + + args.source_url = "https://example.test/abilities.csv" + with pytest.raises(ValueError, match="scales artifact SHA-256 mismatch"): + asyncio.run(import_ratings(args)) + assert connected is False diff --git a/tests/test_job_architecture_schema.py b/tests/test_job_architecture_schema.py new file mode 100644 index 000000000..765df2ad6 --- /dev/null +++ b/tests/test_job_architecture_schema.py @@ -0,0 +1,19 @@ +"""Static schema guards for the authorized job-architecture contract.""" + +from pathlib import Path + + +def test_job_architecture_schema_is_normalized_and_immutable() -> None: + migration = Path("migrations/0223_authorized_job_architecture.sql").read_text() + + for table in ( + "job_architecture_source", + "job_architecture_node", + "job_architecture_hierarchy_edge", + "job_architecture_occupation_binding", + ): + assert f"create table if not exists {table}" in migration + assert "job_architecture_kind_code in ('job_family', 'job_series')" in migration + assert "reject_job_architecture_mutation" in migration + assert "broader_job_architecture_code <> narrower_job_architecture_code" in migration + assert "occupation_scheme_iri" in migration diff --git a/tests/test_occupation_rating_ingestion.py b/tests/test_occupation_rating_ingestion.py new file mode 100644 index 000000000..3f8ec21dd --- /dev/null +++ b/tests/test_occupation_rating_ingestion.py @@ -0,0 +1,257 @@ +"""Tests for the provenance-bearing occupation-rating read projection.""" + +import asyncio +from decimal import Decimal + +from backend.app.main import ( + read_occupation_rating_sources, + read_occupation_ratings, + read_rating_source_occupations, +) +from backend.app.occupation_rating_ingestion import ( + fetch_occupation_rating_sources, + fetch_occupation_ratings, + fetch_rating_source_occupations, +) + + +class FakeConnection: + """Minimal ordered asyncpg stand-in for one projection query.""" + + def __init__(self, source, rows=()): + self.source = source + self.rows = list(rows) + self.fetch_called = False + self.last_fetch_query = "" + + async def fetchrow(self, _query: str, *_args: object): + """Return configured source metadata.""" + return self.source + + async def fetch(self, query: str, *_args: object): + """Return configured observation rows.""" + self.fetch_called = True + self.last_fetch_query = query + return self.rows + + +class FakeAcquire: + """Async pool-acquire context for route wiring.""" + + def __init__(self, conn: FakeConnection): + self.conn = conn + + async def __aenter__(self) -> FakeConnection: + """Return the configured connection.""" + return self.conn + + async def __aexit__(self, *_args: object) -> None: + """Release without external state.""" + + +class FakePool: + """Minimal pool exposing one acquisition context.""" + + def __init__(self, conn: FakeConnection): + self.conn = conn + + def acquire(self) -> FakeAcquire: + """Return one deterministic acquisition context.""" + return FakeAcquire(self.conn) + + +def test_unimported_source_is_not_an_empty_observed_profile() -> None: + conn = FakeConnection(None) + + result = asyncio.run( + fetch_occupation_ratings( + conn, + data_release_code="onet-31.0", + source_table_code="abilities", + onetsoc_code="15-1252.00", + limit=100, + offset=0, + ) + ) + + assert result["source_available"] is False + assert result["items"] == [] + assert conn.fetch_called is False + + +def test_rating_projection_preserves_exact_decimal_and_warning_flags() -> None: + source = { + "source_table_name": "Abilities", + "source_artifact_url": "https://example.test/abilities.csv", + "source_artifact_sha256": "a" * 64, + "source_row_count": 2, + "scale_artifact_url": "https://example.test/scales.csv", + "scale_artifact_sha256": "b" * 64, + "scale_source_row_count": 33, + } + row = { + "element_id": "1.A.1.a.1", + "element_name": "Oral Comprehension", + "scale_id": "IM", + "scale_name": "Importance", + "minimum_value": Decimal("1.00"), + "maximum_value": Decimal("5.00"), + "category_value": None, + "data_value": Decimal("4.10"), + "sample_size": 8, + "standard_error": Decimal("0.1830"), + "lower_ci_bound": Decimal("3.7414"), + "upper_ci_bound": Decimal("4.4586"), + "recommend_suppress": True, + "not_relevant": None, + "source_updated_month": "08/2026", + "domain_source_code": "Analyst", + } + conn = FakeConnection(source, (row, row)) + + result = asyncio.run( + fetch_occupation_ratings( + conn, + data_release_code="onet-31.0", + source_table_code="abilities", + onetsoc_code="15-1252.00", + limit=1, + offset=0, + ) + ) + + item = result["items"][0] + assert item["data_value"] == "4.10" + assert item["standard_error"] == "0.1830" + assert item["recommend_suppress"] is True + assert item["not_relevant"] is None + assert result["source"]["scale_artifact_sha256"] == "b" * 64 + assert result["next_offset"] == 1 + + +def test_empty_profile_keeps_imported_scale_provenance() -> None: + source = { + "source_table_name": "Abilities", + "source_artifact_url": "https://example.test/abilities.csv", + "source_artifact_sha256": "a" * 64, + "source_row_count": 2, + "scale_artifact_url": "https://example.test/scales.csv", + "scale_artifact_sha256": "b" * 64, + "scale_source_row_count": 33, + } + + result = asyncio.run( + fetch_occupation_ratings( + FakeConnection(source), + data_release_code="onet-31.0", + source_table_code="abilities", + onetsoc_code="15-9999.99", + limit=100, + offset=0, + ) + ) + + assert result["source_available"] is True + assert result["items"] == [] + assert result["source"]["scale_artifact_sha256"] == "b" * 64 + + +def test_authenticated_route_delegates_to_bounded_projection() -> None: + result = asyncio.run( + read_occupation_ratings( + onetsoc_code="15-1252.00", + data_release_code="onet-31.0", + source_table_code="abilities", + limit=100, + offset=0, + _account=object(), + pool=FakePool(FakeConnection(None)), + ) + ) + + assert result["source_available"] is False + + +def test_source_catalog_returns_only_query_selected_imports() -> None: + source = { + "data_release_code": "onet-31.0", + "release_version": "31.0", + "source_publisher_name": "National Center for O*NET Development", + "source_license_url": "https://example.test/license", + "source_table_code": "abilities", + "source_table_name": "Abilities", + "source_artifact_url": "https://example.test/abilities.csv", + "source_artifact_sha256": "a" * 64, + "source_row_count": 94640, + } + + conn = FakeConnection(None, (source,)) + result = asyncio.run(fetch_occupation_rating_sources(conn)) + + assert result == {"sources": [source]} + assert "source_table_code <> 'scales_reference'" in conn.last_fetch_query + assert "and exists" in conn.last_fetch_query + + +def test_authenticated_source_catalog_route_uses_shared_projection() -> None: + result = asyncio.run( + read_occupation_rating_sources( + _account=object(), + pool=FakePool(FakeConnection(None)), + ) + ) + + assert result == {"sources": []} + + +def test_source_occupation_catalog_distinguishes_unavailable_from_empty() -> None: + unavailable = asyncio.run( + fetch_rating_source_occupations( + FakeConnection(None), + data_release_code="onet-31.0", + source_table_code="abilities", + ) + ) + empty = asyncio.run( + fetch_rating_source_occupations( + FakeConnection({"exists": 1}), + data_release_code="onet-31.0", + source_table_code="abilities", + ) + ) + + assert unavailable["source_available"] is False + assert empty["source_available"] is True + assert empty["occupations"] == [] + + +def test_source_occupation_catalog_returns_authoritative_codes_and_titles() -> None: + rows = ( + {"onetsoc_code": "11-1011.00", "occupation_title": "Chief Executives"}, + {"onetsoc_code": "15-1252.00", "occupation_title": "Software Developers"}, + ) + conn = FakeConnection({"exists": 1}, rows) + + result = asyncio.run( + fetch_rating_source_occupations( + conn, + data_release_code="onet-31.0", + source_table_code="abilities", + ) + ) + + assert result["occupations"] == list(rows) + assert "and exists" in conn.last_fetch_query + + +def test_authenticated_source_occupation_route_uses_shared_projection() -> None: + result = asyncio.run( + read_rating_source_occupations( + data_release_code="onet-31.0", + source_table_code="abilities", + _account=object(), + pool=FakePool(FakeConnection({"exists": 1})), + ) + ) + + assert result["source_available"] is True diff --git a/tests/test_onet_rating_schema.py b/tests/test_onet_rating_schema.py new file mode 100644 index 000000000..095dbdddf --- /dev/null +++ b/tests/test_onet_rating_schema.py @@ -0,0 +1,34 @@ +"""Static contracts for the normalized O*NET rating observation store.""" + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MIGRATION = ROOT / "migrations" / "0222_onet_rating_observation_store.sql" + + +def test_migration_declares_normalized_partitioned_observation_contract() -> None: + assert MIGRATION.is_file() + sql = MIGRATION.read_text(encoding="utf-8").casefold() + for table_name in ( + "occupational_data_release", + "occupational_source_table", + "occupational_scale_definition", + "occupational_classification_entry", + "occupational_element_definition", + "occupational_rating_observation", + ): + assert f"create table if not exists {table_name}" in sql + assert "partition by list (data_release_code)" in sql + assert "unique nulls not distinct" in sql + assert "occupational_scale_source_table_fkey" in sql + assert "validate_occupational_rating_insert" in sql + assert "reject_occupational_rating_mutation" in sql + assert "before truncate on occupational_rating_observation" in sql + assert "identity conflicts with immutable evidence" in sql + assert "recommend_suppress" in sql + assert "not_relevant" in sql + assert "standard_error" in sql + assert "lower_ci_bound" in sql + assert "upper_ci_bound" in sql + assert "source_updated_month text not null" in sql + assert "source_updated_date" not in sql