diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index edb9a32..9f8de3e 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -2,7 +2,7 @@ name: Learning Contracts Quality on: pull_request: - branches: [develop, main] + branches: [develop, main, agent/bootstrap-learning-contracts] push: branches: [develop, main] @@ -22,6 +22,14 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + - name: Set up pinned Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Install hash-locked contract validator + run: >- + python -m pip install --disable-pip-version-check --only-binary=:all: + --require-hashes -r requirements-contracts-ci-hashes.txt - name: Validate documentation and schemas shell: python3 {0} run: | @@ -39,6 +47,7 @@ jobs: Path("docs/ARCHITECTURE.md"), Path("docs/adr/0001-contract-authority-boundary.md"), Path("docs/doctoring/STANDARD_TRACEABILITY.md"), + Path("requirements-contracts-ci-hashes.txt"), schema_path, ] missing = [str(path) for path in required if not path.is_file()] @@ -80,3 +89,11 @@ jobs: raise SystemExit(f"unresolved bootstrap marker {marker!r} in {path}") print("learning interoperability bootstrap contract validation passed") + - name: Run CEFR schema and blueprint authority regressions + run: python -m unittest -v tests/test_validate_cefr_profile.py + - name: Validate CEFR language-assessment profile + run: python scripts/validate_cefr_profile.py + - name: Compile validator and tests + run: python -m compileall -q scripts tests + - name: Verify clean diff + run: git diff --check diff --git a/CHANGELOG.md b/CHANGELOG.md index c08624d..eb2e576 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,3 +8,9 @@ - Standards traceability baseline for xAPI, cmi5, LTI, QTI, CASE, Open Badges, CLR, and accessibility. - Versioned learning-domain event envelope schema. - Repository agent development rules. +- Rights-safe `cwl_cefr_language_assessment/v1` blueprint, task, and immutable domain-result contracts. +- Hash-locked JSON Schema Draft 2020-12 validation with offline `$ref` resolution and metaschema checks. +- CEFR positive/negative fixture gates covering standard-setting, protected content, probability mass, required-domain completeness, blueprint overall-reporting authority, and exact reporting-policy equality. +- Exact target-language profile/RLD revision or dated-snapshot requirement; mutable revision aliases fail closed. +- Distinct `cefr_aligned`, `cefr_linked`, and certification-decision evidence gates, including governed certification authority and policy references. +- CEFR architecture decision, research doctoring, fixed standards traceability, design specification, implementation plan, and focused regression tests. diff --git a/README.md b/README.md index e871da6..ea65963 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,14 @@ Shared, versioned interoperability contracts for the CWL Learning Platform. This repository contains schemas, profiles, generated-client contracts, and conformance fixtures shared by the Learning Management Platform, Learning Content Studio, Learning Record Store, Psychometrics Commons, and other CWL consumers. -Initial standards portfolio: xAPI 2.0, cmi5 Quartz compatibility, LTI 1.3, QTI 3, CASE 1.1, Open Badges 3.0, CLR 2.0, and accessibility-related contract metadata. +Initial standards portfolio: xAPI 2.0, cmi5 Quartz compatibility, LTI 1.3, QTI 3, CASE 1.1, Open Badges 3.0, CLR 2.0, CEFR language-assessment metadata, and accessibility-related contract metadata. It contains no product runtime state and no application database. +## Profiles + +- `profiles/cwl_cefr_language_assessment/v1` defines a rights-safe CEFR assessment blueprint, task metadata, immutable domain-result snapshot, and executable fixtures. It stores references rather than official descriptor prose, task content, responses, media, or numerical scoring payloads. + ## Branching Product work targets `develop`; release promotion to `main` occurs only after exact-head review and required checks. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3a6d1b3..d3f0a28 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,12 +2,20 @@ This repository owns versioned learning interoperability contracts and no application runtime state. -Primary families: xAPI 2.0, cmi5 Quartz compatibility, LTI 1.3, QTI 3, CASE 1.1, Open Badges 3.0, and CLR 2.0. +Primary families: xAPI 2.0, cmi5 Quartz compatibility, LTI 1.3, QTI 3, CASE 1.1, Open Badges 3.0, CLR 2.0, and rights-safe CEFR language-assessment metadata. Authority boundaries: -- Learning Management Platform: offerings, enrollment, progression, completion policy. -- Learning Content Studio: authoring state and immutable releases. +- Learning Management Platform: offerings, enrollment, progression, placement/completion policy, and credential references. +- Learning Content Studio: authoring state, tasks, rubrics, media, rights, and immutable releases. - Learning Record Store: xAPI statements and document resources. -- Psychometrics Commons: assessment sessions, responses, and score snapshots. +- Psychometrics Commons: assessment blueprints/instrument publication, sessions, responses, and immutable result snapshots. +- fast-mlsirm: psychometric estimation, many-facet calibration, standard-setting/cut-score evidence, linking, DIF, uncertainty, and recovery. +- Semantic Data Portal: rights-aware descriptor/RLD/competency catalog references where adopted. +- TEPP: longitudinal, temporal, multilevel, and multiple-membership language-development analysis. +- contextual-orchestrator: bounded AI-rater orchestration; its observations are evidence, not score authority. + +The CEFR profile stores immutable references and bounded result-envelope summaries only. Allowed summaries include domain `level_probabilities`, `credible_level_set`, `standard_error`, measurement status, descriptor-coverage references, claim status, limitations, and evidence identities. The profile never stores official descriptor prose, authored task content, raw responses, audio, provider payloads, PII, item/person/rater parameter arrays, likelihood traces, raw scores, response-level calculations, or other scoring-engine internals. + +A reported overall result is not self-authorizing. The executable validator resolves the exact immutable assessment blueprint and accepts an overall result only when that blueprint permits overall reporting, every required domain is measured, and the result cites the blueprint's exact reporting policy. Consumers integrate through versioned contracts; cross-repository database access is not part of the architecture. diff --git a/docs/adr/0002-cefr-language-assessment-profile.md b/docs/adr/0002-cefr-language-assessment-profile.md new file mode 100644 index 0000000..34601ec --- /dev/null +++ b/docs/adr/0002-cefr-language-assessment-profile.md @@ -0,0 +1,61 @@ +# ADR 0002: Rights-safe CEFR language-assessment profile + +## Status + +Accepted for the active stacked PR; not protected-`develop` truth until merged. + +Approved by: ContextualWisdomLab repository owner +Approval date: 2026-08-27 + +## Context + +CWL can compose learning management, content authoring, assessment hosting, psychometric estimation, model orchestration and longitudinal analysis. It lacks a shared contract for reporting a domain-level language-proficiency profile in relation to the Common European Framework of Reference for Languages (CEFR). + +A naive contract would create material risks: + +- copying official descriptor prose or translations into a public repository; +- treating CEFR as a single equally spaced numerical score; +- averaging incomplete skill results into one overall label; +- allowing a result document to self-authorize an overall label; +- confusing alignment with empirical linking or certification; +- moving scoring arithmetic or result authority into an interoperability repository; +- leaking raw responses, audio, task content, provider payloads or PII. + +## Decision + +Create `profiles/cwl_cefr_language_assessment/v1` as a metadata-only, versioned interoperability profile. + +The profile: + +1. references the CEFR Companion Volume, an exact target-language RLD/profile source and immutable revision/snapshot, descriptors, task/rubric releases, scoring profile, cut-score revision, standard-setting study and validation evidence by opaque identity; +2. supports Pre-A1, A1, A2, A2+, B1, B1+, B2, B2+, C1 and C2 while keeping domain results explicit; +3. models reception, production, interaction and mediation through typed activity-domain codes; +4. requires level probabilities, uncertainty, credible-level sets and descriptor-coverage references for every measured domain; +5. structurally validates every document with its committed JSON Schema Draft 2020-12 schema before semantic checks; +6. resolves the immutable assessment blueprint before accepting a result and prohibits an overall level unless that blueprint authorizes overall reporting, every required domain is measured and the exact blueprint reporting policy is cited; +7. requires standard-setting and linking-validation references before `cefr_linked` claims; +8. requires an exact certification authority and certification policy, in addition to linked evidence, before a `certification_decision` claim; +9. requires standard-setting evidence for high-stakes or certification blueprints; +10. rejects copied descriptor/task/response payload fields in fixtures and the quality gate; +11. leaves numerical scoring in fast-mlsirm, instrument/result authority in Psychometrics Commons, content authority in Learning Content Studio, and learner actions in Learning Management Platform. + +## Consequences + +### Positive + +- Consumers can exchange an auditable proficiency profile without cross-service SQL or payload duplication. +- Domain-level uncertainty and incomplete evidence remain visible. +- A result cannot escalate its own overall-reporting or certification authority. +- Rights and claim-state boundaries fail closed. +- Future target languages can use different exact RLD/profile authorities without changing the common contract. + +### Negative + +- The profile cannot prove that an assessment is linked to the CEFR. +- Consumers must resolve referenced artifacts through their authorized owning systems. +- The CI gate adds a small hash-locked Draft 2020-12 validator dependency set. +- Generated SDKs and cross-repository consumer conformance remain a later release slice. + +## Reversal conditions + +A new major profile version is required if the CEFR framework representation, level system, domain taxonomy, claim semantics, blueprint-authorization semantics or result envelope changes incompatibly. New optional evidence references may be added compatibly only with conformance fixtures and consumer contract tests. diff --git a/docs/doctoring/CEFR_LANGUAGE_ASSESSMENT.md b/docs/doctoring/CEFR_LANGUAGE_ASSESSMENT.md new file mode 100644 index 0000000..4f1f37e --- /dev/null +++ b/docs/doctoring/CEFR_LANGUAGE_ASSESSMENT.md @@ -0,0 +1,100 @@ +# CEFR language-assessment research and standards basis + +## Product interpretation + +The CEFR is a non-prescriptive reference framework and common meta-language for curriculum, materials and assessment. It describes six common levels and three plus levels, and the Companion Volume adds Pre-A1 and expands descriptors for mediation, online interaction, plurilingual/pluricultural competence, phonology and signing. + +This repository does not reproduce the descriptor corpus. A contract stores only an immutable `descriptor_reference`, exact framework authority, exact target-language profile/RLD source identity, and an immutable publisher revision or dated source snapshot. + +## Assessment-development boundary + +The 2026 revised *Manual for Language Test Development and Examining—For use with the CEFR* is the Council of Europe/ALTE test-development baseline adopted by this profile. The separate 2009 *Manual for Relating Language Examinations to the CEFR* and its explicitly identified supporting grids provide transparent, cumulative procedures for supporting a linking claim. The Council of Europe does not verify or validate an examination provider's claimed link. + +The product claim states therefore have different evidence gates: + +```text +experimental +- research-only contract evidence; no operational CEFR interpretation claim + +cefr_aligned +- the blueprint references CEFR constructs, descriptors, and a target-language profile; +- no empirical examination-linking claim is made + +cefr_linked +- exact standard-setting evidence and empirical linking/classification-validation + evidence are pinned in addition to the aligned blueprint + +certification_decision +- the result is already CEFR-linked; +- an exact certification authority and certification policy are pinned; +- the governed decision and publication requirements of that authority are met +``` + +A label cannot advance by editing narrative copy. A `cefr_aligned` result does not become `cefr_linked` without standard-setting and empirical validation, and a linked result does not become a certification decision without a governed decision authority and policy. + +## Language-specific content + +Reference Level Descriptions are language-specific inventories of linguistic forms and communicative content. CEFR profile v1 requires every target-language blueprint to pin both: + +```text +language_reference_level_description_reference +language_reference_level_description_revision +``` + +The revision must be an immutable publisher revision, edition, digest-bound snapshot, or dated registry snapshot. Mutable aliases such as `latest` or `current` are rejected. Version 1 has no no-RLD exception; introducing one would require an explicit contract change, new claim semantics, and positive/negative conformance fixtures. + +The public Council of Europe English RLD registry identifies the English Profile programme but exposes no single immutable edition identifier. The English fixtures therefore pin the authority as `coe_rld_registry_english_profile` and the dated registry snapshot `english_profile_registry_snapshot_2026_08_27`. That snapshot is a source-discovery identity, not a claim that all English Profile research has become one fixed operational RLD edition. + +For content-specification research, the exact published forerunner retained in doctoring is North, Ortega, and Sheehan's *A Core Inventory for General English* (2010; ISBN 978-0-86355-653-1). It is not silently substituted for the English Profile registry identity in the contract. + +## Measurement governance + +The 2014 *Standards for Educational and Psychological Testing* governs intended interpretation and use, validity evidence, reliability/precision, fairness, administration, reporting and the rights of test takers. CEFR alignment alone does not establish these properties. + +The downstream scientific owner must evaluate, as applicable: + +- multidimensional structure and local dependence; +- rater, task, criterion, occasion and scoring-engine facets; +- standard-setting and cut-score uncertainty; +- classification consistency and decision error; +- form linking and anchor stability; +- language, population, mode and accommodation DIF/invariance; +- human/AI rater drift and adjudication; +- true-parameter and classification recovery; +- longitudinal comparability before change interpretation. + +## Fixed examination-linking sources + +This profile does not use mutable labels such as “current manual and supplements.” It pins these exact sources where applicable: + +- Council of Europe (2009), *Relating Language Examinations to the CEFR: A Manual*; +- Council of Europe (2012), *Highlights from the Manual for Relating Language Examinations to the CEFR* (ISBN 978-92-871-7169-6); +- Council of Europe, *CEFR Speaking Content Analysis Grid*, February 2014; +- Council of Europe, *CEFR Writing Content Analysis Grid*, version 3.1. + +Additional Council materials require their own exact title, version/date, source, digest or snapshot identity before they enter a blueprint or validation record. + +## APA 7th references + +American Educational Research Association, American Psychological Association, & National Council on Measurement in Education. (2014). *Standards for educational and psychological testing*. American Educational Research Association. + +Association of Language Testers in Europe. (2026). *New revised manual for language test development and examining: For use with the CEFR*. Council of Europe. + +Council of Europe. (2009). *Relating language examinations to the Common European Framework of Reference for Languages: Learning, teaching, assessment (CEFR): A manual*. Council of Europe. + +Council of Europe. (2012). *Highlights from the manual for relating language examinations to the CEFR*. Council of Europe. ISBN 978-92-871-7169-6. + +Council of Europe. (2020). *Common European Framework of Reference for Languages: Learning, teaching, assessment—Companion volume*. Council of Europe Publishing. + +North, B., Ortega, A., & Sheehan, S. (2010). *A core inventory for general English*. British Council and EAQUALS. ISBN 978-0-86355-653-1. + +## Official sources + +- https://www.coe.int/en/web/common-european-framework-reference-languages/cefr-companion-volume-and-its-language-versions +- https://www.coe.int/en/web/common-european-framework-reference-languages/introduction-and-context +- https://www.coe.int/en/web/common-european-framework-reference-languages/cefr-descriptors +- https://www.coe.int/en/web/common-european-framework-reference-languages/cefr-reference-level-descriptions-language-by-language-components-and-forerunners +- https://www.coe.int/en/web/common-european-framework-reference-languages/english +- https://www.coe.int/en/web/education/-/manual-for-language-test-development-and-examining-1 +- https://www.coe.int/en/web/common-european-framework-reference-languages/relating-examinations-to-the-cefr +- https://www.eaquals.org/resources/a-core-inventory-for-general-english/ diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index 70961fc..f209b3a 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -2,7 +2,7 @@ Adoption status and conformance evidence are intentionally separate. `Adopt` records a product decision; it does not imply implementation conformance or third-party certification. -| Standard | Revision | Normative source | Scope | Adoption status | Evidence status | +| Standard or source | Revision | Normative or authoritative source | Scope | Adoption status | Evidence status | |---|---|---|---|---|---| | xAPI / ISO/IEC/IEEE 39274-1-1 | xAPI 2.0; ISO/IEC/IEEE 39274-1-1:2025 | https://www.iso.org/standard/91131.html | Canonical learning-experience record contract | Adopt | Not evidenced | | cmi5 Quartz | Quartz, 1st Edition (2016), xAPI 1.0.3 compatibility | https://github.com/AICC/CMI-5_Spec_Current | Version-pinned LMS launch and package compatibility | Adopt as compatibility profile | Not evidenced | @@ -15,7 +15,20 @@ Adoption status and conformance evidence are intentionally separate. `Adopt` rec | CASE Service | 1.1 Final | https://standards.1edtech.org/case/ | Competency and learning-outcome interchange | Adopt | Not evidenced | | Open Badges | 3.0 | https://www.1edtech.org/standards/open-badges | Portable achievement credential | Adopt | Not evidenced | | Comprehensive Learner Record | 2.0 | https://www.1edtech.org/standards/clr | Portable learner achievement record | Adopt | Not evidenced | +| CEFR Companion Volume | 2020 | https://www.coe.int/en/web/common-european-framework-reference-languages/cefr-companion-volume-and-its-language-versions | Framework, levels, communicative modes, descriptor-reference baseline | Adopt as reference-only profile | Contract fixtures on active PR; no assessment-linking claim | +| Manual for Language Test Development and Examining | Revised 2026 edition | https://www.coe.int/en/web/education/-/manual-for-language-test-development-and-examining-1 | CEFR-related language-test development baseline | Adopt for product/scientific traceability | Documentation only | +| Manual for Relating Examinations to the CEFR | 2009 edition | https://www.coe.int/en/web/common-european-framework-reference-languages/relating-examinations-to-the-cefr | Familiarisation, specification, standardisation, standard setting and empirical validation | Adopt for linking-claim gate | No linking study evidenced | +| Highlights from the Manual for Relating Examinations to the CEFR | 2012; ISBN 978-92-871-7169-6 | https://www.coe.int/en/web/common-european-framework-reference-languages/relating-examinations-to-the-cefr | Fixed summary reference for examination-linking practice | Adopt for reviewer orientation | Documentation only | +| CEFR Speaking Content Analysis Grid | February 2014 | https://www.coe.int/en/web/common-european-framework-reference-languages/relating-examinations-to-the-cefr | Exact speaking-content analysis support artifact | Adopt when speaking-validation evidence uses it | Not evidenced | +| CEFR Writing Content Analysis Grid | Version 3.1 | https://www.coe.int/en/web/common-european-framework-reference-languages/relating-examinations-to-the-cefr | Exact writing-content analysis support artifact | Adopt when writing-validation evidence uses it | Not evidenced | +| CEFR English RLD registry / English Profile | Registry snapshot dated 2026-08-27; source exposes no single immutable programme edition | https://www.coe.int/en/web/common-european-framework-reference-languages/english | Target-language authority/source discovery for English; blueprint pins `english_profile_registry_snapshot_2026_08_27` | Adopt as exact dated source snapshot, not as a final empirical validity artifact | No descriptor prose copied; downstream English profile evidence required | +| A Core Inventory for General English | 2010; ISBN 978-0-86355-653-1 | https://www.eaquals.org/resources/a-core-inventory-for-general-english/ | Exact English content-specification forerunner retained for research traceability | Reference only; not silently substituted for the English Profile registry | Downstream content-validation evidence required | +| Standards for Educational and Psychological Testing | 2014 | https://www.testingstandards.net/ | Intended interpretation/use, validity, reliability/precision, fairness and reporting | Adopt for assessment governance | Downstream product evidence required | | WCAG | 2.2, W3C Recommendation 2024-12-12 | https://www.w3.org/TR/WCAG22/ | Accessible learning and contract-facing web content | Adopt | Not evidenced | | ATAG | 2.0, W3C Recommendation 2015-09-24 | https://www.w3.org/TR/ATAG20/ | Accessible authoring-tool contract | Adopt | Not evidenced | +The Council of Europe does not verify or validate an examination provider's CEFR link. This repository must not use the Council of Europe logo or the European emblem to imply certification or endorsement. + +A target-language blueprint must pin an exact publisher revision, edition, digest-bound snapshot, or dated registry snapshot. Mutable aliases such as `latest`, `current`, `head`, or `main` do not satisfy the contract. A dated registry snapshot is a reproducibility boundary, not a substitute for task, standard-setting, linking, DIF/invariance, or classification-validation evidence. + Every implementation PR that claims conformance must link the precise standard revision, normative requirement, implementation location, and executable evidence. Certification claims require the applicable certification process and may not be inferred from implementation alone. diff --git a/docs/superpowers/plans/2026-08-27-cefr-language-assessment-profile.md b/docs/superpowers/plans/2026-08-27-cefr-language-assessment-profile.md new file mode 100644 index 0000000..fe1b165 --- /dev/null +++ b/docs/superpowers/plans/2026-08-27-cefr-language-assessment-profile.md @@ -0,0 +1,102 @@ +# CEFR Language Assessment Profile Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish a rights-safe CEFR assessment blueprint, task, and immutable domain-result contract with executable positive and negative fixtures. + +**Architecture:** The contract repository carries metadata references and conformance evidence only. Learning Content Studio owns authored content, Psychometrics Commons owns sessions/results, fast-mlsirm owns numerical psychometrics, LMS owns placement/completion actions, and TEPP owns longitudinal analysis. + +**Tech Stack:** JSON Schema Draft 2020-12, hash-locked `jsonschema`/`referencing`, standard-library `unittest`, JSON fixtures, Markdown ADR/doctoring, and GitHub Actions. + +**Spec:** `docs/superpowers/specs/2026-08-27-cefr-language-assessment-profile-design.md` + +## Global Constraints + +- Target branch remains the bootstrap PR branch until PR #1 integrates. +- No runtime database or application state. +- No official CEFR descriptor prose, translations, task content, responses, audio, provider payloads or PII. +- No Python/Rust numerical arithmetic. +- Every claim and artifact is version-pinned. +- Exact-head review and required checks remain mandatory. + +--- + +### Task 1: Common CEFR definitions + +**Files:** +- Create: `profiles/cwl_cefr_language_assessment/v1/schemas/cefr-common.schema.json` + +- [x] Define Pre-A1, A1/A2/B1/B2/C1/C2 and A2+/B1+/B2+ codes. +- [x] Define reception, production, interaction and mediation. +- [x] Define activity domains without descriptor prose. +- [x] Pin Draft 2020-12, schema version and published `$id`. + +### Task 2: Blueprint and task contracts + +**Files:** +- Create: `profiles/cwl_cefr_language_assessment/v1/schemas/assessment-blueprint.schema.json` +- Create: `profiles/cwl_cefr_language_assessment/v1/schemas/task-specification.schema.json` + +- [x] Require exact target-language profile/RLD authority and immutable revision/snapshot. +- [x] Require instrument, scoring, cut-score and validation references. +- [x] Require standard-setting evidence for high-stakes/certification blueprints. +- [x] Require mode-specific task evidence and rubric references for constructed responses. +- [x] Prohibit copied descriptor/task payload fields by closed schemas. + +### Task 3: Immutable result contract + +**Files:** +- Create: `profiles/cwl_cefr_language_assessment/v1/schemas/cefr-result-snapshot.schema.json` + +- [x] Require domain probabilities, uncertainty and descriptor coverage for measured domains. +- [x] Require explicit non-measurement states instead of invented scores. +- [x] Require standard-setting and empirical validation for `cefr_linked`. +- [x] Require linked evidence plus exact certification authority/policy for `certification_decision`. +- [x] Document the mandatory cross-artifact blueprint authority check. + +### Task 4: Positive and negative fixtures + +**Files:** +- Create: `profiles/cwl_cefr_language_assessment/v1/conformance/valid/*.json` +- Create: `profiles/cwl_cefr_language_assessment/v1/conformance/invalid/*.json` + +- [x] Add separate profile-only and overall-authorized English A1–B2 blueprints. +- [x] Add a reference-only reading-task specification. +- [x] Add profile-only and linked-overall result examples. +- [x] Add failures for missing standard setting, copied descriptor text, incomplete overall reporting, profile-only overall reporting and non-unit probability mass. + +### Task 5: Governance and traceability + +**Files:** +- Create: `profiles/cwl_cefr_language_assessment/v1/README.md` +- Create: `docs/adr/0002-cefr-language-assessment-profile.md` +- Create: `docs/doctoring/CEFR_LANGUAGE_ASSESSMENT.md` +- Modify: `README.md` +- Modify: `CHANGELOG.md` +- Modify: `docs/ARCHITECTURE.md` +- Modify: `docs/doctoring/STANDARD_TRACEABILITY.md` + +- [x] Record the 2020 Companion Volume and 2026 revised test-development manual. +- [x] State that the Council of Europe does not validate provider linking claims. +- [x] Separate CEFR alignment, linking and certification-decision evidence gates. +- [x] Replace mutable Manual/RLD labels with exact editions, versions, ISBNs or dated registry snapshots. +- [x] Record downstream repository ownership and next actions. + +### Task 6: Executable fixture gate + +**Files:** +- Create: `requirements-contracts-ci-hashes.txt` +- Create: `scripts/validate_cefr_profile.py` +- Create: `tests/test_validate_cefr_profile.py` +- Modify: `.github/workflows/quality.yml` + +- [x] Reproduce missing-schema and blueprint-authority defects as RED tests. +- [x] Validate each document against its committed Draft 2020-12 schema before semantic checks. +- [x] Validate every schema against the Draft 2020-12 metaschema with offline `$ref` resolution. +- [x] Resolve immutable blueprints for every result. +- [x] Require blueprint-authorized reporting scope, all required domains and exact policy equality for overall results. +- [x] Reject forbidden payload fields, duplicate domains and invalid probability mass. +- [x] Compile validator/tests and run `git diff --check` in CI. +- [ ] Obtain terminal hosted checks on the unchanged exact head. +- [ ] Resolve or answer every valid exact-head review thread. +- [ ] Obtain qualifying independent review after the parent branch is ready. diff --git a/docs/superpowers/specs/2026-08-27-cefr-language-assessment-profile-design.md b/docs/superpowers/specs/2026-08-27-cefr-language-assessment-profile-design.md new file mode 100644 index 0000000..72575f4 --- /dev/null +++ b/docs/superpowers/specs/2026-08-27-cefr-language-assessment-profile-design.md @@ -0,0 +1,78 @@ +# CEFR Language Assessment Profile Design + +- Status: Approved design, implemented on an active stacked PR +- Date: 2026-08-27 +- Parent: `agent/bootstrap-learning-contracts` +- Issue: `#4` + +## Goal + +Create the smallest shared contract that allows CWL products to exchange a rights-safe, domain-level CEFR language-assessment profile while keeping content, assessment execution, numerical scoring, learner management and longitudinal analysis in their owning repositories. + +## Architecture + +```text +Council of Europe / target-language profile authority + │ immutable references + fixed revision/snapshot + ▼ +Learning Interoperability Contracts + │ Draft 2020-12 schemas + semantic fixtures + ├────────► Learning Content Studio task metadata + ├────────► Psychometrics Commons blueprint/result snapshot + ├────────► fast-mlsirm scoring-profile input/output contract + ├────────► Learning Management Platform placement reference + └────────► TEPP longitudinal result references +``` + +The contract repository owns no runtime state and performs no numerical calculation. + +## Components + +### Common definitions + +Typed CEFR levels, communicative modes, activity domains, language tags, exact references, digests and timestamps. + +### Assessment blueprint + +Pins intended purpose, stakes, target language, exact target-language RLD/profile authority and immutable revision/snapshot, supported levels, required domains, instrument release, scoring profile, cut-score revision, overall-reporting authority, standard-setting evidence and validation evidence. + +### Task specification + +Pins descriptor identities, target levels, mode/domain, response mode, task release/digest, rubric, rights, accessibility, cognitive/linguistic demand and evidence model without copying task or descriptor text. + +### Result snapshot + +Carries immutable domain-level statuses, level probabilities, credible level sets, standard errors, descriptor coverage, overall-reporting status, claim status and evidence references. It does not carry raw responses, raw scores, item/person/rater parameter arrays, likelihood traces or other scoring-engine internals. + +A result snapshot does not authorize its own overall result. The validator resolves `assessment_blueprint_reference`, verifies source-version equality for instrument/scoring/cut-score/language fields, requires every blueprint domain to be measured, and requires the exact blueprint `overall_reporting_policy_reference` before accepting an overall result. + +## Error handling + +Contracts fail closed when: + +- Draft 2020-12 rejects a required field, type, enum, closed-property or conditional rule; +- descriptor/task/response payload fields appear; +- a target-language profile uses a mutable revision alias; +- high-stakes or certification blueprints omit standard-setting evidence; +- measured domains omit probability/uncertainty/coverage evidence; +- probability mass is not one; +- duplicate domain identities appear; +- a result references an unknown or incompatible blueprint; +- an overall result is reported without blueprint authorization, exact policy equality or complete required domains; +- `cefr_linked` claims omit standard-setting or empirical linking validation; +- certification decisions omit an exact certification authority or policy. + +## Testing + +The quality workflow installs a minimal hash-locked validator set, checks each committed schema against the Draft 2020-12 metaschema, validates every positive and negative fixture through the correct schema with offline `$ref` resolution, then applies semantic gates for probability mass, unique domains, immutable profile revisions and blueprint authority. Standard-library `unittest` regressions lock the structural-schema and cross-artifact authorization failures. Generated SDK tests remain a later release slice. + +## Scope exclusions + +- official descriptor text or translations; +- CEFR logo or endorsement claim; +- task, rubric, audio or response storage; +- scoring, linking, DIF or uncertainty arithmetic; +- instrument/session/result persistence; +- LMS placement policy; +- LLM provider calls; +- longitudinal growth estimation. diff --git a/profiles/cwl_cefr_language_assessment/v1/README.md b/profiles/cwl_cefr_language_assessment/v1/README.md new file mode 100644 index 0000000..1e67489 --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/README.md @@ -0,0 +1,55 @@ +# CWL CEFR Language Assessment Profile v1 + +This profile defines shared **references and result envelopes**, not a CEFR examination, a scoring engine, or a certification claim. + +## Purpose + +The contract lets Learning Content Studio, Psychometrics Commons, fast-mlsirm, Learning Management Platform, Semantic Data Portal, TEPP, and other CWL components exchange a language-assessment blueprint, task metadata, and an immutable domain-level result without sharing application databases or copying protected assessment and descriptor content. + +## Authority boundaries + +| Artifact | Authority | +|---|---| +| CEFR framework, descriptors, official translations | Council of Europe or licensed source | +| Target-language Reference Level Description/profile source and revision | Its published authority; a dated registry snapshot is allowed only when no edition identifier exists | +| Task, prompt, media, rubric, rights | Learning Content Studio or assessment-content owner | +| Instrument publication, session, response, result snapshot | Psychometrics Commons | +| Psychometric estimation, rater/task/facet calibration, DIF, linking and uncertainty | fast-mlsirm | +| Enrollment, placement action, completion and credential reference | Learning Management Platform | +| Longitudinal and multilevel language-development analysis | TEPP | + +## Claims + +- `experimental`: research-only evidence; no operational CEFR interpretation claim. +- `cefr_aligned`: the blueprint references CEFR constructs and an exact target-language profile source/revision, but no empirical examination-linking claim is made. +- `cefr_linked`: exact standard-setting and empirical linking/classification-validation artifacts are pinned. +- `certification_decision`: the result is already linked and also pins the exact governed certification authority and certification policy. The shared contract does not create that authority. + +The Council of Europe does not verify or certify an examination provider's CEFR link. The Council of Europe logo or European emblem must not be used to imply endorsement. + +## Overall reporting + +A result cannot authorize its own overall level. The executable validator resolves the immutable assessment blueprint and accepts a reported overall result only when: + +- the blueprint declares `overall_and_profile`; +- every blueprint-required domain has status `measured`; +- the result's `reporting_policy_reference` exactly equals the blueprint policy; +- all structural and probability/credible-set checks pass. + +## Rights and data minimization + +Official descriptor prose, translations, authored task content, raw responses, audio, model output and PII are prohibited in this shared profile. Contracts carry immutable opaque references, bounded domain probability/uncertainty summaries, digests, status, limitations and evidence identities only. Item/person/rater parameter arrays, raw scores, likelihood traces and response-level calculations remain outside the contract. + +## Files + +- `schemas/cefr-common.schema.json` +- `schemas/assessment-blueprint.schema.json` +- `schemas/task-specification.schema.json` +- `schemas/cefr-result-snapshot.schema.json` +- `conformance/valid/` +- `conformance/invalid/` +- `scripts/validate_cefr_profile.py` +- `tests/test_validate_cefr_profile.py` +- `requirements-contracts-ci-hashes.txt` + +The executable gate lives in `.github/workflows/quality.yml`. It installs a minimal hash-locked Draft 2020-12 validator set, resolves schema references offline, validates every fixture structurally, then applies semantic and cross-artifact authority checks. diff --git a/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/high-stakes-blueprint-without-standard-setting.json b/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/high-stakes-blueprint-without-standard-setting.json new file mode 100644 index 0000000..e06cedf --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/high-stakes-blueprint-without-standard-setting.json @@ -0,0 +1 @@ +{"contract_version":"cwl_cefr_language_assessment/assessment_blueprint/v1","blueprint_reference":"cefr_en_certification_b1_b2_blueprint_v1","framework_version_reference":"coe_cefr_companion_volume_2020","language_reference_level_description_reference":"coe_rld_registry_english_profile","language_reference_level_description_revision":"english_profile_registry_snapshot_2026_08_27","descriptor_source_policy_code":"reference_only_no_descriptor_copy","target_language":"en","assessment_purpose_code":"certification","decision_stakes_code":"high","reporting_scope_code":"overall_and_profile","supported_level_codes":["a1","a2","b1","b2"],"required_domain_codes":["reading_reception","listening_reception","written_production","spoken_production"],"optional_domain_codes":["spoken_interaction","mediating_text"],"instrument_release_reference":"instrument_release_en_placement_v1","scoring_profile_reference":"scoring_profile_en_placement_v1","cut_score_revision_reference":"cut_score_revision_en_placement_v1","validation_evidence_references":["content_validation_study_en_v1","pilot_recovery_study_en_v1","rater_calibration_study_en_v1"],"published_at":"2026-08-27T10:00:00Z","limitations":["No certification use.","No mediation or interaction claim unless separately measured."],"overall_reporting_policy_reference":"overall_reporting_policy_en_cert_v1"} diff --git a/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/overall-result-with-incomplete-required-domain.json b/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/overall-result-with-incomplete-required-domain.json new file mode 100644 index 0000000..f0afa48 --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/overall-result-with-incomplete-required-domain.json @@ -0,0 +1 @@ +{"contract_version":"cwl_cefr_language_assessment/result_snapshot/v1","result_reference":"cefr_result_invalid_overall","participant_reference":"participant_opaque_0001","assessment_session_reference":"assessment_session_0001","instrument_release_reference":"instrument_release_en_placement_v1","assessment_blueprint_reference":"cefr_en_placement_a1_b2_overall_blueprint_v1","scoring_profile_reference":"scoring_profile_en_placement_v1","cut_score_revision_reference":"cut_score_revision_en_placement_v1","target_language":"en","claim_status_code":"cefr_linked","domain_results":[{"domain_code":"reading_reception","measurement_status_code":"measured","reported_level_code":"b2","level_probabilities":{"b1":0.18,"b2":0.76,"c1":0.06},"credible_level_set":["b1","b2"],"standard_error":0.29,"descriptor_coverage_references":["coverage_reading_0001"]},{"domain_code":"listening_reception","measurement_status_code":"measured","reported_level_code":"b1","level_probabilities":{"a2":0.16,"b1":0.69,"b2":0.15},"credible_level_set":["a2","b1","b2"],"standard_error":0.34,"descriptor_coverage_references":["coverage_listening_0001"]},{"domain_code":"written_production","measurement_status_code":"measured","reported_level_code":"b2","level_probabilities":{"b1":0.22,"b2":0.71,"c1":0.07},"credible_level_set":["b1","b2"],"standard_error":0.31,"descriptor_coverage_references":["coverage_writing_0001"]},{"domain_code":"spoken_production","measurement_status_code":"measured","reported_level_code":"b1","level_probabilities":{"a2":0.12,"b1":0.74,"b2":0.14},"credible_level_set":["b1","b2"],"standard_error":0.36,"descriptor_coverage_references":["coverage_speaking_0002"]}],"overall_result":{"reporting_status_code":"reported","required_domain_completeness_code":"incomplete","reason_code":"required_domain_incomplete","reported_level_code":"b1","level_probabilities":{"a2":0.08,"b1":0.72,"b2":0.2},"credible_level_set":["b1","b2"],"reporting_policy_reference":"overall_reporting_policy_en_v1"},"human_review_status_code":"completed","result_evidence_references":["response_snapshot_0001","scoring_run_0001","rater_observation_bundle_0001"],"measurement_limitation_codes":["required_domain_incomplete"],"observed_at":"2026-08-27T09:45:00Z","published_at":"2026-08-27T10:00:00Z","result_digest":"sha256:3333333333333333333333333333333333333333333333333333333333333333","standard_setting_study_reference":"standard_setting_study_en_v1","linking_validation_reference":"empirical_cefr_linking_validation_en_v1"} diff --git a/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/overall-result-with-profile-only-blueprint.json b/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/overall-result-with-profile-only-blueprint.json new file mode 100644 index 0000000..cb263cd --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/overall-result-with-profile-only-blueprint.json @@ -0,0 +1 @@ +{"contract_version":"cwl_cefr_language_assessment/result_snapshot/v1","result_reference":"cefr_result_invalid_profile_scope","participant_reference":"participant_opaque_0001","assessment_session_reference":"assessment_session_0001","instrument_release_reference":"instrument_release_en_placement_v1","assessment_blueprint_reference":"cefr_en_placement_a1_b2_blueprint_v1","scoring_profile_reference":"scoring_profile_en_placement_v1","cut_score_revision_reference":"cut_score_revision_en_placement_v1","target_language":"en","claim_status_code":"cefr_linked","domain_results":[{"domain_code":"reading_reception","measurement_status_code":"measured","reported_level_code":"b2","level_probabilities":{"b1":0.18,"b2":0.76,"c1":0.06},"credible_level_set":["b1","b2"],"standard_error":0.29,"descriptor_coverage_references":["coverage_reading_0001"]},{"domain_code":"listening_reception","measurement_status_code":"measured","reported_level_code":"b1","level_probabilities":{"a2":0.16,"b1":0.69,"b2":0.15},"credible_level_set":["a2","b1","b2"],"standard_error":0.34,"descriptor_coverage_references":["coverage_listening_0001"]},{"domain_code":"written_production","measurement_status_code":"measured","reported_level_code":"b2","level_probabilities":{"b1":0.22,"b2":0.71,"c1":0.07},"credible_level_set":["b1","b2"],"standard_error":0.31,"descriptor_coverage_references":["coverage_writing_0001"]},{"domain_code":"spoken_production","measurement_status_code":"measured","reported_level_code":"b1","level_probabilities":{"a2":0.12,"b1":0.74,"b2":0.14},"credible_level_set":["b1","b2"],"standard_error":0.36,"descriptor_coverage_references":["coverage_speaking_0002"]}],"overall_result":{"reporting_status_code":"reported","required_domain_completeness_code":"complete","reason_code":"governed_overall_policy_satisfied","reported_level_code":"b1","level_probabilities":{"a2":0.08,"b1":0.72,"b2":0.2},"credible_level_set":["b1","b2"],"reporting_policy_reference":"overall_reporting_policy_en_v1"},"human_review_status_code":"completed","result_evidence_references":["response_snapshot_0001","scoring_run_0001","rater_observation_bundle_0001"],"observed_at":"2026-08-27T09:45:00Z","published_at":"2026-08-27T10:00:00Z","result_digest":"sha256:5555555555555555555555555555555555555555555555555555555555555555","standard_setting_study_reference":"standard_setting_study_en_v1","linking_validation_reference":"empirical_cefr_linking_validation_en_v1"} diff --git a/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/result-with-nonunit-probability-mass.json b/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/result-with-nonunit-probability-mass.json new file mode 100644 index 0000000..a709cdc --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/result-with-nonunit-probability-mass.json @@ -0,0 +1 @@ +{"contract_version":"cwl_cefr_language_assessment/result_snapshot/v1","result_reference":"cefr_result_invalid_probability","participant_reference":"participant_opaque_0001","assessment_session_reference":"assessment_session_0001","instrument_release_reference":"instrument_release_en_placement_v1","assessment_blueprint_reference":"cefr_en_placement_a1_b2_blueprint_v1","scoring_profile_reference":"scoring_profile_en_placement_v1","cut_score_revision_reference":"cut_score_revision_en_placement_v1","target_language":"en","claim_status_code":"cefr_aligned","domain_results":[{"domain_code":"reading_reception","measurement_status_code":"measured","reported_level_code":"b2","level_probabilities":{"b1":0.2,"b2":0.7,"c1":0.2},"credible_level_set":["b1","b2"],"standard_error":0.29,"descriptor_coverage_references":["coverage_reading_0001"]},{"domain_code":"listening_reception","measurement_status_code":"measured","reported_level_code":"b1","level_probabilities":{"a2":0.16,"b1":0.69,"b2":0.15},"credible_level_set":["a2","b1","b2"],"standard_error":0.34,"descriptor_coverage_references":["coverage_listening_0001"]},{"domain_code":"written_production","measurement_status_code":"measured","reported_level_code":"b2","level_probabilities":{"b1":0.22,"b2":0.71,"c1":0.07},"credible_level_set":["b1","b2"],"standard_error":0.31,"descriptor_coverage_references":["coverage_writing_0001"]},{"domain_code":"spoken_production","measurement_status_code":"insufficient_evidence","limitation_codes":["audio_evidence_below_minimum_duration"]}],"overall_result":{"reporting_status_code":"not_reported","required_domain_completeness_code":"incomplete","reason_code":"required_domain_insufficient_evidence"},"human_review_status_code":"required","result_evidence_references":["response_snapshot_0001","scoring_run_0001","rater_observation_bundle_0001"],"measurement_limitation_codes":["spoken_production_not_scoreable"],"observed_at":"2026-08-27T09:45:00Z","published_at":"2026-08-27T10:00:00Z","result_digest":"sha256:4444444444444444444444444444444444444444444444444444444444444444"} diff --git a/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/task-with-copied-descriptor-text.json b/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/task-with-copied-descriptor-text.json new file mode 100644 index 0000000..989fd32 --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/conformance/invalid/task-with-copied-descriptor-text.json @@ -0,0 +1 @@ +{"contract_version":"cwl_cefr_language_assessment/task_specification/v1","task_reference":"reading_task_bad_descriptor_copy","task_revision_reference":"reading_task_001_revision_01","assessment_blueprint_reference":"cefr_en_placement_a1_b2_blueprint_v1","descriptor_source_policy_code":"reference_only_no_descriptor_copy","target_language":"en","communication_mode_code":"reception","activity_domain_code":"reading_reception","use_domain_code":"professional","response_mode_code":"selected_response","descriptor_references":["coe_descriptor_reference_opaque_001"],"target_level_codes":["b1","b2"],"source_content_release_reference":"content_release_reading_001","accessibility_profile_reference":"accessibility_profile_reading_v1","rights_reference":"rights_record_reading_001","linguistic_demand_reference":"linguistic_demand_reading_001","cognitive_demand_reference":"cognitive_demand_reading_001","evidence_model_reference":"evidence_model_reading_selected_response_v1","task_content_digest":"sha256:1111111111111111111111111111111111111111111111111111111111111111","descriptor_text":"Copied official descriptor prose is forbidden."} diff --git a/profiles/cwl_cefr_language_assessment/v1/conformance/valid/assessment-blueprint-overall.json b/profiles/cwl_cefr_language_assessment/v1/conformance/valid/assessment-blueprint-overall.json new file mode 100644 index 0000000..0628f03 --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/conformance/valid/assessment-blueprint-overall.json @@ -0,0 +1,35 @@ +{ + "contract_version": "cwl_cefr_language_assessment/assessment_blueprint/v1", + "blueprint_reference": "cefr_en_placement_a1_b2_overall_blueprint_v1", + "framework_version_reference": "coe_cefr_companion_volume_2020", + "language_reference_level_description_reference": "coe_rld_registry_english_profile", + "language_reference_level_description_revision": "english_profile_registry_snapshot_2026_08_27", + "descriptor_source_policy_code": "reference_only_no_descriptor_copy", + "target_language": "en", + "assessment_purpose_code": "placement", + "decision_stakes_code": "low", + "reporting_scope_code": "overall_and_profile", + "supported_level_codes": ["a1", "a2", "b1", "b2"], + "required_domain_codes": [ + "reading_reception", + "listening_reception", + "written_production", + "spoken_production" + ], + "optional_domain_codes": ["spoken_interaction", "mediating_text"], + "instrument_release_reference": "instrument_release_en_placement_v1", + "scoring_profile_reference": "scoring_profile_en_placement_v1", + "cut_score_revision_reference": "cut_score_revision_en_placement_v1", + "overall_reporting_policy_reference": "overall_reporting_policy_en_v1", + "validation_evidence_references": [ + "content_validation_study_en_v1", + "pilot_recovery_study_en_v1", + "rater_calibration_study_en_v1" + ], + "published_at": "2026-08-27T10:00:00Z", + "limitations": [ + "No certification use.", + "English Profile is pinned as a dated registry snapshot because the public registry exposes no single immutable edition identifier.", + "No mediation or interaction claim unless separately measured." + ] +} diff --git a/profiles/cwl_cefr_language_assessment/v1/conformance/valid/assessment-blueprint.json b/profiles/cwl_cefr_language_assessment/v1/conformance/valid/assessment-blueprint.json new file mode 100644 index 0000000..de3557b --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/conformance/valid/assessment-blueprint.json @@ -0,0 +1,34 @@ +{ + "contract_version": "cwl_cefr_language_assessment/assessment_blueprint/v1", + "blueprint_reference": "cefr_en_placement_a1_b2_blueprint_v1", + "framework_version_reference": "coe_cefr_companion_volume_2020", + "language_reference_level_description_reference": "coe_rld_registry_english_profile", + "language_reference_level_description_revision": "english_profile_registry_snapshot_2026_08_27", + "descriptor_source_policy_code": "reference_only_no_descriptor_copy", + "target_language": "en", + "assessment_purpose_code": "placement", + "decision_stakes_code": "low", + "reporting_scope_code": "profile_only", + "supported_level_codes": ["a1", "a2", "b1", "b2"], + "required_domain_codes": [ + "reading_reception", + "listening_reception", + "written_production", + "spoken_production" + ], + "optional_domain_codes": ["spoken_interaction", "mediating_text"], + "instrument_release_reference": "instrument_release_en_placement_v1", + "scoring_profile_reference": "scoring_profile_en_placement_v1", + "cut_score_revision_reference": "cut_score_revision_en_placement_v1", + "validation_evidence_references": [ + "content_validation_study_en_v1", + "pilot_recovery_study_en_v1", + "rater_calibration_study_en_v1" + ], + "published_at": "2026-08-27T10:00:00Z", + "limitations": [ + "No certification use.", + "English Profile is pinned as a dated registry snapshot because the public registry exposes no single immutable edition identifier.", + "No mediation or interaction claim unless separately measured." + ] +} diff --git a/profiles/cwl_cefr_language_assessment/v1/conformance/valid/cefr-result-snapshot-linked-overall.json b/profiles/cwl_cefr_language_assessment/v1/conformance/valid/cefr-result-snapshot-linked-overall.json new file mode 100644 index 0000000..102ed13 --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/conformance/valid/cefr-result-snapshot-linked-overall.json @@ -0,0 +1 @@ +{"contract_version":"cwl_cefr_language_assessment/result_snapshot/v1","result_reference":"cefr_result_0002","participant_reference":"participant_opaque_0001","assessment_session_reference":"assessment_session_0001","instrument_release_reference":"instrument_release_en_placement_v1","assessment_blueprint_reference":"cefr_en_placement_a1_b2_overall_blueprint_v1","scoring_profile_reference":"scoring_profile_en_placement_v1","cut_score_revision_reference":"cut_score_revision_en_placement_v1","target_language":"en","claim_status_code":"cefr_linked","domain_results":[{"domain_code":"reading_reception","measurement_status_code":"measured","reported_level_code":"b2","level_probabilities":{"b1":0.18,"b2":0.76,"c1":0.06},"credible_level_set":["b1","b2"],"standard_error":0.29,"descriptor_coverage_references":["coverage_reading_0001"]},{"domain_code":"listening_reception","measurement_status_code":"measured","reported_level_code":"b1","level_probabilities":{"a2":0.16,"b1":0.69,"b2":0.15},"credible_level_set":["a2","b1","b2"],"standard_error":0.34,"descriptor_coverage_references":["coverage_listening_0001"]},{"domain_code":"written_production","measurement_status_code":"measured","reported_level_code":"b2","level_probabilities":{"b1":0.22,"b2":0.71,"c1":0.07},"credible_level_set":["b1","b2"],"standard_error":0.31,"descriptor_coverage_references":["coverage_writing_0001"]},{"domain_code":"spoken_production","measurement_status_code":"measured","reported_level_code":"b1","level_probabilities":{"a2":0.12,"b1":0.74,"b2":0.14},"credible_level_set":["b1","b2"],"standard_error":0.36,"descriptor_coverage_references":["coverage_speaking_0002"]}],"overall_result":{"reporting_status_code":"reported","required_domain_completeness_code":"complete","reason_code":"governed_overall_policy_satisfied","reported_level_code":"b1","level_probabilities":{"a2":0.08,"b1":0.72,"b2":0.2},"credible_level_set":["b1","b2"],"reporting_policy_reference":"overall_reporting_policy_en_v1"},"human_review_status_code":"completed","result_evidence_references":["response_snapshot_0001","scoring_run_0001","rater_observation_bundle_0001"],"observed_at":"2026-08-27T09:45:00Z","published_at":"2026-08-27T10:00:00Z","result_digest":"sha256:3333333333333333333333333333333333333333333333333333333333333333","standard_setting_study_reference":"standard_setting_study_en_v1","linking_validation_reference":"empirical_cefr_linking_validation_en_v1"} diff --git a/profiles/cwl_cefr_language_assessment/v1/conformance/valid/cefr-result-snapshot-profile-only.json b/profiles/cwl_cefr_language_assessment/v1/conformance/valid/cefr-result-snapshot-profile-only.json new file mode 100644 index 0000000..6b3266e --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/conformance/valid/cefr-result-snapshot-profile-only.json @@ -0,0 +1 @@ +{"contract_version":"cwl_cefr_language_assessment/result_snapshot/v1","result_reference":"cefr_result_0001","participant_reference":"participant_opaque_0001","assessment_session_reference":"assessment_session_0001","instrument_release_reference":"instrument_release_en_placement_v1","assessment_blueprint_reference":"cefr_en_placement_a1_b2_blueprint_v1","scoring_profile_reference":"scoring_profile_en_placement_v1","cut_score_revision_reference":"cut_score_revision_en_placement_v1","target_language":"en","claim_status_code":"cefr_aligned","domain_results":[{"domain_code":"reading_reception","measurement_status_code":"measured","reported_level_code":"b2","level_probabilities":{"b1":0.18,"b2":0.76,"c1":0.06},"credible_level_set":["b1","b2"],"standard_error":0.29,"descriptor_coverage_references":["coverage_reading_0001"]},{"domain_code":"listening_reception","measurement_status_code":"measured","reported_level_code":"b1","level_probabilities":{"a2":0.16,"b1":0.69,"b2":0.15},"credible_level_set":["a2","b1","b2"],"standard_error":0.34,"descriptor_coverage_references":["coverage_listening_0001"]},{"domain_code":"written_production","measurement_status_code":"measured","reported_level_code":"b2","level_probabilities":{"b1":0.22,"b2":0.71,"c1":0.07},"credible_level_set":["b1","b2"],"standard_error":0.31,"descriptor_coverage_references":["coverage_writing_0001"]},{"domain_code":"spoken_production","measurement_status_code":"insufficient_evidence","limitation_codes":["audio_evidence_below_minimum_duration"]}],"overall_result":{"reporting_status_code":"not_reported","required_domain_completeness_code":"incomplete","reason_code":"required_domain_insufficient_evidence"},"human_review_status_code":"required","result_evidence_references":["response_snapshot_0001","scoring_run_0001","rater_observation_bundle_0001"],"measurement_limitation_codes":["spoken_production_not_scoreable"],"observed_at":"2026-08-27T09:45:00Z","published_at":"2026-08-27T10:00:00Z","result_digest":"sha256:2222222222222222222222222222222222222222222222222222222222222222"} diff --git a/profiles/cwl_cefr_language_assessment/v1/conformance/valid/task-specification.json b/profiles/cwl_cefr_language_assessment/v1/conformance/valid/task-specification.json new file mode 100644 index 0000000..771c1d0 --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/conformance/valid/task-specification.json @@ -0,0 +1,21 @@ +{ + "contract_version": "cwl_cefr_language_assessment/task_specification/v1", + "task_reference": "reading_task_001", + "task_revision_reference": "reading_task_001_revision_01", + "assessment_blueprint_reference": "cefr_en_placement_a1_b2_blueprint_v1", + "descriptor_source_policy_code": "reference_only_no_descriptor_copy", + "target_language": "en", + "communication_mode_code": "reception", + "activity_domain_code": "reading_reception", + "use_domain_code": "professional", + "response_mode_code": "selected_response", + "descriptor_references": ["coe_descriptor_reference_opaque_001"], + "target_level_codes": ["b1", "b2"], + "source_content_release_reference": "content_release_reading_001", + "accessibility_profile_reference": "accessibility_profile_reading_v1", + "rights_reference": "rights_record_reading_001", + "linguistic_demand_reference": "linguistic_demand_reading_001", + "cognitive_demand_reference": "cognitive_demand_reading_001", + "evidence_model_reference": "evidence_model_reading_selected_response_v1", + "task_content_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111" +} diff --git a/profiles/cwl_cefr_language_assessment/v1/schemas/assessment-blueprint.schema.json b/profiles/cwl_cefr_language_assessment/v1/schemas/assessment-blueprint.schema.json new file mode 100644 index 0000000..5bd0611 --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/schemas/assessment-blueprint.schema.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/ContextualWisdomLab/learning-interoperability-contracts/develop/profiles/cwl_cefr_language_assessment/v1/schemas/assessment-blueprint.schema.json", + "x-cwl-schema-version": "1.0.0", + "title": "CWL CEFR Language Assessment Blueprint v1", + "description": "A rights-safe, version-pinned assessment blueprint. It references CEFR, target-language profile/RLD, and scientific artifacts and never embeds official descriptor prose.", + "type": "object", + "additionalProperties": false, + "required": [ + "contract_version", + "blueprint_reference", + "framework_version_reference", + "language_reference_level_description_reference", + "language_reference_level_description_revision", + "descriptor_source_policy_code", + "target_language", + "assessment_purpose_code", + "decision_stakes_code", + "reporting_scope_code", + "supported_level_codes", + "required_domain_codes", + "instrument_release_reference", + "scoring_profile_reference", + "cut_score_revision_reference", + "validation_evidence_references", + "published_at" + ], + "properties": { + "contract_version": {"const": "cwl_cefr_language_assessment/assessment_blueprint/v1"}, + "blueprint_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "framework_version_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "language_reference_level_description_reference": { + "description": "Exact authority/source identity for the target-language RLD or language-profile source.", + "$ref": "cefr-common.schema.json#/$defs/opaque_reference" + }, + "language_reference_level_description_revision": { + "description": "Immutable publisher revision, edition, digest-bound snapshot, or dated registry snapshot. A mutable latest alias is prohibited.", + "$ref": "cefr-common.schema.json#/$defs/opaque_reference" + }, + "descriptor_source_policy_code": {"const": "reference_only_no_descriptor_copy"}, + "target_language": {"$ref": "cefr-common.schema.json#/$defs/language_tag"}, + "assessment_purpose_code": { + "type": "string", + "enum": ["placement", "diagnostic", "progress_monitoring", "certification", "research"] + }, + "decision_stakes_code": {"type": "string", "enum": ["low", "moderate", "high"]}, + "reporting_scope_code": {"type": "string", "enum": ["profile_only", "overall_and_profile"]}, + "supported_level_codes": { + "type": "array", + "minItems": 2, + "maxItems": 10, + "uniqueItems": true, + "items": {"$ref": "cefr-common.schema.json#/$defs/cefr_level_code"} + }, + "required_domain_codes": { + "type": "array", + "minItems": 1, + "maxItems": 12, + "uniqueItems": true, + "items": {"$ref": "cefr-common.schema.json#/$defs/activity_domain_code"} + }, + "optional_domain_codes": { + "type": "array", + "maxItems": 12, + "uniqueItems": true, + "items": {"$ref": "cefr-common.schema.json#/$defs/activity_domain_code"} + }, + "instrument_release_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "scoring_profile_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "cut_score_revision_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "overall_reporting_policy_reference": { + "description": "Immutable policy that alone authorizes and defines any overall result for this blueprint.", + "$ref": "cefr-common.schema.json#/$defs/opaque_reference" + }, + "standard_setting_study_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "validation_evidence_references": { + "type": "array", + "minItems": 1, + "maxItems": 256, + "uniqueItems": true, + "items": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"} + }, + "published_at": {"$ref": "cefr-common.schema.json#/$defs/date_time"}, + "limitations": { + "type": "array", + "maxItems": 64, + "uniqueItems": true, + "items": {"type": "string", "minLength": 3, "maxLength": 256} + } + }, + "allOf": [ + { + "if": { + "properties": {"reporting_scope_code": {"const": "overall_and_profile"}}, + "required": ["reporting_scope_code"] + }, + "then": {"required": ["overall_reporting_policy_reference"]} + }, + { + "if": { + "anyOf": [ + { + "properties": {"decision_stakes_code": {"const": "high"}}, + "required": ["decision_stakes_code"] + }, + { + "properties": {"assessment_purpose_code": {"const": "certification"}}, + "required": ["assessment_purpose_code"] + } + ] + }, + "then": {"required": ["standard_setting_study_reference"]} + } + ] +} diff --git a/profiles/cwl_cefr_language_assessment/v1/schemas/cefr-common.schema.json b/profiles/cwl_cefr_language_assessment/v1/schemas/cefr-common.schema.json new file mode 100644 index 0000000..78bf557 --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/schemas/cefr-common.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/ContextualWisdomLab/learning-interoperability-contracts/develop/profiles/cwl_cefr_language_assessment/v1/schemas/cefr-common.schema.json", + "x-cwl-schema-version": "1.0.0", + "title": "CWL CEFR Language Assessment Common Definitions", + "description": "Reference-only shared definitions. This schema never carries official CEFR descriptor prose, assessment content, responses, or scoring payloads.", + "$defs": { + "opaque_reference": { + "type": "string", + "minLength": 3, + "maxLength": 512, + "pattern": "^(?![0-9]+$)(?!.*[\\u0000-\\u001F\\u007F])\\S(?:[^\\u0000-\\u001F\\u007F]*\\S)?$" + }, + "language_tag": { + "type": "string", + "minLength": 2, + "maxLength": 35, + "format": "language-tag", + "pattern": "^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$" + }, + "cefr_level_code": { + "type": "string", + "enum": ["pre_a1", "a1", "a2", "a2_plus", "b1", "b1_plus", "b2", "b2_plus", "c1", "c2"] + }, + "communication_mode_code": { + "type": "string", + "enum": ["reception", "production", "interaction", "mediation"] + }, + "activity_domain_code": { + "type": "string", + "enum": [ + "listening_reception", + "reading_reception", + "spoken_production", + "written_production", + "spoken_interaction", + "written_interaction", + "online_interaction", + "mediating_text", + "mediating_concepts", + "mediating_communication", + "plurilingual_pluricultural_competence", + "phonological_competence" + ] + }, + "use_domain_code": { + "type": "string", + "enum": ["personal", "public", "educational", "professional"] + }, + "date_time": { + "type": "string", + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\\.[0-9]+)?(?:Z|[+-][0-9]{2}:[0-9]{2})$" + }, + "sha256_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "reference_array": { + "type": "array", + "minItems": 1, + "maxItems": 256, + "uniqueItems": true, + "items": {"$ref": "#/$defs/opaque_reference"} + } + } +} diff --git a/profiles/cwl_cefr_language_assessment/v1/schemas/cefr-result-snapshot.schema.json b/profiles/cwl_cefr_language_assessment/v1/schemas/cefr-result-snapshot.schema.json new file mode 100644 index 0000000..c18a63c --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/schemas/cefr-result-snapshot.schema.json @@ -0,0 +1,239 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/ContextualWisdomLab/learning-interoperability-contracts/develop/profiles/cwl_cefr_language_assessment/v1/schemas/cefr-result-snapshot.schema.json", + "x-cwl-schema-version": "1.0.0", + "title": "CWL CEFR Result Snapshot v1", + "description": "An immutable domain-profile result reference. Numerical scoring remains in fast-mlsirm and result authority remains in Psychometrics Commons.", + "$comment": "Draft 2020-12 validates document structure. The executable profile validator must also resolve the immutable assessment blueprint and enforce its reporting scope, required domains, and exact overall-reporting policy.", + "type": "object", + "additionalProperties": false, + "required": [ + "contract_version", + "result_reference", + "participant_reference", + "assessment_session_reference", + "instrument_release_reference", + "assessment_blueprint_reference", + "scoring_profile_reference", + "cut_score_revision_reference", + "target_language", + "claim_status_code", + "domain_results", + "overall_result", + "human_review_status_code", + "result_evidence_references", + "observed_at", + "published_at", + "result_digest" + ], + "properties": { + "contract_version": {"const": "cwl_cefr_language_assessment/result_snapshot/v1"}, + "result_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "participant_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "assessment_session_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "instrument_release_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "assessment_blueprint_reference": { + "description": "Exact immutable blueprint authority. A reported overall result is valid only when this blueprint authorizes overall reporting and every required domain is measured.", + "$ref": "cefr-common.schema.json#/$defs/opaque_reference" + }, + "scoring_profile_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "cut_score_revision_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "target_language": {"$ref": "cefr-common.schema.json#/$defs/language_tag"}, + "claim_status_code": { + "type": "string", + "enum": ["experimental", "cefr_aligned", "cefr_linked", "certification_decision"] + }, + "standard_setting_study_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "linking_validation_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "certification_authority_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "certification_policy_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "domain_results": { + "type": "array", + "minItems": 1, + "maxItems": 12, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["domain_code", "measurement_status_code"], + "properties": { + "domain_code": {"$ref": "cefr-common.schema.json#/$defs/activity_domain_code"}, + "measurement_status_code": { + "type": "string", + "enum": ["measured", "not_measured", "insufficient_evidence", "invalidated"] + }, + "reported_level_code": {"$ref": "cefr-common.schema.json#/$defs/cefr_level_code"}, + "level_probabilities": { + "type": "object", + "additionalProperties": false, + "minProperties": 2, + "properties": { + "pre_a1": {"type": "number", "minimum": 0, "maximum": 1}, + "a1": {"type": "number", "minimum": 0, "maximum": 1}, + "a2": {"type": "number", "minimum": 0, "maximum": 1}, + "a2_plus": {"type": "number", "minimum": 0, "maximum": 1}, + "b1": {"type": "number", "minimum": 0, "maximum": 1}, + "b1_plus": {"type": "number", "minimum": 0, "maximum": 1}, + "b2": {"type": "number", "minimum": 0, "maximum": 1}, + "b2_plus": {"type": "number", "minimum": 0, "maximum": 1}, + "c1": {"type": "number", "minimum": 0, "maximum": 1}, + "c2": {"type": "number", "minimum": 0, "maximum": 1} + } + }, + "credible_level_set": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "uniqueItems": true, + "items": {"$ref": "cefr-common.schema.json#/$defs/cefr_level_code"} + }, + "standard_error": {"type": "number", "minimum": 0}, + "descriptor_coverage_references": { + "type": "array", + "minItems": 1, + "maxItems": 256, + "uniqueItems": true, + "items": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"} + }, + "limitation_codes": { + "type": "array", + "maxItems": 64, + "uniqueItems": true, + "items": {"type": "string", "minLength": 3, "maxLength": 128} + } + }, + "allOf": [ + { + "if": { + "properties": {"measurement_status_code": {"const": "measured"}}, + "required": ["measurement_status_code"] + }, + "then": { + "required": [ + "reported_level_code", + "level_probabilities", + "credible_level_set", + "standard_error", + "descriptor_coverage_references" + ] + }, + "else": { + "not": { + "anyOf": [ + {"required": ["reported_level_code"]}, + {"required": ["level_probabilities"]}, + {"required": ["credible_level_set"]}, + {"required": ["standard_error"]}, + {"required": ["descriptor_coverage_references"]} + ] + } + } + } + ] + } + }, + "overall_result": { + "description": "Structural result envelope. The executable validator must resolve assessment_blueprint_reference before accepting any reported overall result.", + "type": "object", + "additionalProperties": false, + "required": ["reporting_status_code", "required_domain_completeness_code", "reason_code"], + "properties": { + "reporting_status_code": {"type": "string", "enum": ["reported", "not_reported"]}, + "required_domain_completeness_code": {"type": "string", "enum": ["complete", "incomplete", "unknown"]}, + "reason_code": {"type": "string", "minLength": 3, "maxLength": 128}, + "reported_level_code": {"$ref": "cefr-common.schema.json#/$defs/cefr_level_code"}, + "level_probabilities": { + "type": "object", + "additionalProperties": false, + "minProperties": 2, + "properties": { + "pre_a1": {"type": "number", "minimum": 0, "maximum": 1}, + "a1": {"type": "number", "minimum": 0, "maximum": 1}, + "a2": {"type": "number", "minimum": 0, "maximum": 1}, + "a2_plus": {"type": "number", "minimum": 0, "maximum": 1}, + "b1": {"type": "number", "minimum": 0, "maximum": 1}, + "b1_plus": {"type": "number", "minimum": 0, "maximum": 1}, + "b2": {"type": "number", "minimum": 0, "maximum": 1}, + "b2_plus": {"type": "number", "minimum": 0, "maximum": 1}, + "c1": {"type": "number", "minimum": 0, "maximum": 1}, + "c2": {"type": "number", "minimum": 0, "maximum": 1} + } + }, + "credible_level_set": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "uniqueItems": true, + "items": {"$ref": "cefr-common.schema.json#/$defs/cefr_level_code"} + }, + "reporting_policy_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"} + }, + "allOf": [ + { + "if": { + "properties": {"reporting_status_code": {"const": "reported"}}, + "required": ["reporting_status_code"] + }, + "then": { + "properties": {"required_domain_completeness_code": {"const": "complete"}}, + "required": [ + "reported_level_code", + "level_probabilities", + "credible_level_set", + "reporting_policy_reference" + ] + }, + "else": { + "not": { + "anyOf": [ + {"required": ["reported_level_code"]}, + {"required": ["level_probabilities"]}, + {"required": ["credible_level_set"]}, + {"required": ["reporting_policy_reference"]} + ] + } + } + } + ] + }, + "human_review_status_code": {"type": "string", "enum": ["not_required", "required", "completed"]}, + "result_evidence_references": { + "type": "array", + "minItems": 1, + "maxItems": 256, + "uniqueItems": true, + "items": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"} + }, + "measurement_limitation_codes": { + "type": "array", + "maxItems": 64, + "uniqueItems": true, + "items": {"type": "string", "minLength": 3, "maxLength": 128} + }, + "observed_at": {"$ref": "cefr-common.schema.json#/$defs/date_time"}, + "published_at": {"$ref": "cefr-common.schema.json#/$defs/date_time"}, + "result_digest": {"$ref": "cefr-common.schema.json#/$defs/sha256_digest"} + }, + "allOf": [ + { + "if": { + "properties": {"claim_status_code": {"enum": ["cefr_linked", "certification_decision"]}}, + "required": ["claim_status_code"] + }, + "then": {"required": ["standard_setting_study_reference", "linking_validation_reference"]} + }, + { + "if": { + "properties": {"claim_status_code": {"const": "certification_decision"}}, + "required": ["claim_status_code"] + }, + "then": { + "required": ["certification_authority_reference", "certification_policy_reference"], + "properties": { + "overall_result": { + "properties": {"reporting_status_code": {"const": "reported"}} + } + } + } + } + ] +} diff --git a/profiles/cwl_cefr_language_assessment/v1/schemas/task-specification.schema.json b/profiles/cwl_cefr_language_assessment/v1/schemas/task-specification.schema.json new file mode 100644 index 0000000..203cb3e --- /dev/null +++ b/profiles/cwl_cefr_language_assessment/v1/schemas/task-specification.schema.json @@ -0,0 +1,111 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/ContextualWisdomLab/learning-interoperability-contracts/develop/profiles/cwl_cefr_language_assessment/v1/schemas/task-specification.schema.json", + "x-cwl-schema-version": "1.0.0", + "title": "CWL CEFR Task Specification v1", + "description": "Metadata-only task contract. Authored task content, official descriptor prose, rubrics, media, and responses stay with their owning systems.", + "type": "object", + "additionalProperties": false, + "required": [ + "contract_version", + "task_reference", + "task_revision_reference", + "assessment_blueprint_reference", + "descriptor_source_policy_code", + "target_language", + "communication_mode_code", + "activity_domain_code", + "use_domain_code", + "response_mode_code", + "descriptor_references", + "target_level_codes", + "source_content_release_reference", + "accessibility_profile_reference", + "rights_reference", + "linguistic_demand_reference", + "cognitive_demand_reference", + "evidence_model_reference", + "task_content_digest" + ], + "properties": { + "contract_version": {"const": "cwl_cefr_language_assessment/task_specification/v1"}, + "task_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "task_revision_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "assessment_blueprint_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "descriptor_source_policy_code": {"const": "reference_only_no_descriptor_copy"}, + "target_language": {"$ref": "cefr-common.schema.json#/$defs/language_tag"}, + "communication_mode_code": {"$ref": "cefr-common.schema.json#/$defs/communication_mode_code"}, + "activity_domain_code": {"$ref": "cefr-common.schema.json#/$defs/activity_domain_code"}, + "use_domain_code": {"$ref": "cefr-common.schema.json#/$defs/use_domain_code"}, + "response_mode_code": { + "type": "string", + "enum": [ + "selected_response", + "constructed_written", + "constructed_spoken", + "interactive_spoken", + "interactive_written", + "multimodal", + "mediated" + ] + }, + "descriptor_references": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "uniqueItems": true, + "items": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"} + }, + "target_level_codes": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "uniqueItems": true, + "items": {"$ref": "cefr-common.schema.json#/$defs/cefr_level_code"} + }, + "source_content_release_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "rubric_revision_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "accessibility_profile_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "rights_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "linguistic_demand_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "cognitive_demand_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "interaction_demand_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "mediation_demand_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "evidence_model_reference": {"$ref": "cefr-common.schema.json#/$defs/opaque_reference"}, + "task_content_digest": {"$ref": "cefr-common.schema.json#/$defs/sha256_digest"} + }, + "allOf": [ + { + "if": { + "properties": { + "response_mode_code": { + "enum": [ + "constructed_written", + "constructed_spoken", + "interactive_spoken", + "interactive_written", + "multimodal", + "mediated" + ] + } + }, + "required": ["response_mode_code"] + }, + "then": {"required": ["rubric_revision_reference"]} + }, + { + "if": { + "properties": {"communication_mode_code": {"const": "interaction"}}, + "required": ["communication_mode_code"] + }, + "then": {"required": ["interaction_demand_reference"]} + }, + { + "if": { + "properties": {"communication_mode_code": {"const": "mediation"}}, + "required": ["communication_mode_code"] + }, + "then": {"required": ["mediation_demand_reference"]} + } + ] +} diff --git a/requirements-contracts-ci-hashes.txt b/requirements-contracts-ci-hashes.txt new file mode 100644 index 0000000..13f47a0 --- /dev/null +++ b/requirements-contracts-ci-hashes.txt @@ -0,0 +1,12 @@ +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 +jsonschema==4.26.0 \ + --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce +jsonschema-specifications==2025.9.1 \ + --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe +referencing==0.37.0 \ + --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 +rpds-py==2026.6.3 \ + --hash=sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6 +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 diff --git a/scripts/validate_cefr_profile.py b/scripts/validate_cefr_profile.py new file mode 100644 index 0000000..0bf8672 --- /dev/null +++ b/scripts/validate_cefr_profile.py @@ -0,0 +1,424 @@ +"""Validate CEFR contracts with Draft 2020-12 and cross-artifact semantics.""" + +from __future__ import annotations + +import json +import math +from functools import lru_cache +from pathlib import Path +from typing import Any, Callable + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError +from referencing import Registry, Resource + +PROFILE_ROOT = Path("profiles/cwl_cefr_language_assessment/v1") +SCHEMA_ROOT = PROFILE_ROOT / "schemas" +VALID_ROOT = PROFILE_ROOT / "conformance" / "valid" +INVALID_ROOT = PROFILE_ROOT / "conformance" / "invalid" +PUBLISHED_PREFIX = ( + "https://raw.githubusercontent.com/ContextualWisdomLab/" + "learning-interoperability-contracts/develop/" + "profiles/cwl_cefr_language_assessment/v1/schemas/" +) +FORBIDDEN_KEYS = { + "descriptor_text", + "descriptor_prose", + "can_do_text", + "task_content", + "raw_response", + "response_text", + "audio_bytes", + "provider_payload", + "model_output", + "personal_data", +} +SCHEMA_FILENAMES = { + "blueprint": "assessment-blueprint.schema.json", + "task": "task-specification.schema.json", + "result": "cefr-result-snapshot.schema.json", +} + + +def load_json(path: Path) -> dict[str, Any]: + """Load one JSON object and reject non-object roots.""" + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a JSON object") + return value + + +@lru_cache(maxsize=1) +def load_schemas() -> dict[str, dict[str, Any]]: + """Load every committed CEFR schema by filename exactly once per process.""" + return { + path.name: load_json(path) + for path in sorted(SCHEMA_ROOT.glob("*.json")) + } + + +@lru_cache(maxsize=1) +def build_schema_registry() -> Registry: + """Build an offline registry for absolute and relative CEFR schema references.""" + resources = [] + for schema in load_schemas().values(): + schema_id = schema.get("$id") + if not isinstance(schema_id, str): + raise ValueError("every CEFR schema must declare a string $id") + resources.append((schema_id, Resource.from_contents(schema))) + return Registry().with_resources(resources) + + +def validate_against_schema(value: dict[str, Any], schema_filename: str) -> None: + """Apply the committed Draft 2020-12 schema before semantic validation.""" + try: + schema = load_schemas()[schema_filename] + except KeyError as error: + raise ValueError(f"unknown CEFR schema {schema_filename!r}") from error + validator = Draft202012Validator(schema, registry=build_schema_registry()) + errors = sorted(validator.iter_errors(value), key=lambda error: error.json_path) + if errors: + first = errors[0] + raise ValueError( + f"{schema_filename} JSON Schema violation at {first.json_path}: " + f"{first.message}" + ) + + +def reject_forbidden_payload(value: Any, location: str = "$") -> None: + """Reject protected prose, content, response, provider, and personal-data fields.""" + if isinstance(value, dict): + for key, nested in value.items(): + if key in FORBIDDEN_KEYS: + raise ValueError(f"forbidden payload field {key!r} at {location}") + reject_forbidden_payload(nested, f"{location}.{key}") + elif isinstance(value, list): + for index, nested in enumerate(value): + reject_forbidden_payload(nested, f"{location}[{index}]") + + +def require_reference(value: Any, field: str) -> None: + """Require one exact, nonnumeric, control-free opaque reference.""" + if ( + not isinstance(value, str) + or len(value) < 3 + or value != value.strip() + or value.isnumeric() + ): + raise ValueError(f"{field} must be an exact nonnumeric opaque reference") + if any(ord(character) < 32 or ord(character) == 127 for character in value): + raise ValueError(f"{field} contains an ASCII control") + + +def require_immutable_revision(value: Any, field: str) -> None: + """Reject mutable revision aliases while permitting publisher or snapshot IDs.""" + require_reference(value, field) + if str(value).strip().lower() in {"latest", "current", "head", "main"}: + raise ValueError(f"{field} must not use a mutable revision alias") + + +def validate_probability_set(value: Any, field: str) -> None: + """Require finite probabilities in [0, 1] whose mass equals one.""" + if not isinstance(value, dict) or len(value) < 2: + raise ValueError(f"{field} must contain at least two level probabilities") + numbers = list(value.values()) + if any( + not isinstance(item, (int, float)) + or isinstance(item, bool) + or not math.isfinite(item) + or item < 0 + or item > 1 + for item in numbers + ): + raise ValueError(f"{field} contains an invalid probability") + if not math.isclose(sum(numbers), 1.0, rel_tol=0.0, abs_tol=1e-9): + raise ValueError(f"{field} probability mass must equal one") + + +def validate_blueprint(value: dict[str, Any]) -> None: + """Validate structural, scientific, rights, and claim gates for a blueprint.""" + validate_against_schema(value, SCHEMA_FILENAMES["blueprint"]) + reject_forbidden_payload(value) + for field in ( + "blueprint_reference", + "framework_version_reference", + "language_reference_level_description_reference", + "instrument_release_reference", + "scoring_profile_reference", + "cut_score_revision_reference", + ): + require_reference(value.get(field), field) + require_immutable_revision( + value.get("language_reference_level_description_revision"), + "language_reference_level_description_revision", + ) + if value.get("descriptor_source_policy_code") != "reference_only_no_descriptor_copy": + raise ValueError("descriptor source policy must prohibit copies") + required = value.get("required_domain_codes") + optional = value.get("optional_domain_codes", []) + if not isinstance(required, list) or not required or len(required) != len(set(required)): + raise ValueError("required domains must be nonempty and unique") + if ( + not isinstance(optional, list) + or len(optional) != len(set(optional)) + or set(required) & set(optional) + ): + raise ValueError("required and optional domains must be disjoint") + if value.get("reporting_scope_code") == "overall_and_profile": + require_reference( + value.get("overall_reporting_policy_reference"), + "overall_reporting_policy_reference", + ) + if ( + value.get("decision_stakes_code") == "high" + or value.get("assessment_purpose_code") == "certification" + ): + require_reference( + value.get("standard_setting_study_reference"), + "standard_setting_study_reference", + ) + + +def validate_task(value: dict[str, Any]) -> None: + """Validate a closed, reference-only CEFR task document.""" + validate_against_schema(value, SCHEMA_FILENAMES["task"]) + reject_forbidden_payload(value) + for field in ( + "task_reference", + "task_revision_reference", + "assessment_blueprint_reference", + "source_content_release_reference", + "accessibility_profile_reference", + "rights_reference", + "linguistic_demand_reference", + "cognitive_demand_reference", + "evidence_model_reference", + ): + require_reference(value.get(field), field) + descriptors = value.get("descriptor_references") + if not isinstance(descriptors, list) or not descriptors or len(descriptors) != len(set(descriptors)): + raise ValueError("descriptor references must be nonempty and unique") + for descriptor in descriptors: + require_reference(descriptor, "descriptor_reference") + if value.get("descriptor_source_policy_code") != "reference_only_no_descriptor_copy": + raise ValueError("descriptor source policy must prohibit copies") + if value.get("response_mode_code") != "selected_response": + require_reference(value.get("rubric_revision_reference"), "rubric_revision_reference") + if value.get("communication_mode_code") == "interaction": + require_reference( + value.get("interaction_demand_reference"), + "interaction_demand_reference", + ) + if value.get("communication_mode_code") == "mediation": + require_reference( + value.get("mediation_demand_reference"), + "mediation_demand_reference", + ) + digest = value.get("task_content_digest") + if ( + not isinstance(digest, str) + or len(digest) != 71 + or not digest.startswith("sha256:") + or any(character not in "0123456789abcdef" for character in digest[7:]) + ): + raise ValueError("task digest must be canonical lowercase sha256") + + +def load_blueprint_registry() -> dict[str, dict[str, Any]]: + """Load and validate every positive blueprint by its immutable reference.""" + registry: dict[str, dict[str, Any]] = {} + for path in sorted(VALID_ROOT.glob("assessment-blueprint*.json")): + blueprint = load_json(path) + validate_blueprint(blueprint) + reference = blueprint["blueprint_reference"] + if reference in registry: + raise ValueError(f"duplicate blueprint reference {reference!r}") + registry[reference] = blueprint + if not registry: + raise ValueError("at least one positive assessment blueprint is required") + return registry + + +def resolve_blueprint(reference: Any) -> dict[str, Any]: + """Resolve one result's exact blueprint or fail closed.""" + require_reference(reference, "assessment_blueprint_reference") + try: + return load_blueprint_registry()[str(reference)] + except KeyError as error: + raise ValueError(f"unknown assessment blueprint reference {reference!r}") from error + + +def validate_result(value: dict[str, Any]) -> None: + """Validate structure, uncertainty, claims, and blueprint reporting authority.""" + validate_against_schema(value, SCHEMA_FILENAMES["result"]) + reject_forbidden_payload(value) + for field in ( + "result_reference", + "participant_reference", + "assessment_session_reference", + "instrument_release_reference", + "assessment_blueprint_reference", + "scoring_profile_reference", + "cut_score_revision_reference", + ): + require_reference(value.get(field), field) + blueprint = resolve_blueprint(value["assessment_blueprint_reference"]) + for result_field, blueprint_field in ( + ("instrument_release_reference", "instrument_release_reference"), + ("scoring_profile_reference", "scoring_profile_reference"), + ("cut_score_revision_reference", "cut_score_revision_reference"), + ("target_language", "target_language"), + ): + if value.get(result_field) != blueprint.get(blueprint_field): + raise ValueError( + f"result {result_field} must equal its immutable blueprint value" + ) + if value.get("claim_status_code") in {"cefr_linked", "certification_decision"}: + require_reference( + value.get("standard_setting_study_reference"), + "standard_setting_study_reference", + ) + require_reference( + value.get("linking_validation_reference"), + "linking_validation_reference", + ) + if value.get("claim_status_code") == "certification_decision": + require_reference( + value.get("certification_authority_reference"), + "certification_authority_reference", + ) + require_reference( + value.get("certification_policy_reference"), + "certification_policy_reference", + ) + domains = value.get("domain_results") + if not isinstance(domains, list) or not domains: + raise ValueError("domain results must be nonempty") + codes = [item.get("domain_code") for item in domains] + if len(codes) != len(set(codes)): + raise ValueError("domain identities must be unique") + measured_fields = { + "reported_level_code", + "level_probabilities", + "credible_level_set", + "standard_error", + "descriptor_coverage_references", + } + measured_domains: set[str] = set() + for item in domains: + if item.get("measurement_status_code") == "measured": + measured_domains.add(str(item.get("domain_code"))) + missing = sorted(field for field in measured_fields if field not in item) + if missing: + raise ValueError(f"measured domain is missing {missing}") + validate_probability_set( + item["level_probabilities"], + f"{item.get('domain_code')}.level_probabilities", + ) + if item["reported_level_code"] not in item["level_probabilities"]: + raise ValueError("reported domain level must have probability evidence") + if item["reported_level_code"] not in item["credible_level_set"]: + raise ValueError("reported domain level must be in credible set") + elif measured_fields & item.keys(): + raise ValueError("unmeasured domain must not carry an invented score") + required_domains = set(blueprint["required_domain_codes"]) + missing_required_domains = sorted(required_domains - measured_domains) + overall = value.get("overall_result") + if not isinstance(overall, dict): + raise ValueError("overall result is required") + declared_completeness = overall.get("required_domain_completeness_code") + actual_completeness = "incomplete" if missing_required_domains else "complete" + if declared_completeness != actual_completeness: + raise ValueError( + "overall required-domain completeness disagrees with the immutable blueprint" + ) + if overall.get("reporting_status_code") == "reported": + if blueprint.get("reporting_scope_code") != "overall_and_profile": + raise ValueError("assessment blueprint does not authorize overall reporting") + if missing_required_domains: + raise ValueError( + "overall result cannot be reported with unmeasured required domains" + ) + expected_policy = blueprint.get("overall_reporting_policy_reference") + if overall.get("reporting_policy_reference") != expected_policy: + raise ValueError( + "overall reporting policy must equal the immutable blueprint policy" + ) + validate_probability_set( + overall.get("level_probabilities"), + "overall.level_probabilities", + ) + if overall.get("reported_level_code") not in overall.get("level_probabilities", {}): + raise ValueError("reported overall level must have probability evidence") + if overall.get("reported_level_code") not in overall.get("credible_level_set", []): + raise ValueError("reported overall level must be in credible set") + elif any( + field in overall + for field in ( + "reported_level_code", + "level_probabilities", + "credible_level_set", + "reporting_policy_reference", + ) + ): + raise ValueError("not-reported overall result must not carry an invented score") + + +def validate_schema_metadata() -> None: + """Validate exact draft, version, identifiers, and schema metaschema syntax.""" + schemas = load_schemas() + if len(schemas) != 4: + raise ValueError("the CEFR v1 profile must publish exactly four schemas") + for name, schema in schemas.items(): + if schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema": + raise ValueError(f"{name} must use JSON Schema Draft 2020-12") + if schema.get("x-cwl-schema-version") != "1.0.0": + raise ValueError(f"{name} must declare schema version 1.0.0") + if schema.get("$id") != PUBLISHED_PREFIX + name: + raise ValueError(f"{name} has the wrong published $id") + try: + Draft202012Validator.check_schema(schema) + except SchemaError as error: + raise ValueError(f"{name} is not a valid Draft 2020-12 schema") from error + build_schema_registry() + + +def main() -> None: + """Run presence, Draft 2020-12, positive, and negative fixture validation.""" + required = [ + PROFILE_ROOT / "README.md", + Path("docs/adr/0002-cefr-language-assessment-profile.md"), + Path("docs/doctoring/CEFR_LANGUAGE_ASSESSMENT.md"), + Path("docs/superpowers/specs/2026-08-27-cefr-language-assessment-profile-design.md"), + Path("docs/superpowers/plans/2026-08-27-cefr-language-assessment-profile.md"), + Path("requirements-contracts-ci-hashes.txt"), + Path("tests/test_validate_cefr_profile.py"), + ] + missing = [str(path) for path in required if not path.is_file()] + if missing: + raise ValueError(f"missing CEFR profile files: {missing}") + validate_schema_metadata() + for path in sorted(VALID_ROOT.glob("assessment-blueprint*.json")): + validate_blueprint(load_json(path)) + for path in sorted(VALID_ROOT.glob("task-specification*.json")): + validate_task(load_json(path)) + for path in sorted(VALID_ROOT.glob("cefr-result-snapshot*.json")): + validate_result(load_json(path)) + negative: dict[str, Callable[[dict[str, Any]], None]] = { + "high-stakes-blueprint-without-standard-setting.json": validate_blueprint, + "task-with-copied-descriptor-text.json": validate_task, + "overall-result-with-incomplete-required-domain.json": validate_result, + "overall-result-with-profile-only-blueprint.json": validate_result, + "result-with-nonunit-probability-mass.json": validate_result, + } + for filename, validator in negative.items(): + try: + validator(load_json(INVALID_ROOT / filename)) + except ValueError: + continue + raise ValueError(f"negative CEFR fixture unexpectedly passed: {filename}") + print("CEFR language-assessment profile validation passed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_validate_cefr_profile.py b/tests/test_validate_cefr_profile.py new file mode 100644 index 0000000..af73b26 --- /dev/null +++ b/tests/test_validate_cefr_profile.py @@ -0,0 +1,91 @@ +"""Regression tests for CEFR schema and cross-artifact authority validation.""" + +from __future__ import annotations + +import copy +import unittest + +from scripts import validate_cefr_profile as validator + + +class CefrProfileValidationTests(unittest.TestCase): + """Lock structural schema and immutable blueprint authorization behavior.""" + + def test_blueprint_missing_schema_required_target_language_is_rejected(self) -> None: + """Custom checks cannot admit a blueprint rejected by Draft 2020-12.""" + blueprint = validator.load_json( + validator.VALID_ROOT / "assessment-blueprint.json" + ) + del blueprint["target_language"] + + with self.assertRaises(ValueError): + validator.validate_blueprint(blueprint) + + def test_blueprint_unknown_field_is_rejected(self) -> None: + """Closed schemas reject unknown payload fields before semantic checks.""" + blueprint = validator.load_json( + validator.VALID_ROOT / "assessment-blueprint.json" + ) + blueprint["unexpected_contract_field"] = "not-permitted" + + with self.assertRaises(ValueError): + validator.validate_blueprint(blueprint) + + def test_overall_result_is_rejected_by_profile_only_blueprint(self) -> None: + """A snapshot cannot self-authorize overall reporting.""" + profile_blueprint = validator.load_json( + validator.VALID_ROOT / "assessment-blueprint.json" + ) + result = validator.load_json( + validator.VALID_ROOT / "cefr-result-snapshot-linked-overall.json" + ) + result["assessment_blueprint_reference"] = profile_blueprint[ + "blueprint_reference" + ] + + with self.assertRaises(ValueError): + validator.validate_result(result) + + def test_overall_result_requires_every_blueprint_domain_measured(self) -> None: + """Snapshot completeness must match the immutable blueprint domain set.""" + overall_blueprint = validator.load_json( + validator.VALID_ROOT / "assessment-blueprint-overall.json" + ) + result = validator.load_json( + validator.VALID_ROOT / "cefr-result-snapshot-linked-overall.json" + ) + result["assessment_blueprint_reference"] = overall_blueprint[ + "blueprint_reference" + ] + result["domain_results"] = [ + domain + for domain in result["domain_results"] + if domain["domain_code"] != "spoken_production" + ] + + with self.assertRaises(ValueError): + validator.validate_result(result) + + def test_overall_policy_must_equal_blueprint_policy(self) -> None: + """A result cannot substitute its own overall-reporting policy.""" + overall_blueprint = validator.load_json( + validator.VALID_ROOT / "assessment-blueprint-overall.json" + ) + result = copy.deepcopy( + validator.load_json( + validator.VALID_ROOT / "cefr-result-snapshot-linked-overall.json" + ) + ) + result["assessment_blueprint_reference"] = overall_blueprint[ + "blueprint_reference" + ] + result["overall_result"][ + "reporting_policy_reference" + ] = "different_overall_reporting_policy" + + with self.assertRaises(ValueError): + validator.validate_result(result) + + +if __name__ == "__main__": + unittest.main()