diff --git a/docs/changelog.d/rag-facets-calibration.md b/docs/changelog.d/rag-facets-calibration.md new file mode 100644 index 000000000..813661265 --- /dev/null +++ b/docs/changelog.d/rag-facets-calibration.md @@ -0,0 +1,13 @@ +# Governed RAG facets calibration + +## Added + +- Added the governed RAG facets calibration adapter. +- The adapter reuses the existing MFRM rater-severity/threshold calibration + machinery (Linacre, 1989; Eckes, 2015; Bock & Aitkin, 1981; Andrich, 1978) + for RAG evaluation executions. It does not introduce a new psychometric + estimator or rely on legacy evaluation package implementations (e.g. + RAGAS-style tooling) as the source of psychometric validity; all + likelihood/threshold arithmetic is delegated to the existing Rust-backed MFRM + fit grounded in the primary many-facet Rasch measurement literature. Full + citations are in `docs/scoring_facets_calibration_handoff.md`. diff --git a/python/fast_mlsirm/scoring/rag_calibration.py b/python/fast_mlsirm/scoring/rag_calibration.py new file mode 100644 index 000000000..56786d5fa --- /dev/null +++ b/python/fast_mlsirm/scoring/rag_calibration.py @@ -0,0 +1,111 @@ +"""Project governed RAG scoring executions into shared many-facet designs. + +This adapter replays the package-managed RAG request provenance before handing +criterion observations to the existing scoring calibration contracts. Python +only validates identities and marshals records. All many-facet likelihood, +threshold, estimation, optimization, and uncertainty arithmetic remains owned +by the existing Rust-backed calibration path. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from ._contract_safety import bounded_values +from ._validation import assessment_error +from .calibration import ( + MAX_SCORING_FACETS_RATINGS, + ScoringFacetsCalibrationBundle, + ScoringFacetsRatingRecord, + build_scoring_facets_calibration_bundle, + build_scoring_facets_rating_records, +) +from .execution import EngineDescriptor, ScoringRequest, ScoringResult +from .rag import _canonical_rag_request + +MAX_RAG_CALIBRATION_EXECUTIONS = MAX_SCORING_FACETS_RATINGS +"""Maximum governed RAG executions accepted by one facets bundle assembly.""" + + +def build_rag_facets_rating_records( + *, + request: ScoringRequest, + result: ScoringResult, + engine: EngineDescriptor, +) -> tuple[ScoringFacetsRatingRecord, ...]: + """Project one provenance-complete RAG execution into shared facets records. + + The RAG request must contain the package-managed evidence-regime, candidate + visibility, system-configuration, retrieval-run, and query-revision + provenance produced by :func:`fast_mlsirm.scoring.rag.build_rag_scoring_request`. + The shared projector remains authoritative for exact request/result/engine + binding, terminal observation states, criterion scales, and rater identity. + + Passing this boundary establishes replayable measurement provenance only. + It does not establish factual truth, retrieval recall, model adequacy, + validity, fairness, invariance, or a buyer-facing quality conclusion. + """ + normalized_request = _canonical_rag_request(request, "rag_scoring_request") + return build_scoring_facets_rating_records( + request=normalized_request, + result=result, + engine=engine, + ) + + +def build_rag_facets_calibration_bundle( + executions: Iterable[tuple[ScoringRequest, ScoringResult, EngineDescriptor]], + *, + require_connected: bool = True, +) -> ScoringFacetsCalibrationBundle: + """Assemble governed RAG executions into the existing many-facet bundle. + + Each execution is an exact three-value tuple containing a canonical RAG + scoring request, its bound scoring result, and the producing engine + descriptor. Every request is replay-validated as RAG provenance before its + observations are projected into the shared criterion-separated design. + + The returned value is the existing :class:`ScoringFacetsCalibrationBundle`; + this module introduces no parallel psychometric result hierarchy and no new + numerical estimator. System-run identity occupies the respondent axis, + query revision occupies the task-revision axis, and engine fingerprint + occupies the rater axis, preserving the hierarchy already encoded by the + canonical RAG scoring request. + """ + values = bounded_values( + executions, + "executions", + minimum=1, + maximum=MAX_RAG_CALIBRATION_EXECUTIONS, + ) + + def validated_records() -> Iterable[ScoringFacetsRatingRecord]: + """Yield replay-validated RAG records for bounded shared assembly.""" + for index, execution in enumerate(values): + if type(execution) is not tuple or len(execution) != 3: + raise assessment_error( + "invalid_rag_calibration_execution", + f"$.executions[{index}]", + ( + "each execution must be an exact three-value tuple of " + "request, result, and engine" + ), + ) + request, result, engine = execution + yield from build_rag_facets_rating_records( + request=request, + result=result, + engine=engine, + ) + + return build_scoring_facets_calibration_bundle( + validated_records(), + require_connected=require_connected, + ) + + +__all__ = [ + "MAX_RAG_CALIBRATION_EXECUTIONS", + "build_rag_facets_calibration_bundle", + "build_rag_facets_rating_records", +] diff --git a/tests/test_scoring_rag_facets_calibration.py b/tests/test_scoring_rag_facets_calibration.py new file mode 100644 index 000000000..3b1f76deb --- /dev/null +++ b/tests/test_scoring_rag_facets_calibration.py @@ -0,0 +1,207 @@ +"""Many-facet orchestration contracts for governed RAG scoring executions.""" + +from __future__ import annotations + +import hashlib +import inspect +from pathlib import Path +import runpy + +import pytest + +from fast_mlsirm.scoring import ( + AssessmentSpecError, + FixtureOutcome, + ObservationStatus, + ScoringFacetsCalibrationBundle, + ScoringFacetsRatingRecord, + StaticFixtureEngine, +) +from fast_mlsirm.scoring.rag import build_rag_scoring_request +from fast_mlsirm.scoring.rag_calibration import ( + build_rag_facets_calibration_bundle, + build_rag_facets_rating_records, +) + +_FIXTURES = runpy.run_path( + str(Path(__file__).with_name("scoring_execution_fixtures.py")) +) +assessment = _FIXTURES["assessment"] +rubric = _FIXTURES["rubric"] +automated_engine = _FIXTURES["automated_engine"] +criterion_request = _FIXTURES["criterion_request"] + +QUERY_REVISION_FPS = tuple( + hashlib.sha256(f"rag-calibration-query-revision-{index:03d}".encode()).hexdigest() + for index in (1, 2) +) +SYSTEM_FP = hashlib.sha256(b"rag-calibration-system").hexdigest() + + +def _execution( + respondent_index: int = 1, + task_index: int = 1, + engine_id: str = "fixture_engine", +): + """Return one deterministic governed RAG request/result/engine execution.""" + respondent_suffix = f"{respondent_index:03d}" + task_suffix = f"{task_index:03d}" + scenario_index = (respondent_index - 1) * 2 + (task_index - 1) + grounded_scores = (2, 1, 0, 2) + relevance_scores = (1, 2, 0, 1) + request = build_rag_scoring_request( + request_id=( + f"rag_calibration_request_{respondent_suffix}_{task_suffix}" + ), + assessment=assessment(), + rubric=rubric(), + query_id="refund_policy_query", + query_revision_fingerprint=QUERY_REVISION_FPS[task_index - 1], + query_testlet_id="evidence_review", + evidence_regime="retrieved_context", + candidate_visibility="candidate_blind", + system_configuration_id="retrieval_stack_a", + system_configuration_fingerprint=SYSTEM_FP, + system_run_id=f"retrieval_stack_a_run_{respondent_suffix}", + response_id=f"generated_response_{respondent_suffix}_{task_suffix}", + retrieval_run_fingerprint=hashlib.sha256( + f"rag-calibration-retrieval-{respondent_suffix}-{task_suffix}".encode() + ).hexdigest(), + response_content_fingerprint=hashlib.sha256( + f"rag-calibration-response-{respondent_suffix}-{task_suffix}".encode() + ).hexdigest(), + occasion_id="evaluation_wave_001", + criterion_ids=("grounded_generation", "answer_relevance"), + response_character_count=412 + scenario_index, + response_unit_count=7, + metadata={"evaluation_split": "offline_holdout"}, + ) + engine = automated_engine(engine_id=engine_id) + fixture = StaticFixtureEngine( + descriptor=engine, + outcomes=( + FixtureOutcome( + criterion_id="grounded_generation", + status=ObservationStatus.SCORED, + score_category=grounded_scores[scenario_index], + ), + FixtureOutcome( + criterion_id="answer_relevance", + status=ObservationStatus.SCORED, + score_category=relevance_scores[scenario_index], + ), + ), + ) + return request, fixture.score(request), engine + + +def test_rag_facets_projection_reuses_shared_calibration_contracts() -> None: + """RAG provenance projects into the shared facets record hierarchy only.""" + request, result, engine = _execution() + + records = build_rag_facets_rating_records( + request=request, + result=result, + engine=engine, + ) + + assert all(type(record) is ScoringFacetsRatingRecord for record in records) + assert {record.criterion_id for record in records} == { + "grounded_generation", + "answer_relevance", + } + assert {record.respondent_id for record in records} == {request.respondent_id} + assert {record.task_id for record in records} == {request.task_id} + assert {record.task_revision_fingerprint for record in records} == { + request.task_revision_fingerprint + } + assert {record.engine_fingerprint for record in records} == { + engine.engine_fingerprint + } + + +def test_rag_facets_bundle_delegates_to_shared_many_facet_design() -> None: + """RAG orchestration returns one connected shared many-facet design.""" + executions = ( + _execution(1, 1, "fixture_engine"), + _execution(1, 2, "alternate_engine"), + _execution(2, 1, "alternate_engine"), + _execution(2, 2, "fixture_engine"), + ) + + bundle = build_rag_facets_calibration_bundle(executions) + + assert type(bundle) is ScoringFacetsCalibrationBundle + assert bundle.criterion_ids == ("answer_relevance", "grounded_generation") + expected_revisions = tuple(sorted(QUERY_REVISION_FPS)) + for design in bundle.designs: + assert design.respondent_ids == ( + "retrieval_stack_a_run_001", + "retrieval_stack_a_run_002", + ) + assert design.task_revision_fingerprints == expected_revisions + assert design.task_ids == ( + "refund_policy_query", + "refund_policy_query", + ) + assert set(design.rater_engine_ids) == { + "alternate_engine", + "fixture_engine", + } + assert design.respondent_task_connected + assert design.task_rater_connected + assert design.connected + assert { + record.score_category + for record in design.rating_records + if record.status is ObservationStatus.SCORED + } == {0, 1, 2} + + +def test_rag_facets_projection_rejects_non_rag_request() -> None: + """A generic scoring execution cannot masquerade as governed RAG evidence.""" + generic_request = criterion_request() + engine = automated_engine() + result = StaticFixtureEngine( + descriptor=engine, + outcomes=( + FixtureOutcome( + criterion_id="claim_support", + status=ObservationStatus.SCORED, + score_category=2, + ), + FixtureOutcome( + criterion_id="source_alignment", + status=ObservationStatus.SCORED, + score_category=1, + ), + ), + ).score(generic_request) + + with pytest.raises(AssessmentSpecError) as caught: + build_rag_facets_rating_records( + request=generic_request, + result=result, + engine=engine, + ) + + assert caught.value.code == "invalid_rag_scoring_request" + + +def test_rag_facets_surface_accepts_no_raw_rag_content() -> None: + """Calibration orchestration remains identity-only at the public boundary.""" + forbidden = { + "query_text", + "question_text", + "context_text", + "retrieved_text", + "answer_text", + "response_text", + "source_text", + } + for function in ( + build_rag_facets_rating_records, + build_rag_facets_calibration_bundle, + ): + assert not forbidden.intersection(inspect.signature(function).parameters) + assert function.__doc__ diff --git a/tests/test_scoring_rag_facets_calibration_bounds.py b/tests/test_scoring_rag_facets_calibration_bounds.py new file mode 100644 index 000000000..07fc3c232 --- /dev/null +++ b/tests/test_scoring_rag_facets_calibration_bounds.py @@ -0,0 +1,16 @@ +"""Boundary coverage for governed RAG calibration assembly.""" + +from __future__ import annotations + +import pytest + +from fast_mlsirm.scoring import AssessmentSpecError +from fast_mlsirm.scoring.rag_calibration import build_rag_facets_calibration_bundle + + +def test_rag_facets_bundle_rejects_non_triple_execution() -> None: + """Every calibration execution must preserve request, result, and engine.""" + with pytest.raises(AssessmentSpecError) as caught: + build_rag_facets_calibration_bundle(((object(), object()),)) + + assert caught.value.code == "invalid_rag_calibration_execution"