From ab98f88e9e06b83ca1d7a1147ac9639156c4d5e8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 11 Aug 2026 17:33:21 +0900 Subject: [PATCH 1/2] feat(rag): governed scoring request with privacy fail-closed IDs Add reference-free RAG scoring adapter and reject raw content in system_configuration_id via descriptive identifiers. --- docs/changelog.d/691-rag-governed-request.md | 5 + docs/doctoring/rag_scoring_request_privacy.md | 15 + python/fast_mlsirm/scoring/rag.py | 224 +++++++++++++++ tests/test_scoring_rag_request.py | 262 ++++++++++++++++++ 4 files changed, 506 insertions(+) create mode 100644 docs/changelog.d/691-rag-governed-request.md create mode 100644 docs/doctoring/rag_scoring_request_privacy.md create mode 100644 python/fast_mlsirm/scoring/rag.py create mode 100644 tests/test_scoring_rag_request.py diff --git a/docs/changelog.d/691-rag-governed-request.md b/docs/changelog.d/691-rag-governed-request.md new file mode 100644 index 000000000..ea35e2d10 --- /dev/null +++ b/docs/changelog.d/691-rag-governed-request.md @@ -0,0 +1,5 @@ +# Governed RAG scoring request + +### Added + +- Reference-free RAG scoring request adapter with privacy-preserving identity channels and fail-closed rejection of raw system configuration content. diff --git a/docs/doctoring/rag_scoring_request_privacy.md b/docs/doctoring/rag_scoring_request_privacy.md new file mode 100644 index 000000000..2d315713e --- /dev/null +++ b/docs/doctoring/rag_scoring_request_privacy.md @@ -0,0 +1,15 @@ +# Governed RAG scoring request privacy + +## Standards + +American Educational Research Association, American Psychological Association, & National Council on Measurement in Education. (2014). *Standards for educational and psychological testing*. American Educational Research Association. + +National Institute of Standards and Technology. (2020). *Security and privacy controls for information systems and organizations* (NIST SP 800-53 Rev. 5). https://doi.org/10.6028/NIST.SP.800-53r5 + +## Rationale + +Reference-free RAG evaluation records provenance identities (system configuration, retrieval run, query revision fingerprints) without embedding raw query, context, or answer text. Managed identifiers must reject free-form content so the canonical scoring artifact cannot become a side channel. + +## Implementation + +`python/fast_mlsirm/scoring/rag.py` — `build_rag_scoring_request` validates `system_configuration_id` via `descriptive_identifier` and allowlists caller metadata. diff --git a/python/fast_mlsirm/scoring/rag.py b/python/fast_mlsirm/scoring/rag.py new file mode 100644 index 000000000..8194d9ac1 --- /dev/null +++ b/python/fast_mlsirm/scoring/rag.py @@ -0,0 +1,224 @@ +"""Governed adapters for reference-free RAG scoring requests. + +This module deliberately owns validation and provenance marshalling only. It +reuses the canonical :class:`ScoringRequest` contract and performs no retrieval, +LLM inference, metric arithmetic, thresholding, or truth adjudication. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from enum import Enum +from typing import Any + +from ._contract_safety import enum_value, freeze_metadata +from ._validation import ( + assessment_error, + descriptive_identifier, + fingerprint, + thaw_json_value, +) +from .assessment import AssessmentSpec +from .authorization import build_scoring_request +from .execution import ObservationGranularity, ScoringRequest +from fast_mlsirm.rubric.models import RubricSpecification + + +class RAGEvidenceRegime(str, Enum): + """Evidence available to one governed reference-free RAG evaluation.""" + + PROMPT_ONLY = "prompt_only" + RETRIEVED_CONTEXT = "retrieved_context" + POOLED_CORPUS = "pooled_corpus" + AUTHORITATIVE_CORPUS = "authoritative_corpus" + HUMAN_ANCHOR = "human_anchor" + + +class RAGCandidateVisibility(str, Enum): + """Whether scoring evidence can expose candidate-system identity/content.""" + + CANDIDATE_BLIND = "candidate_blind" + CANDIDATE_VISIBLE_CROSSFIT = "candidate_visible_crossfit" + + +_MANAGED_METADATA_KEYS = frozenset( + { + "rag_evidence_regime", + "rag_candidate_visibility", + "rag_system_configuration_id", + "rag_system_configuration_fingerprint", + "rag_retrieval_run_fingerprint", + "rag_query_revision_fingerprint", + } +) +_ALLOWED_CALLER_METADATA_KEYS = frozenset({"evaluation_split"}) + + +def _rag_metadata( + *, + metadata: Mapping[str, Any] | None, + evidence_regime: RAGEvidenceRegime, + candidate_visibility: RAGCandidateVisibility, + system_configuration_id: str, + system_configuration_fingerprint: str, + retrieval_run_fingerprint: str, + query_revision_fingerprint: str, +) -> dict[str, Any]: + """Return allowlisted caller metadata plus package-managed RAG provenance.""" + raw_metadata: Mapping[str, Any] = {} if metadata is None else metadata + if not isinstance(raw_metadata, Mapping): + raise assessment_error( + "invalid_rag_metadata", + "$.metadata", + "metadata must be a mapping", + ) + if any(key in raw_metadata for key in _MANAGED_METADATA_KEYS): + raise assessment_error( + "reserved_rag_metadata", + "$.metadata", + "RAG provenance metadata is package-managed", + ) + if any(key not in _ALLOWED_CALLER_METADATA_KEYS for key in raw_metadata): + raise assessment_error( + "unsupported_rag_metadata", + "$.metadata", + "metadata key is not allowed for RAG scoring requests", + ) + frozen = freeze_metadata(raw_metadata) + if not isinstance(frozen, Mapping): + raise assessment_error( + "invalid_rag_metadata", + "$.metadata", + "metadata must be a mapping", + ) + output = thaw_json_value(frozen) + if not isinstance(output, dict): + raise assessment_error( + "invalid_rag_metadata", + "$.metadata", + "metadata must be a mapping", + ) + if "evaluation_split" in output: + output["evaluation_split"] = descriptive_identifier( + output["evaluation_split"], + "evaluation_split", + "$.metadata.evaluation_split", + ) + output.update( + { + "rag_evidence_regime": evidence_regime.value, + "rag_candidate_visibility": candidate_visibility.value, + "rag_system_configuration_id": system_configuration_id, + "rag_system_configuration_fingerprint": system_configuration_fingerprint, + "rag_retrieval_run_fingerprint": retrieval_run_fingerprint, + "rag_query_revision_fingerprint": query_revision_fingerprint, + } + ) + return output + + +def build_rag_scoring_request( + *, + request_id: str, + assessment: AssessmentSpec, + rubric: RubricSpecification, + query_id: str, + query_revision_fingerprint: str, + query_testlet_id: str, + evidence_regime: RAGEvidenceRegime | str, + candidate_visibility: RAGCandidateVisibility | str, + system_configuration_id: str, + system_configuration_fingerprint: str, + system_run_id: str, + response_id: str, + retrieval_run_fingerprint: str, + response_content_fingerprint: str, + occasion_id: str, + criterion_ids: Iterable[str] = (), + response_character_count: int, + response_unit_count: int, + metadata: Mapping[str, Any] | None = None, +) -> ScoringRequest: + """Build one provenance-bound criterion-level RAG scoring request. + + The RAG-specific identities are projected onto the shared scoring axes: + stochastic system run -> respondent, generated answer -> response, query -> + task, exact query revision -> task revision, and query testlet -> task + family. System configuration identity, evidence regime, candidate visibility, + retrieval-run identity, and exact configuration/revision fingerprints are + package-managed metadata and therefore participate in the canonical request + fingerprint. + + Raw query, context, answer, and source text are intentionally absent from + this interface. Caller metadata is intentionally allowlisted to prevent raw + content from being smuggled into the canonical artifact. A reference-free + request records its evidence regime; it does not imply world correctness, + absolute retrieval recall, or deployment validity. + """ + normalized_regime = enum_value( + evidence_regime, + RAGEvidenceRegime, + "rag_evidence_regime", + "$.evidence_regime", + ) + normalized_visibility = enum_value( + candidate_visibility, + RAGCandidateVisibility, + "rag_candidate_visibility", + "$.candidate_visibility", + ) + normalized_query_revision = fingerprint( + query_revision_fingerprint, + "query_revision_fingerprint", + "$.query_revision_fingerprint", + ) + normalized_system_fingerprint = fingerprint( + system_configuration_fingerprint, + "system_configuration_fingerprint", + "$.system_configuration_fingerprint", + ) + normalized_retrieval_fingerprint = fingerprint( + retrieval_run_fingerprint, + "retrieval_run_fingerprint", + "$.retrieval_run_fingerprint", + ) + normalized_system_configuration_id = descriptive_identifier( + system_configuration_id, + "system_configuration_id", + "$.system_configuration_id", + ) + + rag_metadata = _rag_metadata( + metadata=metadata, + evidence_regime=normalized_regime, + candidate_visibility=normalized_visibility, + system_configuration_id=normalized_system_configuration_id, + system_configuration_fingerprint=normalized_system_fingerprint, + retrieval_run_fingerprint=normalized_retrieval_fingerprint, + query_revision_fingerprint=normalized_query_revision, + ) + + return build_scoring_request( + request_id=request_id, + assessment=assessment, + rubric=rubric, + granularity=ObservationGranularity.CRITERION_LEVEL, + respondent_id=system_run_id, + response_id=response_id, + task_id=query_id, + task_revision_fingerprint=normalized_query_revision, + task_family_id=query_testlet_id, + occasion_id=occasion_id, + criterion_ids=criterion_ids, + response_content_fingerprint=response_content_fingerprint, + response_character_count=response_character_count, + response_unit_count=response_unit_count, + metadata=rag_metadata, + ) + + +__all__ = [ + "RAGCandidateVisibility", + "RAGEvidenceRegime", + "build_rag_scoring_request", +] diff --git a/tests/test_scoring_rag_request.py b/tests/test_scoring_rag_request.py new file mode 100644 index 000000000..60f230b8f --- /dev/null +++ b/tests/test_scoring_rag_request.py @@ -0,0 +1,262 @@ +"""Fail-first contracts for governed reference-free RAG scoring requests.""" + +from __future__ import annotations + +import hashlib +import inspect +from pathlib import Path +import runpy +from typing import Any + +import pytest + +from fast_mlsirm.scoring import AssessmentSpecError, ObservationGranularity, ScoringRequest +from fast_mlsirm.scoring.rag import ( + RAGCandidateVisibility, + RAGEvidenceRegime, + build_rag_scoring_request, +) + +_FIXTURES = runpy.run_path( + str(Path(__file__).with_name("scoring_execution_fixtures.py")) +) +assessment = _FIXTURES["assessment"] +rubric = _FIXTURES["rubric"] + +QUERY_FP = hashlib.sha256(b"rag-query-revision").hexdigest() +SYSTEM_FP = hashlib.sha256(b"rag-system-configuration").hexdigest() +RETRIEVAL_FP = hashlib.sha256(b"rag-retrieval-run").hexdigest() +RESPONSE_FP = hashlib.sha256(b"rag-response-content").hexdigest() +SECOND_RETRIEVAL_FP = hashlib.sha256(b"rag-retrieval-run-second").hexdigest() +SECOND_RESPONSE_FP = hashlib.sha256(b"rag-response-content-second").hexdigest() + + +def _request(**overrides: Any) -> ScoringRequest: + """Build one deterministic reference-free RAG criterion request.""" + values: dict[str, Any] = { + "request_id": "rag_quality_request", + "assessment": assessment(), + "rubric": rubric(), + "query_id": "refund_policy_query", + "query_revision_fingerprint": QUERY_FP, + "query_testlet_id": "evidence_review", + "evidence_regime": RAGEvidenceRegime.RETRIEVED_CONTEXT, + "candidate_visibility": RAGCandidateVisibility.CANDIDATE_BLIND, + "system_configuration_id": "retrieval_stack_a", + "system_configuration_fingerprint": SYSTEM_FP, + "system_run_id": "retrieval_stack_a_run_001", + "response_id": "generated_response_001", + "retrieval_run_fingerprint": RETRIEVAL_FP, + "response_content_fingerprint": RESPONSE_FP, + "occasion_id": "evaluation_wave_001", + "criterion_ids": ("grounded_generation", "answer_relevance"), + "response_character_count": 412, + "response_unit_count": 7, + "metadata": {"evaluation_split": "offline_holdout"}, + } + values.update(overrides) + return build_rag_scoring_request(**values) + + +def _assert_error(code: str, callback) -> None: + """Assert one stable shared scoring error code.""" + with pytest.raises(AssessmentSpecError) as caught: + callback() + assert caught.value.code == code + + +def test_public_rag_contract_uses_explicit_evidence_and_visibility_enums() -> None: + """Reference-free evaluation must declare what evidence and candidate access mean.""" + assert {member.value for member in RAGEvidenceRegime} == { + "prompt_only", + "retrieved_context", + "pooled_corpus", + "authoritative_corpus", + "human_anchor", + } + assert {member.value for member in RAGCandidateVisibility} == { + "candidate_blind", + "candidate_visible_crossfit", + } + assert build_rag_scoring_request.__doc__ + + +def test_rag_request_reuses_shared_scoring_axes_without_parallel_schema() -> None: + """System run, generated response, and query revision map to shared axes.""" + request = _request() + + assert isinstance(request, ScoringRequest) + assert request.granularity is ObservationGranularity.CRITERION_LEVEL + assert request.respondent_id == "retrieval_stack_a_run_001" + assert request.response_id == "generated_response_001" + assert request.task_id == "refund_policy_query" + assert request.task_revision_fingerprint == QUERY_FP + assert request.task_family_id == "evidence_review" + assert request.response_content_fingerprint == RESPONSE_FP + assert request.occasion_id == "evaluation_wave_001" + assert request.criterion_ids == ("answer_relevance", "grounded_generation") + assert request.to_dict()["metadata"]["rag_system_configuration_id"] == ( + "retrieval_stack_a" + ) + + +def test_rag_request_preserves_reference_free_provenance_without_raw_text() -> None: + """The governed numerical artifact retains identities, not question/context/answer text.""" + request = _request() + payload = request.to_dict() + metadata = payload["metadata"] + + assert metadata["evaluation_split"] == "offline_holdout" + assert metadata["rag_evidence_regime"] == "retrieved_context" + assert metadata["rag_candidate_visibility"] == "candidate_blind" + assert metadata["rag_system_configuration_id"] == "retrieval_stack_a" + assert metadata["rag_system_configuration_fingerprint"] == SYSTEM_FP + assert metadata["rag_retrieval_run_fingerprint"] == RETRIEVAL_FP + assert metadata["rag_query_revision_fingerprint"] == QUERY_FP + assert metadata["engine_policy_fingerprint"] + + serialized = repr(payload).lower() + for forbidden in ( + "query_text", + "question_text", + "context_text", + "retrieved_text", + "answer_text", + "response_text", + "source_text", + ): + assert forbidden not in serialized + + parameters = inspect.signature(build_rag_scoring_request).parameters + assert not set(parameters).intersection( + { + "query_text", + "question_text", + "context_text", + "retrieved_text", + "answer_text", + "response_text", + "source_text", + } + ) + + +def test_stochastic_runs_do_not_collapse_into_one_system_identity() -> None: + """One configuration can produce distinct governed run respondents.""" + first = _request() + second = _request( + system_run_id="retrieval_stack_a_run_002", + response_id="generated_response_002", + retrieval_run_fingerprint=SECOND_RETRIEVAL_FP, + response_content_fingerprint=SECOND_RESPONSE_FP, + ) + + assert first.respondent_id != second.respondent_id + assert first.response_id != second.response_id + assert first.response_content_fingerprint != second.response_content_fingerprint + assert first.request_fingerprint != second.request_fingerprint + assert first.to_dict()["metadata"]["rag_system_configuration_id"] == ( + second.to_dict()["metadata"]["rag_system_configuration_id"] + ) + assert first.to_dict()["metadata"]["rag_system_configuration_fingerprint"] == ( + second.to_dict()["metadata"]["rag_system_configuration_fingerprint"] + ) + + +def test_evidence_regime_and_candidate_visibility_are_identity_bearing() -> None: + """Changing the evidence claim or candidate access must change request identity.""" + baseline = _request() + authoritative = _request(evidence_regime=RAGEvidenceRegime.AUTHORITATIVE_CORPUS) + crossfit = _request( + candidate_visibility=RAGCandidateVisibility.CANDIDATE_VISIBLE_CROSSFIT + ) + + assert baseline.request_fingerprint != authoritative.request_fingerprint + assert baseline.request_fingerprint != crossfit.request_fingerprint + + +def test_rag_managed_metadata_cannot_be_spoofed_by_callers() -> None: + """Caller metadata cannot rebind package-managed RAG provenance.""" + for key, value in ( + ("rag_evidence_regime", "human_anchor"), + ("rag_candidate_visibility", "candidate_visible_crossfit"), + ("rag_system_configuration_id", "spoofed_configuration"), + ("rag_system_configuration_fingerprint", SYSTEM_FP), + ("rag_retrieval_run_fingerprint", RETRIEVAL_FP), + ("rag_query_revision_fingerprint", QUERY_FP), + ): + _assert_error( + "reserved_rag_metadata", + lambda key=key, value=value: _request(metadata={key: value}), + ) + + +def test_rag_contract_rejects_unknown_enums_and_malformed_fingerprints() -> None: + """Evidence meaning and exact provenance fail closed instead of being guessed.""" + _assert_error( + "invalid_rag_evidence_regime", + lambda: _request(evidence_regime="web_truth"), + ) + _assert_error( + "invalid_rag_candidate_visibility", + lambda: _request(candidate_visibility="candidate_visible"), + ) + _assert_error( + "invalid_system_configuration_fingerprint", + lambda: _request(system_configuration_fingerprint="not-a-sha256"), + ) + _assert_error( + "invalid_retrieval_run_fingerprint", + lambda: _request(retrieval_run_fingerprint="not-a-sha256"), + ) + + +def test_query_revision_metadata_replays_shared_task_revision_identity() -> None: + """RAG metadata cannot disagree with the shared scoring task-revision axis.""" + request = _request() + assert request.task_revision_fingerprint == ( + request.to_dict()["metadata"]["rag_query_revision_fingerprint"] + ) + + +def test_rag_identity_separates_configuration_run_and_generated_response() -> None: + """A system run is the measured respondent while each generated answer is a response.""" + request = _request(response_id="generated_response_001") + + assert request.respondent_id == "retrieval_stack_a_run_001" + assert request.response_id == "generated_response_001" + assert request.to_dict()["metadata"]["rag_system_configuration_id"] == ( + "retrieval_stack_a" + ) + + +@pytest.mark.parametrize( + "metadata_key", + ("audit_note", "query_text", "context_text", "answer_text", "source_text"), +) +def test_rag_metadata_rejects_non_allowlisted_content_keys(metadata_key: str) -> None: + """Caller metadata cannot smuggle source or response content into artifacts.""" + _assert_error( + "unsupported_rag_metadata", + lambda: _request(metadata={metadata_key: "sensitive_content"}), + ) + + +def test_rag_evaluation_split_rejects_raw_content_value() -> None: + """The sole allowlisted metadata value is an identifier, never a content channel.""" + raw_query = "What is the refund policy for enterprise customers?" + with pytest.raises(AssessmentSpecError) as caught: + _request(metadata={"evaluation_split": raw_query}) + + assert caught.value.code == "invalid_evaluation_split" + assert raw_query not in str(caught.value) + + +def test_rag_system_configuration_id_rejects_raw_content_value() -> None: + """Managed system identity cannot become a raw query/context side channel.""" + raw_configuration = "Use this retrieved context to answer the customer refund question" + with pytest.raises(AssessmentSpecError) as caught: + _request(system_configuration_id=raw_configuration) + + assert caught.value.code == "invalid_system_configuration_id" + assert raw_configuration not in str(caught.value) From dbac4899cbb2b180826237f6f2b213ef56f68bf3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 11 Aug 2026 18:04:22 +0900 Subject: [PATCH 2/2] docs(changelog): use H2 section headings for fragment contract --- docs/changelog.d/691-rag-governed-request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.d/691-rag-governed-request.md b/docs/changelog.d/691-rag-governed-request.md index ea35e2d10..f8eff1856 100644 --- a/docs/changelog.d/691-rag-governed-request.md +++ b/docs/changelog.d/691-rag-governed-request.md @@ -1,5 +1,5 @@ # Governed RAG scoring request -### Added +## Added - Reference-free RAG scoring request adapter with privacy-preserving identity channels and fail-closed rejection of raw system configuration content.