From 1a1403eb521d14650fd4666e4dba4e8be62aa523 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 23:37:23 -0700 Subject: [PATCH 1/5] test(rag): reproduce evidence limitation replay mutation --- .../test_scoring_rag_evidence_limitations.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_scoring_rag_evidence_limitations.py b/tests/test_scoring_rag_evidence_limitations.py index eda4a7e36..7ca8f9eb0 100644 --- a/tests/test_scoring_rag_evidence_limitations.py +++ b/tests/test_scoring_rag_evidence_limitations.py @@ -71,6 +71,33 @@ def test_limitations_contract_is_factory_sealed() -> None: ) +def test_limitations_replay_rejects_rebound_container_before_callbacks() -> None: + """Post-construction container rebinding cannot execute during replay.""" + callbacks: list[str] = [] + + class HostileTuple(tuple): + def __iter__(self): # type: ignore[override] + callbacks.append("iter") + raise AssertionError("hostile limitation iteration executed") + + for projection in ( + lambda value: value.to_dict(), + lambda value: value.limitations_fingerprint, + ): + limits = rag_evidence_regime_limitations(RAGEvidenceRegime.RETRIEVED_CONTEXT) + object.__setattr__( + limits, + "limitation_codes", + HostileTuple(limits.limitation_codes), + ) + + with pytest.raises(AssessmentSpecError) as captured: + projection(limits) + + assert captured.value.code == "invalid_rag_evidence_limitations" + assert callbacks == [] + + def test_evidence_limitations_are_explicit_scoring_package_attributes() -> None: """Governed reporting callers should not need an internal-module import path.""" assert scoring.RAGEvidenceRegimeLimitations is RAGEvidenceRegimeLimitations From 607e4d24dda97deba898fae91ae42a501d2451ca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 23:37:56 -0700 Subject: [PATCH 2/5] fix(rag): replay evidence limitation integrity --- python/fast_mlsirm/scoring/rag_evidence.py | 50 +++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/python/fast_mlsirm/scoring/rag_evidence.py b/python/fast_mlsirm/scoring/rag_evidence.py index 1952e6bb7..32a77b9c8 100644 --- a/python/fast_mlsirm/scoring/rag_evidence.py +++ b/python/fast_mlsirm/scoring/rag_evidence.py @@ -84,17 +84,57 @@ def _content_dict(self) -> dict[str, Any]: @property def limitations_fingerprint(self) -> str: - """Return SHA-256 over the exact interpretation-limit contract.""" - return artifact_digest(self) + """Return SHA-256 over the replay-validated interpretation-limit contract.""" + replayed = _replay_limitations(self) + return artifact_digest(replayed) def to_dict(self) -> dict[str, Any]: - """Return canonical limitation content plus its deterministic digest.""" + """Return replay-validated limitation content and deterministic digest.""" + replayed = _replay_limitations(self) return { - **self._content_dict(), - "limitations_fingerprint": self.limitations_fingerprint, + **RAGEvidenceRegimeLimitations._content_dict(replayed), + "limitations_fingerprint": artifact_digest(replayed), } +def _replay_limitations( + value: RAGEvidenceRegimeLimitations, +) -> RAGEvidenceRegimeLimitations: + """Re-establish exact factory-derived state before manifest projection.""" + if type(value) is not RAGEvidenceRegimeLimitations: + raise assessment_error( + "invalid_rag_evidence_limitations", + "$", + "RAG evidence limitations must retain factory-derived state", + ) + regime = value.regime + limitation_codes = value.limitation_codes + if type(regime) is not RAGEvidenceRegime or type(limitation_codes) is not tuple: + raise assessment_error( + "invalid_rag_evidence_limitations", + "$", + "RAG evidence limitations must retain factory-derived state", + ) + if any(type(code) is not str for code in limitation_codes): + raise assessment_error( + "invalid_rag_evidence_limitations", + "$", + "RAG evidence limitations must retain factory-derived state", + ) + expected = _REGIME_LIMITATIONS[regime] + if limitation_codes != expected: + raise assessment_error( + "invalid_rag_evidence_limitations", + "$", + "RAG evidence limitations must retain factory-derived state", + ) + return RAGEvidenceRegimeLimitations( + regime=regime, + limitation_codes=expected, + _limitations_token=_RAG_EVIDENCE_LIMITATIONS_TOKEN, + ) + + def rag_evidence_regime_limitations( regime: RAGEvidenceRegime | str, ) -> RAGEvidenceRegimeLimitations: From fe525d780b82c53b66585fe6b85f3ae4621f473c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 23:38:39 -0700 Subject: [PATCH 3/5] test(rag): cover replay integrity state gate --- python/fast_mlsirm/scoring/rag_evidence.py | 33 ++++++---------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/python/fast_mlsirm/scoring/rag_evidence.py b/python/fast_mlsirm/scoring/rag_evidence.py index 32a77b9c8..61bea51e5 100644 --- a/python/fast_mlsirm/scoring/rag_evidence.py +++ b/python/fast_mlsirm/scoring/rag_evidence.py @@ -101,36 +101,21 @@ def _replay_limitations( value: RAGEvidenceRegimeLimitations, ) -> RAGEvidenceRegimeLimitations: """Re-establish exact factory-derived state before manifest projection.""" - if type(value) is not RAGEvidenceRegimeLimitations: - raise assessment_error( - "invalid_rag_evidence_limitations", - "$", - "RAG evidence limitations must retain factory-derived state", - ) - regime = value.regime - limitation_codes = value.limitation_codes - if type(regime) is not RAGEvidenceRegime or type(limitation_codes) is not tuple: - raise assessment_error( - "invalid_rag_evidence_limitations", - "$", - "RAG evidence limitations must retain factory-derived state", - ) - if any(type(code) is not str for code in limitation_codes): - raise assessment_error( - "invalid_rag_evidence_limitations", - "$", - "RAG evidence limitations must retain factory-derived state", - ) - expected = _REGIME_LIMITATIONS[regime] - if limitation_codes != expected: + if ( + type(value) is not RAGEvidenceRegimeLimitations + or type(value.regime) is not RAGEvidenceRegime + or type(value.limitation_codes) is not tuple + or any(type(code) is not str for code in value.limitation_codes) + or value.limitation_codes != _REGIME_LIMITATIONS[value.regime] + ): raise assessment_error( "invalid_rag_evidence_limitations", "$", "RAG evidence limitations must retain factory-derived state", ) return RAGEvidenceRegimeLimitations( - regime=regime, - limitation_codes=expected, + regime=value.regime, + limitation_codes=_REGIME_LIMITATIONS[value.regime], _limitations_token=_RAG_EVIDENCE_LIMITATIONS_TOKEN, ) From 4ded2d01336b6ba726589543785142468ce883ce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 23:39:03 -0700 Subject: [PATCH 4/5] test(rag): document hostile replay fixture --- tests/test_scoring_rag_evidence_limitations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_scoring_rag_evidence_limitations.py b/tests/test_scoring_rag_evidence_limitations.py index 7ca8f9eb0..a855a0009 100644 --- a/tests/test_scoring_rag_evidence_limitations.py +++ b/tests/test_scoring_rag_evidence_limitations.py @@ -77,6 +77,7 @@ def test_limitations_replay_rejects_rebound_container_before_callbacks() -> None class HostileTuple(tuple): def __iter__(self): # type: ignore[override] + """Fail if package replay invokes caller-controlled iteration.""" callbacks.append("iter") raise AssertionError("hostile limitation iteration executed") From 9a10e212e73e079c264444f1d19ea60f82976210 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 23:39:17 -0700 Subject: [PATCH 5/5] docs(changelog): record RAG evidence replay integrity --- docs/changelog.d/1135-rag-evidence-replay-integrity.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog.d/1135-rag-evidence-replay-integrity.md diff --git a/docs/changelog.d/1135-rag-evidence-replay-integrity.md b/docs/changelog.d/1135-rag-evidence-replay-integrity.md new file mode 100644 index 000000000..f9f1a6bf2 --- /dev/null +++ b/docs/changelog.d/1135-rag-evidence-replay-integrity.md @@ -0,0 +1,5 @@ +# RAG evidence limitation replay integrity + +## Fixed + +- Replay factory-derived RAG evidence limitation records before manifest or fingerprint projection so post-construction mutation fails closed before caller callbacks can execute.