From 7cf9eb6c2937020b5e755b5ae0a6cc2380fc068d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 01:04:41 +0900 Subject: [PATCH 1/9] test(validation): specify judge category control boundary --- ...test_validation_category_control_safety.py | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tests/test_validation_category_control_safety.py diff --git a/tests/test_validation_category_control_safety.py b/tests/test_validation_category_control_safety.py new file mode 100644 index 000000000..8f26a5d86 --- /dev/null +++ b/tests/test_validation_category_control_safety.py @@ -0,0 +1,119 @@ +"""Security regressions for judge category-count validation.""" + +from __future__ import annotations + +import builtins +from types import SimpleNamespace + +import numpy as np +import pytest + +import fast_mlsirm +from fast_mlsirm import validation + + +class _HostileInt(int): + """Integer subclass whose conversion callback records execution.""" + + calls = 0 + + def __int__(self) -> int: + type(self).calls += 1 + return 2 + + +class _HostileNumpyInt(np.int64): + """NumPy integer subclass whose conversion callback records execution.""" + + calls = 0 + + def __int__(self) -> int: + type(self).calls += 1 + return 2 + + +class _IntegerProtocolProvider: + """Arbitrary integer protocol provider that must never be invoked.""" + + calls = 0 + + def __int__(self) -> int: + type(self).calls += 1 + return 2 + + +def _fake_core(calls: list[tuple[int, type[int]]]) -> SimpleNamespace: + """Return a deterministic Rust-boundary stand-in for marshalling tests.""" + + def validate_scoring(judge, human, k, **kwargs): + del judge, human, kwargs + calls.append((k, type(k))) + return { + "gates": [], + "exact_agreement": 1.0, + "adjacent_agreement": 1.0, + "pass": True, + } + + return SimpleNamespace(validate_scoring=validate_scoring) + + +@pytest.mark.parametrize( + "control_type", + [_HostileInt, _HostileNumpyInt, _IntegerProtocolProvider], +) +def test_validate_judge_rejects_executable_integer_controls_without_callbacks( + monkeypatch: pytest.MonkeyPatch, + control_type: type, +) -> None: + """Rejected category controls cannot run conversion code or Rust dispatch.""" + control_type.calls = 0 + rust_calls: list[tuple[int, type[int]]] = [] + monkeypatch.setattr(fast_mlsirm, "_core", _fake_core(rust_calls), raising=False) + + value = control_type(2) if control_type is not _IntegerProtocolProvider else control_type() + with pytest.raises(ValueError): + validation.validate_judge(np.array([0, 1]), np.array([0, 1]), k=value) + + assert control_type.calls == 0 + assert rust_calls == [] + + +def test_validate_judge_rejects_invalid_k_before_core_import( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An invalid exact category count fails before compiled-core discovery.""" + real_import = builtins.__import__ + core_import_calls = 0 + + def guarded_import(name, globals=None, locals=None, fromlist=(), level=0): + nonlocal core_import_calls + if "_core" in fromlist: + core_import_calls += 1 + raise AssertionError("compiled core discovered before k validation") + return real_import(name, globals, locals, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + + with pytest.raises(ValueError, match="must be >= 2"): + validation.validate_judge(np.array([0, 1]), np.array([0, 1]), k=1) + + assert core_import_calls == 0 + + +@pytest.mark.parametrize( + "k", + [np.int8(2), np.uint16(2), np.int32(2), np.uint64(2)], +) +def test_validate_judge_normalizes_genuine_numpy_category_counts( + monkeypatch: pytest.MonkeyPatch, + k: np.integer, +) -> None: + """Supported concrete NumPy integer scalars marshal as exact Python ints.""" + rust_calls: list[tuple[int, type[int]]] = [] + monkeypatch.setattr(fast_mlsirm, "_core", _fake_core(rust_calls), raising=False) + + verdict = validation.validate_judge(np.array([0, 1]), np.array([0, 1]), k=k) + + assert verdict.passed is True + assert rust_calls == [(2, int)] From db0cb5848d317d39f197933043e289e00cdf522b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 01:06:31 +0900 Subject: [PATCH 2/9] fix(validation): trust category controls before Rust discovery --- python/fast_mlsirm/validation.py | 58 +++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/python/fast_mlsirm/validation.py b/python/fast_mlsirm/validation.py index 2094da636..3d2de817a 100644 --- a/python/fast_mlsirm/validation.py +++ b/python/fast_mlsirm/validation.py @@ -16,6 +16,40 @@ import numpy as np +MAX_JUDGE_CATEGORIES = 1_000 +_TRUSTED_NUMPY_INTEGER_SCALAR_TYPES = frozenset( + { + np.dtype(code).type + for code in ("b", "B", "h", "H", "i", "I", "l", "L", "q", "Q", "p", "P") + } +) + + +def _trusted_judge_category_count(value: object) -> int: + """Return a trusted built-in category count without caller coercion callbacks. + + Only an exact built-in :class:`int` or an exact concrete NumPy integer scalar + identity is normalized. Python/NumPy subclasses, booleans, and arbitrary + integer-protocol providers are rejected before ``int`` can execute caller + code. The returned value is always an exact built-in integer suitable for + downstream NumPy validation and PyO3 marshalling. + """ + value_type = type(value) + if value_type is int: + normalized = value + elif value_type in _TRUSTED_NUMPY_INTEGER_SCALAR_TYPES: + normalized = int(value) + else: + raise ValueError("k (number of categories) must be an integer") + + if normalized < 2: + raise ValueError("k (number of categories) must be >= 2") + if normalized > MAX_JUDGE_CATEGORIES: + # k drives a dense k-by-k confusion matrix in the Rust core. + raise ValueError(f"k (number of categories) must be <= {MAX_JUDGE_CATEGORIES}") + return normalized + + def _validate_labels(a, name: str, *, k: int | None = None, n: int | None = None) -> np.ndarray: """Validate caller-supplied category labels before the uint32 conversion the Rust gate expects: reject non-1-D, wrong-length, non-finite, non-integer, @@ -130,27 +164,20 @@ def validate_judge( degradation criterion; ``subgroup`` labels each observation for the fairness SMD. """ - from . import _core # computation lives in the Rust core - active_policy = policy if policy is not None else ValidationPolicy() if not isinstance(active_policy, ValidationPolicy): raise TypeError("policy must be a ValidationPolicy") - MAX_JUDGE_CATEGORIES = 1_000 - if int(k) < 2: - raise ValueError("k (number of categories) must be >= 2") - if int(k) > MAX_JUDGE_CATEGORIES: - # k drives a dense k-by-k confusion matrix in the Rust core. - raise ValueError(f"k (number of categories) must be <= {MAX_JUDGE_CATEGORIES}") - judge_v = _validate_labels(judge, "judge", k=int(k)) - human_v = _validate_labels(human, "human", k=int(k), n=judge_v.shape[0]) + category_count = _trusted_judge_category_count(k) + judge_v = _validate_labels(judge, "judge", k=category_count) + human_v = _validate_labels(human, "human", k=category_count, n=judge_v.shape[0]) kwargs: dict[str, Any] = {} if human_human is not None: kwargs["human_a"] = _validate_labels( - human_human[0], "human_a", k=int(k), n=judge_v.shape[0] + human_human[0], "human_a", k=category_count, n=judge_v.shape[0] ) kwargs["human_b"] = _validate_labels( - human_human[1], "human_b", k=int(k), n=kwargs["human_a"].shape[0] + human_human[1], "human_b", k=category_count, n=kwargs["human_a"].shape[0] ) if subgroup is not None: sg = _validate_labels(subgroup, "subgroup", n=judge_v.shape[0]) @@ -159,10 +186,13 @@ def validate_judge( _uniq, sg_compact = np.unique(sg, return_inverse=True) kwargs["subgroup"] = sg_compact.astype(np.uint32) kwargs.update(active_policy.rust_kwargs()) + + from . import _core # computation lives in the Rust core + res = _core.validate_scoring( judge_v, human_v, - int(k), + category_count, **kwargs, ) gates = [dict(g) for g in res["gates"]] @@ -267,6 +297,8 @@ def fleiss_kappa( category_z=np.asarray(res["category_z"]), category_p=np.asarray(res["category_p"]), ) + + @dataclass class LightKappaResult: """Result of :func:`light_kappa`: mean pairwise unweighted Cohen's kappa From 7b6c93c56a0c30a63c52f22fe8b514615680fab1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 01:07:04 +0900 Subject: [PATCH 3/9] docs(changelog): record judge category control hardening --- docs/changelog.d/912-judge-category-control-safety.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/changelog.d/912-judge-category-control-safety.md diff --git a/docs/changelog.d/912-judge-category-control-safety.md b/docs/changelog.d/912-judge-category-control-safety.md new file mode 100644 index 000000000..b248d585f --- /dev/null +++ b/docs/changelog.d/912-judge-category-control-safety.md @@ -0,0 +1,7 @@ +# Judge category-count control hardening + +- Validate the public `validate_judge(..., k=...)` category count before compiled-core discovery. +- Accept exact built-in integers and genuine concrete NumPy integer scalars while rejecting booleans, subclasses, and arbitrary integer-conversion protocol providers without executing caller conversion callbacks. +- Marshal only a trusted built-in integer into the existing Rust-owned judge-validation computation; psychometric/fairness formulas, thresholds, and result schemas are unchanged. + +Closes #912. From 0ebcc06a619e8eb9a84b2b92d1b40898087fdcaa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 01:07:25 +0900 Subject: [PATCH 4/9] docs(doctoring): trace judge category control boundary --- .../judge_category_control_safety.md | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/doctoring/judge_category_control_safety.md diff --git a/docs/doctoring/judge_category_control_safety.md b/docs/doctoring/judge_category_control_safety.md new file mode 100644 index 000000000..f020c387e --- /dev/null +++ b/docs/doctoring/judge_category_control_safety.md @@ -0,0 +1,38 @@ +# Judge category-count control safety + +## Scope + +Issue #912 hardens only the Python validation/marshalling boundary for the public `validate_judge(..., k=...)` category-count control. The Rust-owned agreement, quadratic-weighted kappa, degradation, standardized-mean-difference/fairness, gate-decision arithmetic, governed thresholds, and result schema are unchanged. + +Protected `main` previously imported the compiled core before validating `k` and repeatedly invoked `int(k)`. Because Python integer subclasses and arbitrary integer-protocol objects may supply executable conversion methods, that order allowed caller-controlled conversion code to run while establishing a security- and fairness-relevant control. + +The bounded correction admits only an exact built-in `int` or an exact concrete NumPy integer scalar identity, normalizes a trusted NumPy scalar once, enforces the existing `2..=1000` domain, completes label/policy marshalling, and only then imports the Rust core. Booleans, Python/NumPy subclasses, and arbitrary conversion-protocol providers fail before conversion callbacks or Rust dispatch. + +## Verification contract + +The regression suite must prove all of the following on the exact PR head: + +- hostile Python integer subclasses execute zero `__int__` callbacks; +- hostile NumPy integer subclasses execute zero `__int__` callbacks; +- arbitrary integer-protocol providers execute zero callbacks; +- invalid exact category counts fail before compiled-core import; +- genuine concrete NumPy integer scalars remain compatible and arrive at the Rust boundary as an exact built-in integer; and +- no judge-validation numerical formula or policy threshold changes. + +The initial test commit `7cf9eb6c2937020b5e755b5ae0a6cc2380fc068d` records the RED contract. The implementation commit `db0cb5848d317d39f197933043e289e00cdf522b` supplies the bounded validation-order correction. Hosted exact-head evidence remains authoritative over these remembered identities and must be refetched before lifecycle or integration decisions. + +## Standards and research basis + +The security references below govern input-validation and development-process evidence; they are not psychometric validity authorities. OWASP ASVS 5.0.0 is the latest stable ASVS release as checked on 2026-08-16; OWASP separately labels its bleeding-edge build as preview-only. NIST SP 800-218 Rev. 1 / SSDF 1.2 remains an Initial Public Draft, so this work retains final SSDF 1.1 as the normative NIST baseline and records the draft only as a standards-watch item. + +### References (APA 7th) + +MITRE. (2026). *CWE-1287: Improper validation of specified type of input* (CWE Version 4.20). Common Weakness Enumeration. https://cwe.mitre.org/data/definitions/1287.html + +National Institute of Standards and Technology. (2022). *Secure software development framework (SSDF) version 1.1: Recommendations for mitigating the risk of software vulnerabilities* (NIST Special Publication 800-218). U.S. Department of Commerce. https://doi.org/10.6028/NIST.SP.800-218 + +National Institute of Standards and Technology. (2025). *Secure software development framework (SSDF) version 1.2: Recommendations for mitigating the risk of software vulnerabilities* (NIST Special Publication 800-218 Rev. 1, Initial Public Draft). U.S. Department of Commerce. https://csrc.nist.gov/pubs/sp/800/218/r1/ipd + +OWASP Foundation. (2025). *OWASP application security verification standard 5.0.0*. https://github.com/OWASP/ASVS/tree/v5.0.0_release + +Williamson, D. M., Xi, X., & Breyer, F. J. (2012). A framework for evaluation and use of automated scoring. *Educational Measurement: Issues and Practice, 31*(1), 2–13. https://doi.org/10.1111/j.1745-3992.2011.00223.x From 3979f064b3e32fe44892cab369f8ffc1e3af4d73 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 01:12:16 +0900 Subject: [PATCH 5/9] test(validation): reject hostile scalar metaclass callbacks --- ...test_validation_category_control_safety.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/test_validation_category_control_safety.py b/tests/test_validation_category_control_safety.py index 8f26a5d86..8f57c453e 100644 --- a/tests/test_validation_category_control_safety.py +++ b/tests/test_validation_category_control_safety.py @@ -32,6 +32,24 @@ def __int__(self) -> int: return 2 +class _HostileScalarMeta(type): + """Metaclass proving scalar-type admission cannot hash caller types.""" + + calls = 0 + + def __hash__(cls) -> int: + type(cls).calls += 1 + return type.__hash__(np.int64) + + def __eq__(cls, other: object) -> bool: + type(cls).calls += 1 + return type.__eq__(cls, other) + + +class _MetaclassHostileNumpyInt(np.int64, metaclass=_HostileScalarMeta): + """NumPy scalar subclass with caller-controlled type hash/equality hooks.""" + + class _IntegerProtocolProvider: """Arbitrary integer protocol provider that must never be invoked.""" @@ -79,6 +97,25 @@ def test_validate_judge_rejects_executable_integer_controls_without_callbacks( assert rust_calls == [] +def test_validate_judge_rejects_hostile_scalar_metaclass_without_callbacks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Scalar-type admission cannot dispatch caller metaclass hash/equality hooks.""" + _HostileScalarMeta.calls = 0 + rust_calls: list[tuple[int, type[int]]] = [] + monkeypatch.setattr(fast_mlsirm, "_core", _fake_core(rust_calls), raising=False) + + with pytest.raises(ValueError): + validation.validate_judge( + np.array([0, 1]), + np.array([0, 1]), + k=_MetaclassHostileNumpyInt(2), + ) + + assert _HostileScalarMeta.calls == 0 + assert rust_calls == [] + + def test_validate_judge_rejects_invalid_k_before_core_import( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -116,4 +153,4 @@ def test_validate_judge_normalizes_genuine_numpy_category_counts( verdict = validation.validate_judge(np.array([0, 1]), np.array([0, 1]), k=k) assert verdict.passed is True - assert rust_calls == [(2, int)] + assert rust_calls == [(2, int)] \ No newline at end of file From 54fe33b2dd9d2a287c04635f2acba7bfc94f10fa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 01:14:58 +0900 Subject: [PATCH 6/9] fix(validation): use identity-only scalar admission --- python/fast_mlsirm/validation.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/python/fast_mlsirm/validation.py b/python/fast_mlsirm/validation.py index 3d2de817a..38bf67641 100644 --- a/python/fast_mlsirm/validation.py +++ b/python/fast_mlsirm/validation.py @@ -17,14 +17,22 @@ MAX_JUDGE_CATEGORIES = 1_000 -_TRUSTED_NUMPY_INTEGER_SCALAR_TYPES = frozenset( - { - np.dtype(code).type - for code in ("b", "B", "h", "H", "i", "I", "l", "L", "q", "Q", "p", "P") - } +_TRUSTED_NUMPY_INTEGER_SCALAR_TYPES = tuple( + np.dtype(code).type + for code in ("b", "B", "h", "H", "i", "I", "l", "L", "q", "Q", "p", "P") ) +def _is_exact_numpy_integer_scalar_type(value_type: type) -> bool: + """Return whether ``value_type`` is a package-trusted NumPy integer type. + + Identity comparisons deliberately avoid hashing or equality on a + caller-controlled metaclass. This keeps type admission inert even for a + NumPy scalar subclass that overrides metaclass ``__hash__`` or ``__eq__``. + """ + return any(value_type is trusted_type for trusted_type in _TRUSTED_NUMPY_INTEGER_SCALAR_TYPES) + + def _trusted_judge_category_count(value: object) -> int: """Return a trusted built-in category count without caller coercion callbacks. @@ -37,7 +45,7 @@ def _trusted_judge_category_count(value: object) -> int: value_type = type(value) if value_type is int: normalized = value - elif value_type in _TRUSTED_NUMPY_INTEGER_SCALAR_TYPES: + elif _is_exact_numpy_integer_scalar_type(value_type): normalized = int(value) else: raise ValueError("k (number of categories) must be an integer") From ee16ffa047aa5c24ff7011407a848cd57af59369 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 01:15:34 +0900 Subject: [PATCH 7/9] docs(doctoring): record identity-only scalar admission --- docs/doctoring/judge_category_control_safety.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/judge_category_control_safety.md b/docs/doctoring/judge_category_control_safety.md index f020c387e..bfed21c7d 100644 --- a/docs/doctoring/judge_category_control_safety.md +++ b/docs/doctoring/judge_category_control_safety.md @@ -6,7 +6,7 @@ Issue #912 hardens only the Python validation/marshalling boundary for the publi Protected `main` previously imported the compiled core before validating `k` and repeatedly invoked `int(k)`. Because Python integer subclasses and arbitrary integer-protocol objects may supply executable conversion methods, that order allowed caller-controlled conversion code to run while establishing a security- and fairness-relevant control. -The bounded correction admits only an exact built-in `int` or an exact concrete NumPy integer scalar identity, normalizes a trusted NumPy scalar once, enforces the existing `2..=1000` domain, completes label/policy marshalling, and only then imports the Rust core. Booleans, Python/NumPy subclasses, and arbitrary conversion-protocol providers fail before conversion callbacks or Rust dispatch. +The bounded correction admits only an exact built-in `int` or an exact concrete NumPy integer scalar identity, normalizes a trusted NumPy scalar once, enforces the existing `2..=1000` domain, completes label/policy marshalling, and only then imports the Rust core. Booleans, Python/NumPy subclasses, and arbitrary conversion-protocol providers fail before conversion callbacks or Rust dispatch. NumPy scalar-type admission uses identity comparisons rather than set membership so a caller-controlled scalar metaclass cannot inject `__hash__` or `__eq__` execution into the trust decision. ## Verification contract @@ -15,11 +15,12 @@ The regression suite must prove all of the following on the exact PR head: - hostile Python integer subclasses execute zero `__int__` callbacks; - hostile NumPy integer subclasses execute zero `__int__` callbacks; - arbitrary integer-protocol providers execute zero callbacks; +- caller-controlled NumPy scalar metaclasses execute zero hashing/equality callbacks during type admission; - invalid exact category counts fail before compiled-core import; - genuine concrete NumPy integer scalars remain compatible and arrive at the Rust boundary as an exact built-in integer; and - no judge-validation numerical formula or policy threshold changes. -The initial test commit `7cf9eb6c2937020b5e755b5ae0a6cc2380fc068d` records the RED contract. The implementation commit `db0cb5848d317d39f197933043e289e00cdf522b` supplies the bounded validation-order correction. Hosted exact-head evidence remains authoritative over these remembered identities and must be refetched before lifecycle or integration decisions. +The initial test commit `7cf9eb6c2937020b5e755b5ae0a6cc2380fc068d` records the conversion/native-discovery RED contract, and `db0cb5848d317d39f197933043e289e00cdf522b` supplies its first bounded GREEN. A second RED at `3979f064b3e32fe44892cab369f8ffc1e3af4d73` demonstrates that hashed type-container membership would still execute caller-controlled metaclass hooks; `54fe33b2dd9d2a287c04635f2acba7bfc94f10fa` replaces that admission with identity-only comparisons. Hosted exact-head evidence remains authoritative over these remembered identities and must be refetched before lifecycle or integration decisions. ## Standards and research basis From 8da4990e8db4940ababe6ef033742739eea4ae57 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 02:01:12 +0900 Subject: [PATCH 8/9] docs(changelog): classify judge control hardening --- docs/changelog.d/912-judge-category-control-safety.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.d/912-judge-category-control-safety.md b/docs/changelog.d/912-judge-category-control-safety.md index b248d585f..8c1590877 100644 --- a/docs/changelog.d/912-judge-category-control-safety.md +++ b/docs/changelog.d/912-judge-category-control-safety.md @@ -1,5 +1,7 @@ # Judge category-count control hardening +## Security + - Validate the public `validate_judge(..., k=...)` category count before compiled-core discovery. - Accept exact built-in integers and genuine concrete NumPy integer scalars while rejecting booleans, subclasses, and arbitrary integer-conversion protocol providers without executing caller conversion callbacks. - Marshal only a trusted built-in integer into the existing Rust-owned judge-validation computation; psychometric/fairness formulas, thresholds, and result schemas are unchanged. From f1c232c0bb612bd66fb6a9bb633b6c571e170c93 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 03:30:10 +0900 Subject: [PATCH 9/9] test(validation): avoid builtin shadowing in import guard --- tests/test_validation_category_control_safety.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_validation_category_control_safety.py b/tests/test_validation_category_control_safety.py index 8f57c453e..04d920834 100644 --- a/tests/test_validation_category_control_safety.py +++ b/tests/test_validation_category_control_safety.py @@ -123,12 +123,12 @@ def test_validate_judge_rejects_invalid_k_before_core_import( real_import = builtins.__import__ core_import_calls = 0 - def guarded_import(name, globals=None, locals=None, fromlist=(), level=0): + def guarded_import(name, globals_=None, locals_=None, fromlist=(), level=0): nonlocal core_import_calls if "_core" in fromlist: core_import_calls += 1 raise AssertionError("compiled core discovered before k validation") - return real_import(name, globals, locals, fromlist, level) + return real_import(name, globals_, locals_, fromlist, level) monkeypatch.setattr(builtins, "__import__", guarded_import)