fix(rocke): never pass a compiler option the loaded compiler does not know

dnn-providers/hip-kernel-provider/ROCKE_ENGINE_test_categories_external.yaml

@@ -20,9 +20,11 @@
 # Most tests carry no Full*/Standard*/Comprehensive* variant, so every tier
 # runs the same coverage. The arch-domain reproducibility gate is intentionally
 # standard-tier: it probes the external LLVM toolchain across the full
-# intrinsic/architecture matrix and does not belong in quick or FFM. It is
-# registered in both build-tree and installed-artifact CTest, so developers and
-# provider CI execute the same logical gate against their respective package data.
+# intrinsic/architecture matrix and does not belong in quick or FFM. The
+# compiler-option reproducibility gate joins it: it re-measures which -mllvm
+# options the loaded compiler accepts. Both are registered in build-tree and
+# installed-artifact CTest, so developers and provider CI execute the same
+# logical gates against their respective package data.
 #
 # The pinned GPU reference entries are regex patterns, unlike the other
 # entries here: rocke/platform/CMakeLists.txt registers
@@ -194,6 +196,7 @@ test_categories:
     test_patterns:
       - "rocke_arch_domain_reproducibility"
       - "rocke_library_attention_cache_key_sweep_pytest"
+      - "rocke_compiler_options_reproducibility"
     labels:
       - "host"
       - "toolchain"

dnn-providers/hip-kernel-provider/rocke/library/tests/_compiler_test_utils.py

@@ -0,0 +1,64 @@
+# Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
+# SPDX-License-Identifier: MIT
+
+"""The shared skip policy for library tests that compile a kernel through COMGR.
+
+Such a test depends on the host: COMGR and the code-object tools must be
+installed, and the loaded compiler must accept every ``-mllvm`` option the
+kernel needs. A test that uses this policy skips when one of those is missing,
+because that is a fact about the host, not a defect in the kernel. A run that
+sets ``ROCKE_REQUIRE_COMPILER_TOOLS=1`` (the CI lane that supplies the tools)
+fails instead: a gate that skips everywhere looks green and checks nothing.
+
+* :func:`require_tools_or_skip`: skip (or fail under the variable) for a
+  stated reason.
+* :func:`skip_where_compiler_refuses_option`: apply that policy when the body
+  raises ``CompilerOptionNotAccepted``.
+
+A skip is raised as ``unittest.SkipTest``, which pytest test functions and a
+``unittest`` ``setUpClass`` both report as a skip.
+"""
+
+from __future__ import annotations
+
+import os
+import unittest
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import NoReturn
+
+from rocke.runtime import CompilerOptionNotAccepted
+
+__all__ = [
+    "REQUIRE_COMPILER_TOOLS_ENV",
+    "require_tools_or_skip",
+    "skip_where_compiler_refuses_option",
+]
+
+# platform/CMakeLists.txt sets it to "1" only for the installed ctest entries
+# whose tests consume it, where ROCM_PATH supplies the tools (Linux); see
+# TESTING.md section 5.1.4.
+REQUIRE_COMPILER_TOOLS_ENV = "ROCKE_REQUIRE_COMPILER_TOOLS"
+
+
+def require_tools_or_skip(reason: str) -> NoReturn:
+    """Skip the running test for ``reason``, or fail it with ``AssertionError``
+    when ``ROCKE_REQUIRE_COMPILER_TOOLS=1`` says the tools must be there."""
+    if os.environ.get(REQUIRE_COMPILER_TOOLS_ENV) == "1":
+        raise AssertionError(f"{REQUIRE_COMPILER_TOOLS_ENV}=1 but {reason}")
+    raise unittest.SkipTest(reason)
+
+
+@contextmanager
+def skip_where_compiler_refuses_option() -> Iterator[None]:
+    """Apply :func:`require_tools_or_skip` if the body raises
+    ``CompilerOptionNotAccepted``.
+
+    That error means the loaded compiler does not accept a ``-mllvm`` option
+    the kernel needs (see ``rocke.runtime.compiler_options``). Every other
+    error propagates unchanged.
+    """
+    try:
+        yield
+    except CompilerOptionNotAccepted as exc:
+        require_tools_or_skip(f"loaded compiler cannot build this kernel: {exc}")

dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_bwd_agpr_probe.py

@@ -49,6 +49,7 @@
     mfma_result_wait_states,
 )
 from ._attention_bwd_test_utils import device_arch
+from ._compiler_test_utils import skip_where_compiler_refuses_option
 
 ARCH = os.environ.get("ROCKE_BWD_ARCH") or os.environ.get("ROCKE_TEST_ARCH")
 
@@ -109,7 +110,10 @@ def _compile_isa(kernel, arch):
     try:
         from rocke.helpers.compile import compile_kernel
 
-        art = compile_kernel(kernel, arch=arch, capture_ir_text=False, backend="python")
+        with skip_where_compiler_refuses_option():
+            art = compile_kernel(
+                kernel, arch=arch, capture_ir_text=False, backend="python"
+            )
     except (ImportError, OSError) as exc:  # pragma: no cover - environment-dependent
         pytest.skip(f"comgr unavailable: {exc}")
     return disassemble(art.hsaco, arch=arch)

dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_bwd_compile.py

@@ -18,6 +18,8 @@
 from benchmarks.common import attention_bwd_compile as bc
 from kernels.common.attention_bwd import AttnBwdSpec, validate_attn_bwd_spec
 
+from ._compiler_test_utils import skip_where_compiler_refuses_option
+
 TC = bc.Toolchain("llvm22", "/x/libamd_comgr.so.3", "7.2", "abcdef012345")
 
 
@@ -297,11 +299,27 @@ def _can_compile():
     return tc.comgr_path is not None
 
 
+def _skip_unless_compiler_accepts(arch, *specs):
+    """Skip (or fail, under ``ROCKE_REQUIRE_COMPILER_TOOLS=1``) where the
+    loaded compiler refuses a codegen option these main kernels need (see
+    ``rocke.runtime.compiler_options``): these tests are about the backward
+    compile, not about that compiler."""
+    from rocke.helpers.compile import accepted_comgr_options
+
+    from kernels.common.attention_bwd import build_attn_bwd_main
+
+    for spec in specs:
+        kernel = build_attn_bwd_main(spec, arch=arch)
+        with skip_where_compiler_refuses_option():
+            accepted_comgr_options(kernel)
+
+
 @pytest.mark.skipif(not _can_compile(), reason="no comgr library found")
 def test_real_main_kernel_compiles_and_is_cached(tmp_path):
     tc = bc.toolchain_identity()
     c = bc.CompileCache(tmp_path, toolchain=tc, source_hash=bc.backward_source_hash())
     s = _spec()
+    _skip_unless_compiler_accepts("gfx942", s)
     res = bc.compile_batch([("main", s)], arch="gfx942", cache=c, mode="serial")
     assert res[0].ok, res[0].detail
     assert res[0].entry.hsaco[:4] == b"\x7fELF"
@@ -323,6 +341,7 @@ def test_thread_pool_compiles_the_serial_code_objects(tmp_path):
         ("main", _spec(waves=4, block_n=64, block_m=32, warp_grid_g4=(2, 2))),
         ("main", _spec(head_size=128)),
     ]
+    _skip_unless_compiler_accepts("gfx942", *(s for _, s in specs))
     a = bc.compile_batch(
         specs, arch="gfx942", mode="serial",
         cache=bc.CompileCache(tmp_path / "s", toolchain=tc, source_hash=src),
@@ -367,11 +386,13 @@ def lowered(s):
         return lower_kernel_to_llvm(k, arch=arch), _comgr_options_for_kernel(k)
 
     probe = spec("atomic", stage_order="g4g1g3")
+    other = spec("direct", head_pack=True, stage_order="g3g4g1")
+    _skip_unless_compiler_accepts(arch, probe, other)
     ll, opts = lowered(probe)
     alone, _ = build_hsaco_from_llvm_ir(ll, isa=isa, options=opts, fresh_process=True)
     # in-process iglp_opt(1) compiles move the strategy's counts here: another
     # kernel, then the probe itself (the history of the observed failure)
-    for s in (spec("direct", head_pack=True, stage_order="g3g4g1"), probe):
+    for s in (other, probe):
         ll, opts = lowered(s)
         build_hsaco_from_llvm_ir(ll, isa=isa, options=opts)
     _name, blob, _secs = bc.compile_one("main", probe, arch)

dnn-providers/hip-kernel-provider/rocke/library/tests/test_compiler_test_utils.py

@@ -0,0 +1,99 @@
+# Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
+# SPDX-License-Identifier: MIT
+
+"""The skip policy every compile test shares (``_compiler_test_utils``).
+
+CPU only. Each error is a real one from the committed option table: a
+compiler whose LLVM major the table records as refusing the option
+(``CompilerOptionNotAccepted``), or one the table has no answer for (a plain
+``ValueError``). These tests pin that a refusal skips, that it fails when
+``ROCKE_REQUIRE_COMPILER_TOOLS=1`` requires the tools, and that no other error
+is turned into a skip.
+"""
+
+from __future__ import annotations
+
+import unittest
+
+import pytest
+from rocke.runtime.comgr import CompilerInfo
+from rocke.runtime.compiler_options import measured_answers, require_accepted
+
+from ._compiler_test_utils import (
+    REQUIRE_COMPILER_TOOLS_ENV,
+    skip_where_compiler_refuses_option,
+)
+
+OPTION = "-amdgpu-mfma-vgpr-form"
+# How the refusal starts, and how the policy introduces it.
+REFUSAL = f"kernel 'probe' needs the compiler option '-mllvm {OPTION}'"
+SKIP_REASON = f"loaded compiler cannot build this kernel: {REFUSAL}"
+
+
+def _compiler(llvm_major: int) -> CompilerInfo:
+    """A compiler identity as ``loaded_compiler_info`` reports one."""
+    return CompilerInfo(
+        llvm_version=(llvm_major, 0, 0),
+        source="LLVMGetVersion",
+        requested_comgr="libamd_comgr.so",
+        comgr_path=None,
+        query_library_path=None,
+    )
+
+
+def _compile_on(llvm_major: int) -> None:
+    """Ask the committed table whether LLVM ``llvm_major`` accepts ``OPTION``,
+    as ``compile_kernel`` does before compiling a kernel that needs it."""
+    require_accepted(
+        ["-O3", "-mllvm", OPTION],
+        compiler=_compiler(llvm_major),
+        needed_by="kernel 'probe'",
+    )
+
+
+def _refusing_major() -> int:
+    """An LLVM major the table records as refusing ``OPTION``."""
+    return next(
+        major
+        for major, answer in measured_answers()[OPTION].items()
+        if not answer.accepted
+    )
+
+
+def _unmeasured_major() -> int:
+    """An LLVM major newer than every one the table has an answer for."""
+    return 1 + max(measured_answers()[OPTION])
+
+
+def test_refused_option_skips(monkeypatch):
+    monkeypatch.delenv(REQUIRE_COMPILER_TOOLS_ENV, raising=False)
+    with (
+        pytest.raises(unittest.SkipTest, match=f"^{SKIP_REASON}"),
+        skip_where_compiler_refuses_option(),
+    ):
+        _compile_on(_refusing_major())
+
+
+def test_refused_option_fails_where_tools_are_required(monkeypatch):
+    monkeypatch.setenv(REQUIRE_COMPILER_TOOLS_ENV, "1")
+    required = f"^{REQUIRE_COMPILER_TOOLS_ENV}=1 but {SKIP_REASON}"
+    with (
+        pytest.raises(AssertionError, match=required),
+        skip_where_compiler_refuses_option(),
+    ):
+        _compile_on(_refusing_major())
+
+
+@pytest.mark.parametrize("required", ["0", "1"])
+def test_other_errors_propagate(monkeypatch, required):
+    # A compiler the table has no answer for raises a plain ValueError (the
+    # base class of the refusal): that is a missing measurement, not a skip.
+    monkeypatch.setenv(REQUIRE_COMPILER_TOOLS_ENV, required)
+    with (
+        pytest.raises(
+            ValueError, match="which has not been measured on LLVM"
+        ) as caught,
+        skip_where_compiler_refuses_option(),
+    ):
+        _compile_on(_unmeasured_major())
+    assert type(caught.value) is ValueError

dnn-providers/hip-kernel-provider/rocke/library/tests/test_mla_prefill_gfx942.py

@@ -16,7 +16,6 @@
 
 from __future__ import annotations
 
-import os
 import tempfile
 import unittest
 from pathlib import Path
@@ -35,21 +34,16 @@
 )
 from rocke.core.lower_llvm import lower_kernel_to_llvm
 
+from ._compiler_test_utils import (
+    require_tools_or_skip,
+    skip_where_compiler_refuses_option,
+)
+
 
 def _spec(**kw) -> MlaPrefillSpec:
     return MlaPrefillSpec(num_heads=kw.pop("num_heads", 16), **kw)
 
 
-def _require_tools_or_skip(reason: str) -> None:
-    """Skip a resource guard whose compiler tools are missing, unless the run
-    requires them (``ROCKE_REQUIRE_COMPILER_TOOLS=1``, set by the CI lane that
-    supplies them): a gate that skips everywhere looks green and checks
-    nothing."""
-    if os.environ.get("ROCKE_REQUIRE_COMPILER_TOOLS") == "1":
-        raise AssertionError(f"ROCKE_REQUIRE_COMPILER_TOOLS=1 but {reason}")
-    raise unittest.SkipTest(reason)
-
-
 def _assert_no_private_memory(case: unittest.TestCase, ir: str) -> None:
     case.assertNotIn("alloca", ir)
     case.assertNotIn("addrspace(5)", ir)
@@ -442,47 +436,34 @@ def setUpClass(cls):
         try:
             from rocke.analysis.isa import analyze_hsaco
             from rocke.helpers.compile import compile_kernel
+            from rocke.runtime.comgr import loaded_compiler_info
         except ImportError as exc:  # pragma: no cover - environment-dependent
-            _require_tools_or_skip(f"comgr/resource tools unavailable: {exc}")
-        # The forward kernel compiles with ``-amdgpu-mfma-vgpr-form``, which an
-        # LLVM 20 comgr (ROCm 7.0/7.1) does not know: its option parser exits
-        # the whole process instead of raising. That happens when an earlier
-        # test in the session already loaded the system comgr ahead of torch's.
-        try:
-            from rocke.runtime.comgr import _require_compiler_info
-
-            llvm_major = int(_require_compiler_info().llvm_version[0])
-        except (ImportError, OSError, RuntimeError) as exc:  # pragma: no cover
-            _require_tools_or_skip(f"comgr unavailable: {exc}")
-        if llvm_major < 21:
-            raise unittest.SkipTest(
-                f"loaded comgr is LLVM {llvm_major}; the forward kernel's "
-                "-amdgpu-mfma-vgpr-form option needs LLVM 21+"
-            )
-
+            require_tools_or_skip(f"comgr/resource tools unavailable: {exc}")
+        # Probe before compiling: compile_kernel reports a missing COMGR as a
+        # ComgrError, the same type as a real compile failure.
+        compiler = loaded_compiler_info()
+        if compiler is None or compiler.llvm_version is None:
+            found = "none loaded" if compiler is None else compiler.describe()
+            require_tools_or_skip(f"no queryable comgr ({found})")
         cls.spec = cls.make_spec()
         try:
-            art = compile_kernel(
-                build_mla_prefill_fwd(cls.spec, arch="gfx942"),
-                arch="gfx942",
-                capture_ir_text=False,
-            )
+            # The forward kernel needs -amdgpu-mfma-vgpr-form; skip where the
+            # loaded compiler refuses it (see rocke.runtime.compiler_options).
+            with skip_where_compiler_refuses_option():
+                art = compile_kernel(
+                    build_mla_prefill_fwd(cls.spec, arch="gfx942"),
+                    arch="gfx942",
+                    capture_ir_text=False,
+                )
         except ImportError as exc:  # pragma: no cover - environment-dependent
-            _require_tools_or_skip(f"comgr toolchain unavailable: {exc}")
-        except RuntimeError as exc:
-            # A forced ROCKE_LLVM_FLAVOR that the loaded comgr cannot take is an
-            # environment mismatch, not a kernel defect. Any other compile
-            # failure is real.
-            if "vintage mismatch" not in str(exc):
-                raise
-            raise unittest.SkipTest(f"comgr cannot take this IR flavor: {exc}")
+            require_tools_or_skip(f"comgr toolchain unavailable: {exc}")
         with tempfile.TemporaryDirectory() as tmp:
             path = Path(tmp) / "mla_prefill_fwd.hsaco"
             path.write_bytes(bytes(art.hsaco))
             try:
                 cls.res = analyze_hsaco(path).resources
             except (FileNotFoundError, RuntimeError) as exc:  # pragma: no cover
-                _require_tools_or_skip(f"HSACO introspection unavailable: {exc}")
+                require_tools_or_skip(f"HSACO introspection unavailable: {exc}")
 
     def test_lds_fits_two_workgroups(self):
         self.assertIsNotNone(self.res.lds_bytes)
@@ -614,7 +595,7 @@ def setUpClass(cls):
             from rocke.core.lower_llvm import _is_modern_flavor, _resolve_llvm_flavor
             from rocke.helpers.compile import compile_kernel
         except ImportError as exc:  # pragma: no cover - environment-dependent
-            _require_tools_or_skip(f"comgr/resource tools unavailable: {exc}")
+            require_tools_or_skip(f"comgr/resource tools unavailable: {exc}")
         if not _is_modern_flavor(_resolve_llvm_flavor()):
             raise unittest.SkipTest("agpr_acc needs an llvm22 / llvm23 comgr")
         try:
@@ -624,14 +605,14 @@ def setUpClass(cls):
                 capture_ir_text=False,
             )
         except (ImportError, RuntimeError) as exc:  # pragma: no cover
-            _require_tools_or_skip(f"comgr toolchain unavailable: {exc}")
+            require_tools_or_skip(f"comgr toolchain unavailable: {exc}")
         with tempfile.TemporaryDirectory() as tmp:
             path = Path(tmp) / "mla_prefill_fwd_aa.hsaco"
             path.write_bytes(bytes(art.hsaco))
             try:
                 an = analyze_hsaco(path, keep_text=True)
             except (FileNotFoundError, RuntimeError) as exc:  # pragma: no cover
-                _require_tools_or_skip(f"HSACO introspection unavailable: {exc}")
+                require_tools_or_skip(f"HSACO introspection unavailable: {exc}")
         cls.res = an.resources
         cls.mfma = [
             ln.split("//")[0].strip()