From a2fea5b44f88d42d40026d70ac21d3fb017e3d92 Mon Sep 17 00:00:00 2001 From: Chris Millette Date: Sat, 31 Oct 2026 17:17:15 -0400 Subject: [PATCH] test(rocke): check the representative-IR golden one LLVM flavor per test The representative-IR golden test lowered and hashed every case at every LLVM flavor inside one test, which came close to the per-test timeout of the installed pytest lane. It is now one test per flavor, parametrized over GOLDEN_FLAVORS (the qualified-lane list itself), plus one test that checks the golden's flavor documents and aliases without lowering anything. Together they check every case at every flavor against the same golden shards, and a drift message names its flavor. ir_golden gains check_golden_flavors, the flavor-set part of an all-flavor check_golden call, so a caller that checks flavors one at a time keeps the same coverage. check_golden's results are unchanged. BUILDING.md drops its note that the golden test reports conv drift without the C++ engine: the Python lowerer matches the goldens at every flavor, so the note no longer holds. Co-Authored-By: Claude --- .../hip-kernel-provider/rocke/BUILDING.md | 11 +- .../platform/python/rocke/core/ir_golden.py | 108 ++++++++++++------ .../platform/tests/core/test_ir_golden.py | 24 ++++ .../platform/tests/test_rocke_ci_static.py | 52 +++++---- 4 files changed, 128 insertions(+), 67 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/BUILDING.md b/dnn-providers/hip-kernel-provider/rocke/BUILDING.md index 0618df3048a5..50425b100b7a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/BUILDING.md +++ b/dnn-providers/hip-kernel-provider/rocke/BUILDING.md @@ -119,16 +119,7 @@ guard is designed to catch (regenerate `_DATALAYOUT_*` in `core/lower_llvm.py` for the local toolchain) — it is independent of the platform/library layout and reproduces identically on `develop`. -One more layout-independent residual appears **only when the C++ engine is not -built** (a pure-`pip`/Python-fallback checkout, as above): -`test_rocke_ci_static.py::TestIrParityCoverage::test_ir_cases_match_golden_sha256` -reports IR drift for 6 `conv/*` cases. The golden hashes track the C++ engine's -output; #8841 (cshuffle) changed the Python lowerer's conv epilogue IR, so the -Python-fallback hashes differ until the goldens are regenerated. Like the -datalayout guard it reproduces identically on `develop` and is unrelated to the -platform/library split — build the C++ engine (`platform/cpp/`) to run this test -against its intended baseline. Everything else is green; GPU-only tests skip -without a device. +Everything else is green; GPU-only tests skip without a device. ## Data / asset paths diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_golden.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_golden.py index 768efb6fad68..0cf4bac16145 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_golden.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_golden.py @@ -44,6 +44,7 @@ "GOLDEN_FLAVORS", "GOLDEN_PROFILE_FLAVORS", "check_golden", + "check_golden_flavors", "compare", "golden_shard", "golden_subdocument", @@ -134,6 +135,63 @@ def compare(base: Mapping[str, Any], cur: Mapping[str, Any]) -> list[str]: return errors +def _flavor_set_errors(doc: Mapping[str, Any]) -> list[str]: + """Check which flavor documents and aliases a loaded golden holds.""" + have = doc.get("flavors", {}) + aliases = doc.get("aliases", {}) + errors: list[str] = [] + if aliases != GOLDEN_ALIASES: + errors.append( + f"golden aliases {aliases} do not match emission-profile aliases " + f"{GOLDEN_ALIASES}; re-bless it" + ) + if set(have) != set(GOLDEN_PROFILE_FLAVORS): + errors.append( + f"golden profile documents {sorted(have)} do not match " + f"{list(GOLDEN_PROFILE_FLAVORS)}; re-bless it" + ) + overlap = set(have) & set(aliases) + if overlap: + errors.append( + f"golden lanes are both direct and aliased: {sorted(overlap)}; " + "re-bless it" + ) + if errors: + # Report the documents themselves first; the checks below only add + # detail once those are right. + return errors + represented = set(have) | set(aliases) + for fl in sorted(represented - set(GOLDEN_FLAVORS)): + errors.append( + f"golden has stale flavor {fl!r} not in LLVM_FLAVORS; re-bless it" + ) + for fl, source in sorted(aliases.items()): + if source not in have: + errors.append( + f"golden alias {fl!r} names missing flavor {source!r}; re-bless it" + ) + return errors + + +def check_golden_flavors(golden_path: Path) -> list[str]: + """Check that a golden holds exactly the flavor documents and aliases the + current flavor set needs, without lowering anything. + + This is the part of :func:`check_golden` (with no ``flavor``) that looks + only at the golden itself: a document for each emission profile, the + expected aliases, and nothing for a flavor no longer in + :data:`GOLDEN_FLAVORS`. A caller that checks the flavors one at a time, + ``check_golden(path, run, flavor)`` for each entry of + :data:`GOLDEN_FLAVORS`, runs this once as well to get the same coverage as + one all-flavor call. Empty list == OK. + """ + try: + doc = load_golden(golden_path) + except ValueError as e: + return [str(e)] + return _flavor_set_errors(doc) + + def check_golden( golden_path: Path, run: Callable[[str], Mapping[str, Any]], @@ -142,39 +200,26 @@ def check_golden( """Compare fresh runs against a golden's flavor sub-documents. ``run(flavor)`` lowers every case at ``flavor`` and returns a sub-document - in the golden's shape. With no ``flavor``, every entry of - :data:`GOLDEN_FLAVORS` is checked, and a sub-document for a flavor no - longer in that tuple is reported as stale. Drift strings are prefixed - with the flavor when more than one is checked. ``golden_path`` may be a - file or a shard directory (see :func:`load_golden`). Empty list == OK. + in the golden's shape. With no ``flavor``, the golden's flavor documents + are checked first (see :func:`check_golden_flavors`) and then every entry + of :data:`GOLDEN_FLAVORS` is compared. With a ``flavor``, only that + flavor is compared. Drift strings are prefixed with the flavor when more + than one is checked. ``golden_path`` may be a file or a shard directory + (see :func:`load_golden`). Empty list == OK. """ try: doc = load_golden(golden_path) except ValueError as e: return [str(e)] - have = doc.get("flavors", {}) - aliases = doc.get("aliases", {}) - wanted = [flavor] if flavor else list(GOLDEN_FLAVORS) - errors: list[str] = [] - if not flavor: - if aliases != GOLDEN_ALIASES: - errors.append( - f"golden aliases {aliases} do not match emission-profile aliases " - f"{GOLDEN_ALIASES}; re-bless it" - ) - if set(have) != set(GOLDEN_PROFILE_FLAVORS): - errors.append( - f"golden profile documents {sorted(have)} do not match " - f"{list(GOLDEN_PROFILE_FLAVORS)}; re-bless it" - ) - overlap = set(have) & set(aliases) - if overlap: - errors.append( - f"golden lanes are both direct and aliased: {sorted(overlap)}; " - "re-bless it" - ) + if flavor: + wanted = [flavor] + else: + errors = _flavor_set_errors(doc) if errors: return errors + wanted = list(GOLDEN_FLAVORS) + have = doc.get("flavors", {}) + errors = [] for fl in wanted: base = golden_subdocument(doc, fl) if base is None: @@ -185,15 +230,4 @@ def check_golden( continue prefix = "" if len(wanted) == 1 else f"[{fl}] " errors.extend(prefix + e for e in compare(base, run(fl))) - if not flavor: - represented = set(have) | set(aliases) - for fl in sorted(represented - set(GOLDEN_FLAVORS)): - errors.append( - f"golden has stale flavor {fl!r} not in LLVM_FLAVORS; re-bless it" - ) - for fl, source in sorted(aliases.items()): - if source not in have: - errors.append( - f"golden alias {fl!r} names missing flavor {source!r}; re-bless it" - ) return errors diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_ir_golden.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_ir_golden.py index 521b78704cb1..e69c82cd960b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_ir_golden.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_ir_golden.py @@ -21,6 +21,7 @@ GOLDEN_FLAVORS, GOLDEN_PROFILE_FLAVORS, check_golden, + check_golden_flavors, compare, golden_shard, load_golden, @@ -117,6 +118,29 @@ def test_direct_and_aliased_lane_is_rejected(self): errors = check_golden(self.path, lambda fl: _sub()) self.assertTrue(any("both direct and aliased" in error for error in errors)) + def test_flavor_set_checked_without_lowering(self): + self._write(GOLDEN_PROFILE_FLAVORS, GOLDEN_ALIASES) + self.assertEqual(check_golden_flavors(self.path), []) + self._write(GOLDEN_PROFILE_FLAVORS + ("llvm19",), GOLDEN_ALIASES) + errors = check_golden_flavors(self.path) + self.assertEqual(len(errors), 1) + self.assertIn("profile documents", errors[0]) + + def test_per_flavor_checks_match_one_all_flavor_check(self): + # A caller that checks the flavors one at a time gets the same errors + # as one all-flavor call, as long as it also checks the flavor set. + self._write(GOLDEN_PROFILE_FLAVORS, GOLDEN_ALIASES) + bad = GOLDEN_FLAVORS[-1] + + def run(fl): + return _sub("b" if fl == bad else "a") + + split = check_golden_flavors(self.path) + for fl in GOLDEN_FLAVORS: + split.extend(f"[{fl}] {e}" for e in check_golden(self.path, run, fl)) + self.assertEqual(split, check_golden(self.path, run)) + self.assertEqual(split, [f"[{bad}] k: a -> b"]) + def test_compare_case_sets_and_failures(self): base = { "cases": {"k": {"sha256": "a"}, "gone": {"sha256": "x"}}, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/test_rocke_ci_static.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/test_rocke_ci_static.py index f0c56d838892..3def8ff5586d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/test_rocke_ci_static.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/test_rocke_ci_static.py @@ -9,7 +9,7 @@ Run: PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=python:python/test \ - python tests/test_rocke_ci_static.py + python -m pytest tests/test_rocke_ci_static.py """ from __future__ import annotations @@ -21,6 +21,8 @@ from pathlib import Path from unittest.mock import patch +import pytest + # The representative-IR parity harness lives in the instances test layer; make # it importable from this root-level CI test (run via pytest or directly). sys.path.insert(0, str(Path(__file__).resolve().parents[0] / "instances")) @@ -31,7 +33,7 @@ expand_sweep, ) from rocke.core.arch import known_arches -from rocke.core.ir_golden import load_golden +from rocke.core.ir_golden import GOLDEN_FLAVORS, check_golden_flavors, load_golden from rocke_ir_parity_harness import ( # noqa: E402 -- after sys.path shim cases, check_golden, @@ -117,23 +119,33 @@ def test_ir_case_ids_are_unique(self): case_ids = [case["case_id"] for case in cases()] self.assertEqual(len(case_ids), len(set(case_ids))) - def test_ir_cases_match_golden_sha256(self): - """Byte-stability gate: every case's lowered-IR sha256 (and the set of - expected failures) must match the committed golden, for EVERY llvm - flavor the golden holds -- not just the one this host autodetects, or - the llvm23 sub-document would go unchecked below ROCm 7.13. A drift - here means emitted IR changed -- if intended & reviewed, re-bless the - changed families' shards with: - python tests/instances/rocke_ir_parity_harness.py \\ - --write tests/golden/rocke_representative_ir_sha256 --only - - """ - self.assertTrue( - _GOLDEN.exists(), - f"golden missing: {_GOLDEN} (bless with harness --write)", - ) - drift = check_golden(_GOLDEN) - self.assertEqual(drift, [], "IR drift vs golden:\n " + "\n ".join(drift)) + +# The representative-IR byte-stability gate, split so each LLVM flavor is its own +# test: lowering every case at every flavor in one test came close to the +# per-test timeout. Together these check exactly what one all-flavor +# check_golden call does -- the golden's flavor documents once, then every case +# at every flavor in GOLDEN_FLAVORS (every lane rocke qualifies, not just the +# one this host autodetects). A drift means emitted IR changed -- if intended +# and reviewed, re-bless the changed families' shards with: +# python tests/instances/rocke_ir_parity_harness.py \ +# --write tests/golden/rocke_representative_ir_sha256 --only + + +def test_golden_holds_every_llvm_flavor(): + """The golden has a document (or alias) for every LLVM flavor and nothing + for a flavor rocke no longer qualifies.""" + assert _GOLDEN.exists(), f"golden missing: {_GOLDEN} (bless with harness --write)" + errors = check_golden_flavors(_GOLDEN) + assert errors == [], "golden flavor set is wrong:\n " + "\n ".join(errors) + + +@pytest.mark.parametrize("flavor", GOLDEN_FLAVORS) +def test_ir_cases_match_golden_sha256(flavor): + """Every case's lowered-IR sha256 (and the set of expected failures) at + ``flavor`` matches the committed golden.""" + assert _GOLDEN.exists(), f"golden missing: {_GOLDEN} (bless with harness --write)" + drift = check_golden(_GOLDEN, flavor) + assert drift == [], f"IR drift vs golden at {flavor}:\n " + "\n ".join(drift) class TestGoldenShardWriter(unittest.TestCase): @@ -199,4 +211,4 @@ def test_removed_family_shard_deleted(self): if __name__ == "__main__": - unittest.main(verbosity=2) + raise SystemExit(pytest.main([__file__, "-v"]))