fix(rocke): make both engines emit the same HIP source

dnn-providers/hip-kernel-provider/ROCKE_ENGINE_test_categories_external.yaml

@@ -66,6 +66,7 @@ test_categories:
       - "rocke_arch_data_mirror"
       - "rocke_matrix_call_lowering"
       - "rocke_ir_types"
+      - "rocke_float_format"
       - "rocke_opcode_values"
       - "rocke_unsigned_int_ops"
       - "rocke_qk_scale_spec_validate"
@@ -79,6 +80,8 @@ test_categories:
       - "rocke_golden_static"
       - "rocke_differential_golden_ll"
       - "rocke_differential_golden_canonical"
+      # Not registered on Windows (see rocke/platform/CMakeLists.txt).
+      - "rocke_hip_source_identity"
       - "rocke_library_conv_pytest"
       - "rocke_library_attention_cpu_pytest"
       - "rocke_library_attention_pytest"
@@ -124,6 +127,7 @@ test_categories:
       - "rocke_arch_data_mirror"
       - "rocke_matrix_call_lowering"
       - "rocke_ir_types"
+      - "rocke_float_format"
       - "rocke_opcode_values"
       - "rocke_unsigned_int_ops"
       - "rocke_qk_scale_spec_validate"
@@ -137,6 +141,8 @@ test_categories:
       - "rocke_golden_static"
       - "rocke_differential_golden_ll"
       - "rocke_differential_golden_canonical"
+      # Not registered on Windows (see rocke/platform/CMakeLists.txt).
+      - "rocke_hip_source_identity"
       - "rocke_library_conv_pytest"
       - "rocke_library_attention_cpu_pytest"
       - "rocke_library_attention_pytest"

dnn-providers/hip-kernel-provider/rocke/TESTING.md

@@ -325,6 +325,18 @@ Key semantics:
 - **Property/fuzz generation**
   ([`fuzz_diff.py`](platform/tests/instances/differential/fuzz_diff.py)) feeds the
   differential oracle with generated `(spec, arch)` inputs rather than a fixed list.
+- **The HIP-source route has its own identity test.** Both engines also lower a
+  kernel to HIP source. [`test_hip_source_identity.py`](platform/tests/instances/differential/test_hip_source_identity.py)
+  runs every parity emitter in its `target-ir` mode and lowers each Python-built
+  kernel through both HIP lowerers: the text must match, or both must refuse
+  with the same error type and message. A config that fails other than by its
+  builder refusing it fails the test. It runs in the developer's differential
+  pytest pass with `rocke_engine`, and in installed host CI as the
+  `rocke_hip_source_identity` CTest, which lowers through the installed
+  `rocke_ir_lower_cli` program (`ROCKE_IR_LOWER_CLI`, one `--hip-batch` run per
+  family) because the wheel has no binding. That CTest is registered on every
+  platform but Windows, whose CI shards run close to their time limit, so CI
+  does not compare the MSVC-built engine's HIP output with Python's.
 
 ### 4.4 Is the emitted IR legal? (the toolchain gate)
 

dnn-providers/hip-kernel-provider/rocke/library/tests/parity/_emit_common.py

@@ -5,9 +5,10 @@
 # emitters. Every <family>_emit.py parses the same argv (config index in
 # argv[1], optional mode in argv[2] defaulting to "ll"), then dispatches on the
 # mode: "ll" prints lower_kernel_to_llvm, "ir" prints the ck.dsl.ir/v1
-# serialization, "verify" prints verifier diagnostics, and any other mode is a
-# usage error. run_emit() centralizes that boilerplate so each emitter only has
-# to provide its config selector and kernel builder.
+# serialization, "verify" prints verifier diagnostics, "target-ir" prints the
+# kernel's gfx target and then its serialization (see target_ir), and any other
+# mode is a usage error. run_emit() centralizes that boilerplate so each emitter
+# only has to provide its config selector and kernel builder.
 import sys
 
 from rocke.core.ir_serialize import serialize
@@ -24,14 +25,31 @@
     from rocke import lower_kernel_to_llvm as _native_lower
 
 
+# The emitter modes run_emit accepts. A bespoke emitter that parses its own argv
+# accepts the same names.
+MODES = ("ll", "ir", "verify", "target-ir")
+
+
+def target_ir(kernel, arch):
+    """The "target-ir" text for one kernel: a ``target <gfx>`` line, then its
+    ck.dsl.ir/v1 serialization, newline-terminated.
+
+    The serialization does not record the target, so a consumer that lowers the
+    kernel itself -- the HIP-source identity test lowers it through both
+    engines -- reads it from here. An emitter that builds several kernels for one
+    config prints one block per kernel, back to back."""
+    text = serialize(kernel)
+    return f"target {arch}\n{text}" + ("" if text.endswith("\n") else "\n")
+
+
 def run_emit(spec_fn, build_fn, *, usage=None, arch="gfx950"):
     """Drive one parity emitter and return its process exit code.
 
     spec_fn(idx) returns either a spec or a (spec, arch) tuple; a bare spec uses
     the `arch` argument (default "gfx950"), while a tuple supplies its own arch.
     build_fn(spec, arch=arch) returns the kernel. The selected mode (argv[2], default
-    "ll") chooses the native Python lowerer, serialize, or verifier-diagnostic
-    output; an unrecognized mode is rejected.
+    "ll") chooses the native Python lowerer, serialize, verifier-diagnostic or
+    target_ir output; an unrecognized mode is rejected.
 
     The "ll" mode deliberately uses the NATIVE Python lowerer
     (``_lower_kernel_to_llvm_python``) rather than the backend-dispatching
@@ -42,11 +60,11 @@ def run_emit(spec_fn, build_fn, *, usage=None, arch="gfx950"):
     default flip.
     """
     if len(sys.argv) < 2:
-        sys.stderr.write(usage or "usage: <config_index> [ll|ir|verify]\n")
+        sys.stderr.write(usage or "usage: <config_index> [ll|ir|verify|target-ir]\n")
         return 2
     idx = int(sys.argv[1])
     mode = sys.argv[2] if len(sys.argv) > 2 else "ll"
-    if mode not in ("ll", "ir", "verify"):
+    if mode not in MODES:
         sys.stderr.write(f"unknown mode {mode}\n")
         return 2
     selected = spec_fn(idx)
@@ -64,6 +82,8 @@ def run_emit(spec_fn, build_fn, *, usage=None, arch="gfx950"):
         sys.stdout.write(text)
     elif mode == "ir":
         sys.stdout.write(serialize(kernel))
+    elif mode == "target-ir":
+        sys.stdout.write(target_ir(kernel, arch))
     else:  # verify
         sys.stdout.write("".join(str(d) + "\n" for d in verify(kernel)))
     return 0

dnn-providers/hip-kernel-provider/rocke/library/tests/parity/conv_direct_grouped_emit.py

@@ -98,6 +98,7 @@
 except ImportError:  # pragma: no cover - older reference tree
     from rocke import lower_kernel_to_llvm as _native_lower
 from rocke.core.ir_serialize import serialize
+from _emit_common import target_ir
 from rocke.core.verify import verify
 
 
@@ -2242,6 +2243,8 @@ def main() -> int:
         sys.stdout.write(text)
     elif mode == "ir":
         sys.stdout.write(serialize(kernel))
+    elif mode == "target-ir":
+        sys.stdout.write(target_ir(kernel, arch))
     elif mode == "verify":
         sys.stdout.write("".join(str(d) + "\n" for d in verify(kernel)))
     else:

dnn-providers/hip-kernel-provider/rocke/library/tests/parity/fmha_splitkv_decode_emit.py

@@ -23,6 +23,7 @@
 except ImportError:  # pragma: no cover - older reference tree
     from rocke import lower_kernel_to_llvm as _native_lower
 from rocke.core.ir_serialize import serialize
+from _emit_common import MODES, target_ir
 from rocke.core.verify import verify
 
 
@@ -121,7 +122,7 @@ def main() -> int:
     mode = "ll"
     if len(sys.argv) > 2:
         arg2 = sys.argv[2]
-        if arg2 in ("ir", "verify", "ll"):
+        if arg2 in MODES:
             mode = arg2
         elif arg2 in ("seg", "reduce"):
             phase = arg2
@@ -140,6 +141,8 @@ def main() -> int:
         sys.stdout.write(text)
     elif mode == "ir":
         sys.stdout.write(serialize(kernel))
+    elif mode == "target-ir":
+        sys.stdout.write(target_ir(kernel, "gfx950"))
     elif mode == "verify":
         sys.stdout.write("".join(str(d) + "\n" for d in verify(kernel)))
     else:

dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx942_attention_tiled_3d_emit.py

@@ -34,6 +34,7 @@
 except ImportError:  # pragma: no cover - older reference tree
     from rocke import lower_kernel_to_llvm as _native_lower
 from rocke.core.ir_serialize import serialize
+from _emit_common import MODES, target_ir
 from rocke.core.verify import verify
 
 
@@ -407,14 +408,16 @@ def main() -> int:
         return 2
     idx = int(sys.argv[1])
     mode = sys.argv[2] if len(sys.argv) > 2 else "ll"
-    if mode not in ("ll", "ir", "verify"):
+    if mode not in MODES:
         sys.stderr.write(f"unknown mode {mode}\n")
         return 2
     for kernel in _kernels(idx):
         if mode == "ll":
             sys.stdout.write(_native_lower(kernel, arch="gfx942"))
         elif mode == "ir":
             sys.stdout.write(serialize(kernel))
+        elif mode == "target-ir":
+            sys.stdout.write(target_ir(kernel, "gfx942"))
         else:
             sys.stdout.write("".join(str(d) + "\n" for d in verify(kernel)))
     return 0

dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_3d_emit.py

@@ -34,6 +34,7 @@
 except ImportError:  # pragma: no cover - older reference tree
     from rocke import lower_kernel_to_llvm as _native_lower
 from rocke.core.ir_serialize import serialize
+from _emit_common import target_ir
 from rocke.core.verify import verify
 
 
@@ -364,7 +365,7 @@ def main() -> int:
     mode = sys.argv[2] if len(sys.argv) > 2 else "ll"
     if mode == "ll":
         sys.stdout.write(_emit(idx))
-    elif mode == "ir":
+    elif mode in ("ir", "target-ir"):
         cfg = _CONFIGS[idx] if idx in _CONFIGS else None
         if cfg is None:
             raise SystemExit(f"unknown config index {idx}")
@@ -379,8 +380,11 @@ def main() -> int:
             emit_lse=idx in _REDUCE_LSE,
         )
         red_kernel = build_unified_attention_reduce_tiled(red_spec, arch="gfx950")
-        sys.stdout.write(serialize(seg_kernel))
-        sys.stdout.write(serialize(red_kernel))
+        for kernel in (seg_kernel, red_kernel):
+            if mode == "ir":
+                sys.stdout.write(serialize(kernel))
+            else:
+                sys.stdout.write(target_ir(kernel, "gfx950"))
     elif mode == "verify":
         cfg = _CONFIGS[idx] if idx in _CONFIGS else None
         if cfg is None:

dnn-providers/hip-kernel-provider/rocke/platform/CMakeLists.txt

@@ -358,11 +358,14 @@ if(ROCKE_INSTALL_TESTS)
     # compiler). The rest of tests/instances/differential needs the C++ build and
     # stays out of the installed tree (see the tests/ install below); these files
     # are the Python-only check: the runner, its batch worker, the golden store
-    # and file, and the CPU-count helper the runner imports from tools/.
+    # and file, and the CPU-count helper the runner imports from tools/. The
+    # HIP source identity gate reuses the runner and worker, and lowers through
+    # the installed rocke_ir_lower_cli (rocke_hip_source_identity below).
     set(_ROCKE_DIFFERENTIAL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests/instances/differential")
     install(FILES "${_ROCKE_DIFFERENTIAL_DIR}/run_diff.py"
                   "${_ROCKE_DIFFERENTIAL_DIR}/python_batch_worker.py"
                   "${_ROCKE_DIFFERENTIAL_DIR}/golden_store.py"
+                  "${_ROCKE_DIFFERENTIAL_DIR}/test_hip_source_identity.py"
                   "${CMAKE_CURRENT_SOURCE_DIR}/tools/_hostcaps.py"
             DESTINATION ${ROCKE_TEST_INSTALL_DIR}/tests/instances/differential)
     install(FILES "${_ROCKE_DIFFERENTIAL_DIR}/golden/llvm_gfx_all.json"
@@ -620,6 +623,27 @@ if(ROCKE_INSTALL_TESTS)
     string(APPEND _ROCKE_INSTALLED_CTEST_ENTRIES
            "set_tests_properties(rocke_differential_golden_canonical PROPERTIES LABELS \"ckc;python;golden;host;quick\" ENVIRONMENT \"PYTHONPATH=${_ROCKE_INSTALLED_PYTHONPATH}\" TIMEOUT 300)\n")
 
+    # Both engines lower every parity kernel to the same HIP source. The wheel
+    # has no rocke_engine binding, so the C++ lowering runs through the
+    # installed rocke_ir_lower_cli. The whole gate takes about a minute on a
+    # developer host; one family, the per-test item, a few seconds.
+    #
+    # Not registered on Windows: the Windows CI shards run close to their time
+    # limit. The cost is that the MSVC-built engine's HIP output is not compared
+    # with Python's in CI; it matched on every parity kernel on the Windows lanes
+    # when this gate was added, and float formatting, the most host-sensitive
+    # piece, keeps its own rocke_float_format test there. Re-register on Windows
+    # once those shards have room. It stays registered on every other platform.
+    if(TARGET rocke_ir_lower_cli AND NOT WIN32)
+      rocke_cxx_test_installed_name(_rocke_installed_name rocke_ir_lower_cli)
+      string(APPEND _ROCKE_INSTALLED_CTEST_ENTRIES
+             "add_test(rocke_hip_source_identity python -m pytest \"${ROCKE_TEST_CTEST_REL}tests/instances/differential/test_hip_source_identity.py\" -v -rs --timeout=120)\n")
+      string(APPEND _ROCKE_INSTALLED_CTEST_ENTRIES
+             "set_tests_properties(rocke_hip_source_identity PROPERTIES LABELS \"ckc;python;host;quick\" ENVIRONMENT \"PYTHONPATH=${_ROCKE_INSTALLED_PYTHONPATH};ROCKE_IR_LOWER_CLI=${ROCKE_CXX_TEST_CTEST_REL}${_rocke_installed_name}${CMAKE_EXECUTABLE_SUFFIX}\" TIMEOUT 300)\n")
+    elseif(WIN32)
+      message(STATUS "rocKE: rocke_hip_source_identity not registered on Windows (CI shard time; see comment above)")
+    endif()
+
     if(ROCKE_INSTALL_PLATFORM_PYTEST)
       # Full pytest suite. PYTHONPATH points at the CTestTestfile's working
       # directory (which contains the installed rocke package) so `import rocke`
@@ -642,6 +666,7 @@ if(ROCKE_INSTALL_TESTS)
       foreach(_rocke_native_parity
               "ROCKE_STORAGE_TEST=rocke_storage"
               "ROCKE_IR_TYPES_TEST=rocke_ir_types"
+              "ROCKE_FLOAT_FORMAT_TEST=rocke_float_format"
               "ROCKE_NONTEMPORAL_HIP_TEST=rocke_nontemporal_hip"
               "ROCKE_UNSIGNED_INT_OPS_CLI=rocke_unsigned_int_ops"
               "ROCKE_ARCH_TARGET_TEST_EXE=rocke_arch_target_identity"
@@ -658,7 +683,7 @@ if(ROCKE_INSTALL_TESTS)
       string(APPEND _ROCKE_INSTALLED_CTEST_ENTRIES
              "set(_rocke_platform_pytest_environment \"${_ROCKE_PYTEST_ENVIRONMENT}\")\n"
              "list(APPEND _rocke_platform_pytest_environment \${_rocke_compiler_tool_environment})\n"
-             "add_test(rocke_pytest python -m pytest \"${ROCKE_TEST_CTEST_REL}tests\" --ignore \"${ROCKE_TEST_CTEST_REL}tests/library\" -m \"not integration\" -v -rs --timeout=60)\n")
+             "add_test(rocke_pytest python -m pytest \"${ROCKE_TEST_CTEST_REL}tests\" --ignore \"${ROCKE_TEST_CTEST_REL}tests/library\" --ignore \"${ROCKE_TEST_CTEST_REL}tests/instances/differential/test_hip_source_identity.py\" -m \"not integration\" -v -rs --timeout=60)\n")
       string(APPEND _ROCKE_INSTALLED_CTEST_ENTRIES
              "set_tests_properties(rocke_pytest PROPERTIES\n"
              "  LABELS \"ckc\\;python\\;quick\"\n"