danielbolsson · Oct 19, 2026 · Oct 16, 2026 · Oct 19, 2026 · Oct 19, 2026 · Oct 19, 2026
@@ -116,6 +116,20 @@ as CI does.
 `HRR_TEST_IGNORE_CONFIG=1`, or `--ignore-config`, runs the cases the config would
 skip. `--arch gfx1100` (or several, comma-separated) stands in for detection.
 
+## Local Docker
+
+`projects/hrr/tests/scripts/hrr_dev.py` builds an `hrr-dev` image when it is
+missing and runs the suites inside it. The host needs Docker.
+
+```bash
+python3 projects/hrr/tests/scripts/hrr_dev.py emulator
+python3 projects/hrr/tests/scripts/hrr_dev.py gpu
+```
+
+`emulator` runs the integration drivers under rocJitsu (gfx950, two GPUs, and
+gfx1250, four GPUs). `gpu` passes `/dev/kfd` and `/dev/dri` into the container.
+The host unit binary runs once in either command.
+
 ## Testing the Python helpers
 
 `scripts/` holds both the helpers and their tests. The tests need only Python and

@@ -0,0 +1,17 @@
+# Copyright Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+
+# Local HRR development image. The host only needs Docker.
+# CMake, Ninja and Python 3 come from the TheRock manylinux build image.
+# This adds PyYAML for the HRR scripts, the GitHub CLI, and GCC 13, the
+# minimum emulation/rocjitsu/docs/building.md asks for.
+FROM ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:cf4f6d9909056906e4a1c0bc4c245658a0af4c6a6174082a86adfa2d518b7aeb
+
+USER root
+RUN python3 -m pip install --no-cache-dir 'PyYAML==6.0.2' \
+    && curl -fsSL https://cli.github.com/packages/rpm/gh-cli.repo -o /etc/yum.repos.d/gh-cli.repo \
+    && dnf install -y gh gcc-toolset-13 \
+    && dnf clean all \
+    && groupadd -f video
+ENV PATH=/opt/rh/gcc-toolset-13/root/usr/bin:${PATH}
+WORKDIR /workspace
@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+# Copyright Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+
+"""Build and run the HRR suites in a local Docker container.
+
+The host needs Docker. The hrr-dev image supplies the compilers, Python, and
+rocJitsu's GCC. This script builds that image the first time it is missing.
+
+    python3 projects/hrr/tests/scripts/hrr_dev.py emulator
+    python3 projects/hrr/tests/scripts/hrr_dev.py gpu
+
+`emulator` runs the integration drivers under rocJitsu. `gpu` runs them on
+the machine's GPUs. The host unit binary has no device code, so both
+commands run it once inside the container.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+IMAGE = "hrr-dev"
+CONTAINER = "hrr-dev"
+REPO = Path(__file__).resolve().parents[4]
+ROCM_INDEX_URL = "https://rocm.nightlies.amd.com/whl-multi-arch/"
+ROCM_SDK_VERSION = "10.1.0a20260822"
+
+# Default rocJitsu jobs. These are development choices, not the CI matrix.
+EMULATOR_JOBS = (
+    {
+        "family": "gfx950",
+        "arch": "gfx950",
+        "config": "emulation/rocjitsu/configs/gfx950_mi355x_kmd_2gpu.json",
+    },
+    {
+        "family": "gfx125x",
+        "arch": "gfx1250",
+        "config": "emulation/rocjitsu/configs/gfx1250_mi455x_kmd_4gpu.json",
+    },
+)
+
+
+@dataclass(frozen=True)
+class DockerState:
+    image: bool
+    container: str | None  # None, "running", or "exited"
+
+
+def docker_build_command(repo: Path) -> list[str]:
+    return [
+        "docker", "build", "-t", IMAGE,
+        "-f", str(repo / "projects/hrr/tests/docker/Dockerfile"),
+        str(repo),
+    ]
+
+
+def docker_run_command(repo: Path, backend: str) -> list[str]:
+    command = [
+        "docker", "run", "-d", "--name", CONTAINER,
+        "-v", f"{repo}:{repo}", "-w", str(repo),
+        "--user", f"{os.getuid()}:{os.getgid()}",
+    ]
+    if backend == "gpu":
+        command += ["--device", "/dev/kfd", "--device", "/dev/dri", "--group-add", "video"]
+    command += [IMAGE, "sleep", "infinity"]
+    return command
+
+
+def plan_docker(repo: Path, backend: str, state: DockerState) -> list[list[str]]:
+    """Commands that make the hrr-dev container ready. An existing container
+    is started, not recreated."""
+    commands = []
+    if not state.image:
+        commands.append(docker_build_command(repo))
+    if state.container is None:
+        commands.append(docker_run_command(repo, backend))
+    elif state.container != "running":
+        commands.append(["docker", "start", CONTAINER])
+    return commands
+
+
+def inspect_docker(runner=subprocess.run) -> DockerState:
+    image = runner(
+        ["docker", "image", "inspect", IMAGE],
+        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+    )
+    container = runner(
+        ["docker", "inspect", "-f", "{{.State.Status}}", CONTAINER],
+        capture_output=True, text=True,
+    )
+    status = container.stdout.strip() if container.returncode == 0 else None
+    return DockerState(image.returncode == 0, status or None)
+
+
+def container_script(repo: Path, backend: str) -> str:
+    """Bash run inside the container. One rocJitsu family failing still runs
+    the other."""
+    headers = f"{repo}/projects/hip/include;{repo}/projects/clr/hipamd/include"
+    if backend == "gpu":
+        # Build for the GPUs actually present, not the emulator targets.
+        targets = (
+            "$(ROCM_SDK=\"$ROCM_PATH\" python3 -c 'import sys; sys.path.insert(0, \"projects/hrr/tests/scripts\"); "
+            "import hrr_test_config as c; print(\";\".join(c.detect_archs()))')"
+        )
+    else:
+        targets = ";".join(job["arch"] for job in EMULATOR_JOBS)
+    lines = [
+        "set -euo pipefail",
+        # Non-root pip falls back to a user install; put its scripts on PATH.
+        'export PATH="$HOME/.local/bin:$PATH"',
+        f'python3 -m pip install --no-cache-dir --index-url "{ROCM_INDEX_URL}" '
+        f'"rocm[devel]=={ROCM_SDK_VERSION}"',
+        "rocm-sdk init",
+        'export ROCM_PATH="$(rocm-sdk path --root)"',
+        "cmake -S projects/hrr -B build/hrr-dev/unit -G Ninja "
+        "-DCMAKE_BUILD_TYPE=Release "
+        '-DROCM_PATH="$ROCM_PATH" -DCMAKE_PREFIX_PATH="$ROCM_PATH" '
+        f'"-DHRR_HIP_INCLUDE_DIRS={headers}" '
+        "-DHRR_BUILD_PLAYBACK=ON -DHRR_BUILD_TESTS=ON -DHRR_BUILD_INTEGRATION_TESTS=OFF",
+        "cmake --build build/hrr-dev/unit -j",
+        "cmake -S projects/clr -B build/hrr-dev/clr -G Ninja "
+        "-DCMAKE_BUILD_TYPE=Release "
+        "-DCMAKE_CXX_FLAGS=-DHIP_HRR_CAPTURE_ENABLED "
+        '-DCMAKE_PREFIX_PATH="$ROCM_PATH" -DROCM_PATH="$ROCM_PATH" '
+        f'-DHIP_COMMON_DIR="{repo}/projects/hip" '
+        "-DHIP_PLATFORM=amd -DCLR_BUILD_HIP=ON -DCLR_BUILD_OCL=OFF",
+        "cmake --build build/hrr-dev/clr --target amdhip64 -j",
+        f'targets="{targets}"',
+        '[ -n "$targets" ] || { echo "error: no GPU architecture detected" >&2; exit 1; }',
+        "cmake -S projects/hrr -B build/hrr-dev/integration -G Ninja "
+        "-DCMAKE_BUILD_TYPE=Release "
+        '-DROCM_PATH="$ROCM_PATH" -DCMAKE_PREFIX_PATH="$ROCM_PATH" '
+        f'"-DHRR_HIP_INCLUDE_DIRS={headers}" '
+        '"-DGPU_TARGETS=$targets" "-DCMAKE_HIP_ARCHITECTURES=$targets" '
+        "-DHRR_BUILD_PLAYBACK=ON -DHRR_BUILD_TESTS=ON -DHRR_BUILD_INTEGRATION_TESTS=ON",
+        "cmake --build build/hrr-dev/integration -j",
+        "set +e",
+        "status=0",
+        f'export LD_LIBRARY_PATH="{repo}/build/hrr-dev/clr/hipamd/lib:'
+        '${ROCM_PATH}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"',
+        "python3 projects/hrr/tests/scripts/run_catch2.py "
+        "--xml build/hrr-dev/unit.xml --config projects/hrr/tests/config/unit.yaml "
+        "--arch none -- build/hrr-dev/unit/tests/unit/hrr-unit-tests || status=1",
+        "if [ -x build/hrr-dev/unit/tests/unit/hrr-handler-tests ]; then "
+        "python3 projects/hrr/tests/scripts/run_catch2.py "
+        "--xml build/hrr-dev/handler.xml "
+        "-- build/hrr-dev/unit/tests/unit/hrr-handler-tests || status=1; fi",
+    ]
+    binary = f"{repo}/build/hrr-dev/integration/tests/integration/hrr-integration-tests"
+    if backend == "emulator":
+        lines[lines.index("set +e"):lines.index("set +e")] = [
+            "cmake -S emulation/rocjitsu -B build/hrr-dev/rocjitsu -G Ninja "
+            "-DCMAKE_BUILD_TYPE=Release",
+            "cmake --build build/hrr-dev/rocjitsu --target rocjitsu_bin -j",
+            'export PATH="' + str(repo) + '/build/hrr-dev/rocjitsu/tools/rocjitsu:${PATH}"',
+        ]
+        for job in EMULATOR_JOBS:
+            launcher = f"rocjitsu --daemon --config {repo}/{job['config']} --"
+            lines.append(
+                "python3 projects/hrr/tests/scripts/run_catch2.py "
+                f"--xml build/hrr-dev/integration-{job['family']}.xml --timeout 1200 "
+                "--config projects/hrr/tests/config/integration.yaml "
+                f"--os linux --arch {job['arch']} --launcher \"{launcher}\" "
+                f"-- {binary} || status=1"
+            )
+    else:
+        lines.append(
+            "python3 projects/hrr/tests/scripts/run_catch2.py "
+            "--xml build/hrr-dev/integration-gpu.xml --timeout 1200 "
+            "--config projects/hrr/tests/config/integration.yaml "
+            f"--os linux --arch auto -- {binary} || status=1"
+        )
+    lines.append("exit $status")
+    return "\n".join(lines) + "\n"
+
+
+def exec_command(repo: Path, backend: str) -> list[str]:
+    # No passwd entry for --user uid:gid, so HOME would be "/".
+    command = ["docker", "exec", "-w", str(repo), "-e", "HOME=/tmp"]
+    # Pass the token by name so its value stays off the process list.
+    if os.environ.get("GH_TOKEN"):
+        command += ["-e", "GH_TOKEN"]
+    elif os.environ.get("GITHUB_TOKEN"):
+        command += ["-e", "GH_TOKEN"]
+        os.environ["GH_TOKEN"] = os.environ["GITHUB_TOKEN"]
+    command += [CONTAINER, "bash", "-lc", container_script(repo, backend)]
+    return command
+
+
+def prepare(repo: Path, backend: str, runner=subprocess.run) -> None:
+    for command in plan_docker(repo, backend, inspect_docker(runner)):
+        subprocess.run(command, check=True)
+    subprocess.run(exec_command(repo, backend), check=True)
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("backend", choices=("emulator", "gpu"))
+    parser.add_argument("--repo", type=Path, default=REPO)
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    try:
+        prepare(args.repo.resolve(), args.backend)
+    except subprocess.CalledProcessError as exc:
+        return exc.returncode or 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -7,6 +7,7 @@
 
 import argparse
 import os
+import shlex
 import signal
 import subprocess
 import sys
@@ -30,6 +31,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser.add_argument("--select", help="Catch2 selector; overrides the one in the config")
     parser.add_argument("--ignore-config", action="store_true",
                         help="run the cases the config would skip")
+    parser.add_argument(
+        "--launcher", default="",
+        help="Prefix for the suite process. Case listing still uses the Catch2 binary.",
+    )
     parser.add_argument("command", nargs=argparse.REMAINDER)
     args = parser.parse_args(argv)
     if args.command[:1] == ["--"]:
@@ -285,6 +290,8 @@ def _run_suite(args: argparse.Namespace, state: dict) -> int:
             return 0
     else:
         command = list(args.command)
+    if args.launcher:
+        command = shlex.split(args.launcher) + command
     command = command + [
         "--durations",
         "yes",

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
+# SPDX-License-Identifier: MIT
+"""Tests for hrr_dev.py. Docker is not started."""
+
+import sys
+import os
+import unittest
+from unittest import mock
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import hrr_dev  # noqa: E402
+
+
+class ParseTest(unittest.TestCase):
+    def test_emulator_and_gpu(self):
+        self.assertEqual(hrr_dev.parse_args(["emulator"]).backend, "emulator")
+        self.assertEqual(hrr_dev.parse_args(["gpu"]).backend, "gpu")
+
+
+class DockerPlanTest(unittest.TestCase):
+    def test_missing_image_is_a_docker_build(self):
+        repo = Path("/repo")
+        commands = hrr_dev.plan_docker(
+            repo, "emulator", hrr_dev.DockerState(image=False, container=None)
+        )
+        self.assertEqual(commands[0][:3], ["docker", "build", "-t"])
+        self.assertIn("hrr-dev", commands[0])
+        self.assertIn(str(repo / "projects/hrr/tests/docker/Dockerfile"), commands[0])
+
+    def test_stopped_container_is_started(self):
+        commands = hrr_dev.plan_docker(
+            Path("/repo"), "gpu", hrr_dev.DockerState(image=True, container="exited")
+        )
+        self.assertEqual(commands, [["docker", "start", "hrr-dev"]])
+
+    def test_only_gpu_adds_devices(self):
+        repo = Path("/repo")
+        state = hrr_dev.DockerState(image=True, container=None)
+        emulator = hrr_dev.plan_docker(repo, "emulator", state)[0]
+        gpu = hrr_dev.plan_docker(repo, "gpu", state)[0]
+        self.assertNotIn("/dev/kfd", emulator)
+        self.assertNotIn("/dev/dri", emulator)
+        self.assertEqual(
+            gpu[gpu.index("--device"):gpu.index("--device") + 4],
+            ["--device", "/dev/kfd", "--device", "/dev/dri"],
+        )
+
+
+class ScriptTest(unittest.TestCase):
+    def test_emulator_uses_both_default_configs(self):
+        script = hrr_dev.container_script(Path("/repo"), "emulator")
+        self.assertIn("gfx950_mi355x_kmd_2gpu.json", script)
+        self.assertIn("gfx1250_mi455x_kmd_4gpu.json", script)
+        self.assertIn("--arch gfx950", script)
+        self.assertIn("--arch gfx1250", script)
+        self.assertIn("rocjitsu --daemon", script)
+        self.assertIn("HIP_HRR_CAPTURE_ENABLED", script)
+        self.assertIn("HRR_BUILD_INTEGRATION_TESTS=OFF", script)
+        self.assertNotIn("/dev/kfd", script)
+
+    def test_gpu_uses_arch_auto_and_no_launcher(self):
+        script = hrr_dev.container_script(Path("/repo"), "gpu")
+        self.assertIn("--arch auto", script)
+        self.assertNotIn("rocjitsu", script)
+        self.assertNotIn("gfx950_mi355x_kmd_2gpu.json", script)
+        self.assertIn("detect_archs", script)
+        self.assertNotIn("gfx950;gfx1250", script)
+
+    def test_capture_library_on_path_in_both_modes(self):
+        for backend in ("emulator", "gpu"):
+            script = hrr_dev.container_script(Path("/repo"), backend)
+            self.assertLess(
+                script.index("/repo/build/hrr-dev/clr/hipamd/lib"),
+                script.index("run_catch2.py --xml build/hrr-dev/integration"),
+            )
+
+    def test_handler_run_has_no_missing_config(self):
+        script = hrr_dev.container_script(Path("/repo"), "emulator")
+        self.assertNotIn("handler.yaml", script)
+
+    def test_token_not_on_command_line(self):
+        with mock.patch.dict(os.environ, {"GH_TOKEN": "secret"}):
+            command = hrr_dev.exec_command(Path("/repo"), "gpu")
+        self.assertNotIn("GH_TOKEN=secret", command)
+        self.assertIn("GH_TOKEN", command)
+        self.assertIn("HOME=/tmp", command)
+
+    def test_gpu_detection_looks_in_the_sdk_and_fails_closed(self):
+        script = hrr_dev.container_script(Path("/repo"), "gpu")
+        self.assertIn('ROCM_SDK="$ROCM_PATH" python3 -c', script)
+        self.assertIn('[ -n "$targets" ] || {', script)
+
+    def test_user_install_scripts_on_path_before_sdk_use(self):
+        script = hrr_dev.container_script(Path("/repo"), "emulator")
+        self.assertLess(
+            script.index('export PATH="$HOME/.local/bin:$PATH"'),
+            script.index("rocm-sdk init"),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()