From 9d2979c623664d49d6723668da80e88575c98b3a Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 12 Oct 2026 09:42:13 +0200 Subject: [PATCH 01/89] ci(hrr): add the HRR CI workflow Split out of the HRR test/tooling PR so CODEOWNERS routes .github/ and projects/hrr to their own reviewers. Restores the workflow byte-for-byte. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 1158 ++++++++++++++++++++++++++++++++++ 1 file changed, 1158 insertions(+) create mode 100644 .github/workflows/hrr-ci.yml diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml new file mode 100644 index 000000000000..49b9290eb442 --- /dev/null +++ b/.github/workflows/hrr-ci.yml @@ -0,0 +1,1158 @@ +name: HRR CI + +# A GPU-only dispatch names the run after its id so the PR job can find it. +# Every other event keeps a stable title. +run-name: ${{ github.event.inputs.gpu_only == 'true' && format('hrr-gpu-{0}', github.event.inputs.dispatch_id) || format('HRR CI ({0})', github.event_name) }} + +# Pull requests always start the workflow. A workflow skipped by a top-level +# "paths" filter creates no check at all, so a required "HRR CI Summary" would +# wait forever on every PR that does not touch HRR. The gate job below does the +# path filtering instead, and the summary job always reports. +# +# Pushes to develop run the same suites for one reason beyond post-merge signal: +# a pull request can only restore caches from its own ref and from the base +# branch, so without a develop run every PR would start with a cold ccache. +on: + pull_request: + push: + branches: + - develop + paths: + - "projects/hrr/**" + - "!projects/hrr/skills/**" + - "projects/clr/hipamd/src/hrr/**" + - "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp" + - "projects/hip/include/hip/hip_runtime_api.h" + - ".github/workflows/hrr-ci.yml" + - ".github/actions/rocprofiler-sdk-util/resolve-ci-config/**" + workflow_dispatch: + inputs: + ignore_test_config: + description: "Run cases the YAML config would skip" + type: boolean + default: false + # The fields below are set by the integration dispatcher. A new workflow + # file cannot be dispatched until it is on the default branch, so the + # GPU job lives in this file and the PR run starts it with gpu_only. + gpu_only: + description: "Run one integration family against an existing build" + type: boolean + default: false + family: + description: "Integration family (gpu_only)" + type: string + default: "" + runner: + description: "Runner label (gpu_only)" + type: string + default: "" + use_container: + description: "Run inside the GPU container (gpu_only)" + type: boolean + default: false + sdk_version: + description: "ROCm SDK version the build used (gpu_only)" + type: string + default: "" + build_workspace: + description: "Absolute workspace path baked into the binaries (gpu_only)" + type: string + default: "" + build_run_id: + description: "Run id whose hrr-build artifact to download (gpu_only)" + type: string + default: "" + # workflow_dispatch allows at most ten inputs; this list is full. + build_sha: + description: "Commit the build compiled; the GPU job checks this out (gpu_only)" + type: string + default: "" + dispatch_id: + description: "Unique id the dispatcher uses to find this run (gpu_only)" + type: string + default: "" + +concurrency: + # Each GPU dispatch has its own group. Sharing the PR group would queue the + # families behind each other, and the schedule budget would skip them. + group: ${{ github.event.inputs.gpu_only == 'true' && format('hrr-ci-gpu-{0}', github.event.inputs.dispatch_id) || format('hrr-ci-{0}', github.event.pull_request.number || github.ref) }} + # Never cancel a develop run: it is what saves the ccache for later PRs. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + # One fat binary covers every family that has a test runner, so the GPU jobs + # download a single artifact instead of rebuilding per architecture. + # + # Every arch a pool can hand out has to be listed, not one representative per + # family: linux-gfx110X-gpu-rocm and linux-gfx120X-gpu-rocm are wildcard + # pools, and a machine whose arch is missing from the binary fails every + # workload with hipErrorInvalidImage (200) rather than skipping. gfx1250 is + # carried so the binary is ready once its runner is enabled below. + HRR_ARCHS: "gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1153;gfx1200;gfx1201;gfx1250" + ROCM_INDEX_URL: "https://rocm.nightlies.amd.com/whl-multi-arch/" + +jobs: + # --------------------------------------------------------------------------- + # Path gate. The "on:" block cannot filter pull requests by path (see above), + # so this job lists the files the pull request changes and sets run=false when + # none is an HRR path. Every job below is then skipped, not failed, and + # "HRR CI Summary" still reports. Pushes and manual runs always run. + # --------------------------------------------------------------------------- + gate: + name: "HRR paths changed?" + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read + outputs: + run: ${{ steps.decide.outputs.run }} + steps: + - name: Decide + id: decide + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + run=true + reason="not a pull request" + if [ -n "${PR_NUMBER}" ]; then + reason="the pull request changes an HRR path" + # The files API lists at most 3000 files; a PR that large is treated + # as relevant rather than guessed at. + gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files" \ + --jq '.[].filename' > changed-files.txt + if ! python3 - changed-files.txt <<'PY' + import sys + + INCLUDE = ( + "projects/hrr/", + "projects/clr/hipamd/src/hrr/", + "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp", + "projects/hip/include/hip/hip_runtime_api.h", + ".github/workflows/hrr-ci.yml", + ".github/actions/rocprofiler-sdk-util/resolve-ci-config/", + ) + EXCLUDE = ("projects/hrr/skills/",) + + files = [line.strip() for line in open(sys.argv[1]) if line.strip()] + relevant = len(files) >= 3000 or any( + f.startswith(INCLUDE) and not f.startswith(EXCLUDE) for f in files + ) + sys.exit(0 if relevant else 1) + PY + then + run=false + reason="the pull request changes no HRR path" + fi + fi + echo "run=${run}" >> "$GITHUB_OUTPUT" + echo "HRR CI: run=${run} (${reason})" | tee -a "$GITHUB_STEP_SUMMARY" + + # --------------------------------------------------------------------------- + # Generated-source and wire-compatibility gate. This is intentionally + # host-only and separate from the CLR build: an API-header PR must get a fast + # result even when no GPU job runs. + # --------------------------------------------------------------------------- + codegen: + name: "Generated sources" + needs: [gate] + if: ${{ github.event.inputs.gpu_only != 'true' && needs.gate.outputs.run == 'true' }} + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + sparse-checkout: | + projects/hrr + projects/hip/include + projects/clr/hipamd/include + projects/clr/hipamd/src/hrr + set-safe-directory: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + # api_matrix.yaml is read with PyYAML by the matrix scripts below. + # Install only when yaml does not import. A version pin makes pip + # uninstall a newer copy, which fails when site-packages is read-only. + - name: Install PyYAML + run: python3 -c "import yaml" || python3 -m pip install --no-cache-dir PyYAML==6.0.1 + + - name: Generate and compare + run: | + mkdir -p build/hrr-codegen/include/hrr + python3 projects/hrr/tools/gen_hrr_api_args.py \ + --input projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp \ + --public-header projects/hip/include/hip/hip_runtime_api.h \ + --output-header build/hrr-codegen/include/hrr/hrr_api_args.h \ + --output-capture build/hrr-codegen/hip_capture_generated.cpp \ + --output-playback build/hrr-codegen/hip_playback_generated.cpp \ + --check-hrr-coverage \ + --manual-capture-source projects/clr/hipamd/src/hrr/hip_capture.cpp \ + --silent + + failed=0 + diff -u projects/hrr/include/hrr/hrr_api_args.h \ + build/hrr-codegen/include/hrr/hrr_api_args.h || failed=1 + diff -u projects/clr/hipamd/src/hrr/hip_capture_generated.cpp \ + build/hrr-codegen/hip_capture_generated.cpp || failed=1 + diff -u projects/hrr/playback/hip_playback_generated.cpp \ + build/hrr-codegen/hip_playback_generated.cpp || failed=1 + if [ "${failed}" -ne 0 ]; then + echo "::error::checked-in HRR generated sources are stale; run projects/hrr/tools/gen_hrr_api_args.py --check-hrr-coverage" + exit 1 + fi + + - name: Check API ID compatibility + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + run: | + if [ -z "${BASE_SHA}" ] || [[ "${BASE_SHA}" =~ ^0+$ ]]; then + BASE_SHA="$(git rev-parse HEAD^)" + fi + git show "${BASE_SHA}:projects/hrr/include/hrr/hrr_api_args.h" \ + > build/hrr-codegen/base_hrr_api_args.h + python3 projects/hrr/tools/check_hrr_api_compat.py \ + --base-header build/hrr-codegen/base_hrr_api_args.h \ + --current-header build/hrr-codegen/include/hrr/hrr_api_args.h + + # Everything is written under build/hrr-codegen: the derived manifest and + # the regenerated header are compared with the checked-in header, never + # written over it. + - name: Check API matrix + run: | + tools=projects/hrr/tools/api-matrix + out=build/hrr-codegen/api-matrix + tracked=projects/hrr/tests/integration/hrr_api_matrix_expectations.h + mkdir -p "${out}" + + python3 "${tools}/derive_manifest.py" --quiet \ + --output "${out}/api_classes.json" + python3 "${tools}/check_matrix.py" --check-workloads + python3 "${tools}/check_matrix.py" \ + --manifest "${out}/api_classes.json" \ + --emit-cxx "${out}/hrr_api_matrix_expectations.h" + + if ! diff -u "${tracked}" "${out}/hrr_api_matrix_expectations.h"; then + echo "::error::${tracked} is stale; run derive_manifest.py, then check_matrix.py --emit-cxx, in projects/hrr/tools/api-matrix" + exit 1 + fi + + # --------------------------------------------------------------------------- + # Host-only unit suite. These 23 cases link nothing but libstdc++/libc: they + # include for types but issue no HIP calls, so they need + # neither a GPU nor the capture library and can run on any platform. + # --------------------------------------------------------------------------- + unit: + name: "Unit (${{ matrix.os }})" + needs: [gate] + if: ${{ github.event.inputs.gpu_only != 'true' && needs.gate.outputs.run == 'true' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2022] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + projects/hrr + projects/hip/include + projects/clr/hipamd/include + projects/clr/hipamd/src + .github/workflows + set-safe-directory: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install ROCm SDK (headers and toolchain only) + shell: bash + run: | + pip install --no-cache-dir --index-url "${ROCM_INDEX_URL}" "rocm[devel]" + rocm-sdk init + echo "ROCM_SDK=$(python -c 'import _rocm_sdk_devel,os;print(os.path.dirname(_rocm_sdk_devel.__file__))')" >> "$GITHUB_ENV" + + - name: Configure + shell: bash + run: | + extra=() + if [ "${RUNNER_OS}" = "Windows" ]; then + # The runner builds with MinGW, whose libstdc++/libgcc/winpthread + # DLLs are not on PATH for later steps, so a dynamically linked + # binary fails to start at all. Link them in instead. + extra+=(-DCMAKE_EXE_LINKER_FLAGS=-static) + fi + # Declarations of the HIP API come from this tree, ahead of the SDK's: + # the nightly SDK lags the tree, so an API added here is not in its + # headers yet. Only headers: the SDK still supplies the runtime library. + # cygpath -m gives the drive-letter form a native cmake accepts. + tree="$(cygpath -m "${PWD}" 2>/dev/null || echo "${PWD}")" + cmake -S projects/hrr -B build -GNinja -Wno-dev \ + "${extra[@]}" \ + "-DHRR_HIP_INCLUDE_DIRS=${tree}/projects/hip/include;${tree}/projects/clr/hipamd/include" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${ROCM_SDK}" \ + -DROCM_PATH="${ROCM_SDK}" \ + -DHRR_BUILD_PLAYBACK=ON \ + -DHRR_BUILD_TESTS=ON \ + -DHRR_BUILD_INTEGRATION_TESTS=OFF + + - name: Build + run: cmake --build build -j + + - name: Assert the expected unit cases are present + shell: bash + run: | + # A case behind an #if leaves the binary with no signal at all -- not + # failed, not skipped, absent -- so a fully green suite can hide it. + # Assert the compiled set against the YAML before trusting the run. + python3 -c "import yaml" || python3 -m pip install --no-cache-dir PyYAML==6.0.1 + if [ "${RUNNER_OS}" = "Windows" ]; then + PATH="$(cygpath -u "${ROCM_SDK}")/bin:${PATH}" + export PATH + fi + shopt -s nullglob + bin=(build/tests/unit/hrr-unit-tests*) + if [ ${#bin[@]} -eq 0 ]; then + echo "::error::no unit binary under build/tests/unit" + exit 1 + fi + python3 projects/hrr/tests/scripts/hrr_test_config.py lint \ + projects/hrr/tests/config/unit.yaml \ + projects/hrr/tests/config/integration.yaml \ + --workflow .github/workflows/hrr-ci.yml + python3 projects/hrr/tests/scripts/hrr_test_config.py check \ + --binary "${bin[0]}" \ + --config projects/hrr/tests/config/unit.yaml + + - name: Run unit suite + shell: bash + env: + HRR_TEST_IGNORE_CONFIG: ${{ inputs.ignore_test_config && '1' || '' }} + run: | + # Named hrr-unit-tests.exe on Windows. Resolved with a bash glob + # rather than find, which on a Windows runner resolves to FIND.EXE. + if [ "${RUNNER_OS}" = "Windows" ]; then + # The recovery cases spawn hrr-playback.exe, which loads the SDK's + # HIP DLLs. Without them on PATH it dies with STATUS_DLL_NOT_FOUND + # before running anything. + PATH="$(cygpath -u "${ROCM_SDK}")/bin:${PATH}" + export PATH + fi + shopt -s nullglob + bin=(build/tests/unit/hrr-unit-tests*) + if [ ${#bin[@]} -eq 0 ]; then + echo "::error::no unit binary under build/tests/unit" + ls -R build/tests || true + exit 1 + fi + ls -l "${bin[0]}" + python3 projects/hrr/tests/scripts/run_catch2.py \ + --xml unit-${{ matrix.os }}.xml \ + --config projects/hrr/tests/config/unit.yaml \ + --arch none \ + -- "${bin[0]}" + + # The helpers the suites below rely on (JUnit wrapper, summary table, + # suite config). They are POSIX-process code; once is enough. + # test_api_matrix.py imports PyYAML, which setup-python does not provide. + - name: Test the CI helper scripts + if: ${{ runner.os == 'Linux' }} + run: | + python3 -c "import yaml" || python3 -m pip install --no-cache-dir PyYAML==6.0.1 + python3 -m unittest discover -s projects/hrr/tests/scripts -p 'test_*.py' -v + + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: results-unit-${{ matrix.os }} + path: unit-${{ matrix.os }}.xml + retention-days: 3 + + # --------------------------------------------------------------------------- + # Resolve the GPU runner labels from the shared CI config, the same way the + # rocprofiler-sdk workflows do. + # --------------------------------------------------------------------------- + ci_config: + name: "Resolve runners" + needs: [gate] + # Also runs for a gpu_only dispatch. The integration matrix below reads + # these outputs while that job decides to skip, and a skipped resolver + # makes that evaluation fail. + if: ${{ needs.gate.outputs.run == 'true' }} + runs-on: ubuntu-24.04 + outputs: + gfx90a_runner: ${{ steps.runner.outputs.gfx90a_runner }} + gfx94x_runner: ${{ steps.runner.outputs.gfx94x_runner }} + gfx950_runner: ${{ steps.runner.outputs.gfx950_runner }} + gfx103x_runner: ${{ steps.runner.outputs.gfx103x_runner }} + gfx110x_runner: ${{ steps.runner.outputs.gfx110x_runner }} + gfx120x_runner: ${{ steps.runner.outputs.gfx120x_runner }} + gfx1151_runner: ${{ steps.runner.outputs.gfx1151_runner }} + steps: + - name: Checkout workflow actions + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: .github/actions/rocprofiler-sdk-util/resolve-ci-config + set-safe-directory: true + + - name: Resolve runner labels + id: runner + uses: ./.github/actions/rocprofiler-sdk-util/resolve-ci-config + + # --------------------------------------------------------------------------- + # Builds CLR (which brings in-tree ROCr with it) to get an amdhip64 carrying + # the HRR capture shims, then projects/hrr against the SDK. The capture build + # is substituted at run time via LD_LIBRARY_PATH, not linked here. Produces + # one multi-architecture artifact consumed by every GPU job below. + # --------------------------------------------------------------------------- + build: + name: "Build (multi-arch)" + needs: [gate] + if: ${{ github.event.inputs.gpu_only != 'true' && needs.gate.outputs.run == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} + # The integration binaries embed absolute paths into this workspace, so a + # containerised GPU job has to recreate it (see "Recreate build path"). + outputs: + workspace: ${{ github.workspace }} + # The nightly index moves every day. The integration jobs install exactly + # this version so the capture amdhip64 never runs against a newer ROCr + # than the one it was built with, however long a GPU queue is. + sdk_version: ${{ steps.sdk.outputs.version }} + runs-on: azure-linux-scale-rocm + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + # Hash the compiler by content, not mtime: the SDK wheel is reinstalled + # every run, so an mtime check would miss on every single object. + CCACHE_COMPILERCHECK: content + CCACHE_MAXSIZE: "5G" + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + projects/hrr + projects/clr + projects/hip + projects/rocr-runtime + set-safe-directory: true + + # ROCclr pulls in OpenGL through ROCclrHSA.cmake even with CLR_BUILD_OCL + # off, so the GL development packages are required to configure CLR. + - name: Install system dependencies + run: | + sudo apt-get update -q + sudo apt-get install -y --no-install-recommends \ + ninja-build pkg-config libnuma-dev libelf-dev libdrm-dev \ + libgl1-mesa-dev libglx-dev mesa-common-dev \ + libclang-rt-18-dev ccache xxd + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install ROCm SDK + id: sdk + run: | + # CppHeaderParser is required by hipamd/src/CMakeLists.txt. + pip install --no-cache-dir CppHeaderParser + pip install --no-cache-dir --index-url "${ROCM_INDEX_URL}" "rocm[libraries,devel]" + rocm-sdk init + echo "version=$(python -c 'import importlib.metadata as m; print(m.version("rocm"))')" >> "$GITHUB_OUTPUT" + echo "ROCM_SDK=$(python -c 'import _rocm_sdk_devel,os;print(os.path.dirname(_rocm_sdk_devel.__file__))')" >> "$GITHUB_ENV" + + # Nothing survived between runs before this: the artifact fan-out only + # reuses a build WITHIN a run. ccache is the house convention (15 + # workflows) and is carried across runs by actions/cache (9 workflows) + # because azure-linux-scale-rocm is ephemeral and keeps no state. + # + # This cache is a SPEED optimisation only and can never make a stale + # binary look fresh. The key exists to pick a good starting point; it + # is ccache itself that decides reuse, and it keys on the preprocessed + # source plus the command line plus the compiler contents. A changed + # input is therefore a miss by construction -- restoring a stale cache + # costs misses, never a wrong object. Do not replace this with a + # "skip the build if the key matches" scheme: that trades a correctness + # property for the same speed. + # + # Restore everywhere, save only from a push to develop. A pull request can + # restore its own caches and the base branch's but never write to + # develop's scope, and a per-run entry of up to 5 GB from every PR run + # would evict other workflows' caches. One develop entry per push seeds + # every PR instead. + - name: Restore ccache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ github.workspace }}/.ccache + key: hrr-ccache-${{ runner.os }}-${{ github.run_id }} + restore-keys: | + hrr-ccache-${{ runner.os }}- + + - name: Zero ccache statistics + run: ccache --zero-stats + + # CLR is built against this tree's ROCr, never the SDK's: CLR and ROCr + # change together in this repo, and the nightly's ROCr lags (a CLR that + # calls a new hsa_amd_* API does not compile against its headers). + # find_package(hsa-runtime64) finds this install first via + # CMAKE_PREFIX_PATH below. The SDK still supplies the compiler, comgr + # and device libraries. + # + # libdrm, libdrm_amdgpu, libnuma and libelf come from the SDK's bundled + # sysdeps (soname librocm_sysdeps_*), not the apt packages above. The GPU + # machines and the no-ROCm container do not have the system ones, so a + # ROCr linked against them fails to load there (libdrm.so.2). The SDK's + # own ROCr links these same sysdeps. + - name: Build ROCr from this tree + run: | + sysdeps="${ROCM_SDK}/lib/rocm_sysdeps" + export PKG_CONFIG_PATH="${sysdeps}/lib/pkgconfig" + cmake -S projects/rocr-runtime -B build/rocr -GNinja -Wno-dev \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${sysdeps};${ROCM_SDK}" \ + "-DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/build/rocr-install" + cmake --build build/rocr -j + cmake --install build/rocr + + # -DHIP_HRR_CAPTURE_ENABLED is what compiles capture into amdhip64 + # (hip_capture.cpp); the tree has no CMake option for it. CI always sets + # it, so capture is always on in the build under test. + - name: Build amdhip64 with the HRR capture shims + run: | + cmake -S projects/clr -B build/clr -GNinja -Wno-dev \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + "-DCMAKE_CXX_FLAGS=-DHIP_HRR_CAPTURE_ENABLED -I${GITHUB_WORKSPACE}/build/rocr-install/include -I${GITHUB_WORKSPACE}/build/rocr-install/include/hsa" \ + -DCMAKE_BUILD_TYPE=Release \ + "-DCMAKE_PREFIX_PATH=${GITHUB_WORKSPACE}/build/rocr-install;${ROCM_SDK}" \ + -DROCM_PATH="${ROCM_SDK}" \ + -DHIP_COMMON_DIR="${GITHUB_WORKSPACE}/projects/hip" \ + -DHIP_PLATFORM=amd \ + -DCLR_BUILD_HIP=ON \ + -DCLR_BUILD_OCL=OFF + cmake --build build/clr --target amdhip64 -j + + - name: Build projects/hrr for every testable architecture + run: | + # C and C++ only. CMAKE_HIP_COMPILER_LAUNCHER is deliberately NOT + # set: HIP device compilation fans one TU out to a code object per + # --offload-arch and bundles them, and ccache's handling of that + # multi-output bundling is not something this workflow has proven. + # A wrong hit there would mean testing a binary that does not match + # its source, which is exactly the failure mode this cache is + # required not to introduce. The C++ half (CLR/amdhip64, the bulk of + # the wall clock) is cached; extend to HIP only after a run proves + # the arch-embedding check still passes on a fully cached build. + # Host C++ is compiled against this tree's public HIP headers (and the + # hip_version.h the CLR build above generated) ahead of the SDK's, so + # an API added to the tree builds before any nightly declares it. HIP + # device code keeps the SDK's headers. The SDK's libamdhip64 is only + # linked, never run: the integration jobs put build/clr first. + cmake -S projects/hrr -B build/hrr -GNinja -Wno-dev \ + "-DHRR_HIP_INCLUDE_DIRS=${GITHUB_WORKSPACE}/projects/hip/include;${GITHUB_WORKSPACE}/projects/clr/hipamd/include;${GITHUB_WORKSPACE}/build/clr/hipamd/include" \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${ROCM_SDK}" \ + -DROCM_PATH="${ROCM_SDK}" \ + -DCMAKE_HIP_COMPILER="${ROCM_SDK}/bin/amdclang++" \ + -DCMAKE_HIP_FLAGS="--rocm-device-lib-path=${ROCM_SDK}/lib/llvm/amdgcn/bitcode" \ + -DGPU_TARGETS="${HRR_ARCHS}" \ + -DCMAKE_HIP_ARCHITECTURES="${HRR_ARCHS}" \ + -DHRR_BUILD_PLAYBACK=ON \ + -DHRR_BUILD_TESTS=ON \ + -DHRR_BUILD_INTEGRATION_TESTS=ON + cmake --build build/hrr -j + + - name: ccache statistics + if: always() + run: ccache --show-stats --verbose || ccache --show-stats + + - name: Assert the expected integration cases are present + run: | + # Listing needs no GPU: Catch2 registers cases in static initialisers + # and --list-tests never calls into HIP. Checked here rather than in + # the integration jobs so a compiled-out case fails once, straight + # away, instead of nine times after the fan-out. + python3 -c "import yaml" || python3 -m pip install --no-cache-dir PyYAML==6.0.1 + python3 projects/hrr/tests/scripts/hrr_test_config.py check \ + --binary build/hrr/tests/integration/hrr-integration-tests \ + --config projects/hrr/tests/config/integration.yaml + + - name: Confirm every architecture is embedded + run: | + found=$("${ROCM_SDK}"/lib/llvm/bin/llvm-objdump --offloading \ + build/hrr/tests/integration/hrr-integration-tests \ + | grep -o 'gfx[0-9a-z]*' | sort -u | tr '\n' ' ') + echo "embedded: ${found}" + for arch in ${HRR_ARCHS//;/ }; do + case " ${found} " in + *" ${arch} "*) ;; + *) echo "::error::${arch} missing from the fat binary"; exit 1 ;; + esac + done + + - name: Save ccache + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/develop' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ github.workspace }}/.ccache + key: hrr-ccache-${{ runner.os }}-${{ github.run_id }} + + - name: Upload build + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hrr-build + path: | + build/hrr + build/clr/hipamd/lib + build/rocr-install/lib + retention-days: 3 + + # --------------------------------------------------------------------------- + # Integration check for every family that has a test runner. + # + # This job is the PR check. It runs on a hosted runner and dispatches the + # GPU work as its own workflow run. timeout-minutes does not include queue + # time, and cancelling one job inside this run would cancel the run, so a + # family that has no runner within 30 minutes is cancelled over there and + # reported green here. A device-less runner is handled inside the GPU job + # the same way: skip the suite, upload a skipped result, exit 0. A push to + # develop uses this same result. A suite failure or the execution timeout + # still fails the job. + # --------------------------------------------------------------------------- + integration: + name: "Integration (${{ matrix.family }})" + needs: [gate, ci_config, build] + if: ${{ github.event.inputs.gpu_only != 'true' && needs.gate.outputs.run == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} + runs-on: ubuntu-24.04 + permissions: + actions: write + contents: read + # 30 minutes of queue, then the GPU job's own 30 minute execution cap, + # plus time to publish the JUnit. + timeout-minutes: 80 + strategy: + fail-fast: false + matrix: + include: + - { family: gfx90a, container: true, runner: "${{ needs.ci_config.outputs.gfx90a_runner }}" } + # gfx94x uses the resolved 1-GPU runner: the 8-GPU gfx942 pool + # exposes /dev/kfd only to a job with a container spec. + - { family: gfx94x, runner: "${{ needs.ci_config.outputs.gfx94x_runner }}" } + # The T3 tier needs two visible devices and there is no 2-GPU pool, + # so gfx950 takes an 8-GPU machine; every other family skips T3. + # Do not pin HIP_VISIBLE_DEVICES: on the CCS pool fixed indices can + # be empty even though the job has GPUs. + - { family: gfx950, container: true, runner: "linux-gfx950-8gpu-ccs-ossci-rocm" } + - { family: gfx103x, runner: "${{ needs.ci_config.outputs.gfx103x_runner }}" } + - { family: gfx110x, runner: "${{ needs.ci_config.outputs.gfx110x_runner }}" } + - { family: gfx120x, runner: "${{ needs.ci_config.outputs.gfx120x_runner }}" } + - { family: gfx1151, runner: "${{ needs.ci_config.outputs.gfx1151_runner }}" } + # Not exposed by resolve-ci-config; taken from TheRock's family matrix. + - { family: gfx1150, runner: "linux-gfx1150-gpu-rocm" } + - { family: gfx1153, runner: "linux-gfx1153-gpu-rocm" } + # gfx1250 (MI455) is withheld until linux-mi455-gpu-rocm is healthy. + # If it is wanted sooner, add it as a separate non-blocking job, as + # therock-ci.yml does, so the report job never waits on that queue: + # + # - { family: gfx1250, runner: "linux-mi455-gpu-rocm" } + steps: + - name: Checkout schedule helper + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: projects/hrr/tests/scripts/hrr_ci_schedule.py + sparse-checkout-cone-mode: false + set-safe-directory: true + + - name: Dispatch GPU integration + id: gpu + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + # A branch, not a SHA: workflow_dispatch selects its workflow file + # from this ref. The GPU job then checks out BUILD_SHA, which is + # the commit this run compiled (the pull request's merge commit, or + # the pushed commit). The branch tip can move while the build runs. + REF: ${{ github.head_ref || github.ref_name }} + BUILD_SHA: ${{ github.sha }} + FAMILY: ${{ matrix.family }} + RUNNER: ${{ matrix.runner }} + USE_CONTAINER: ${{ matrix.container && 'true' || 'false' }} + SDK_VERSION: ${{ needs.build.outputs.sdk_version }} + BUILD_WORKSPACE: ${{ needs.build.outputs.workspace }} + BUILD_RUN_ID: ${{ github.run_id }} + IGNORE_TEST_CONFIG: ${{ inputs.ignore_test_config && 'true' || 'false' }} + # run_attempt changes on "Rerun failed jobs". The run id does not, so + # without it the previous GPU run is still listed under this title. + DISPATCH_ID: ${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.family }} + run: | + set -euo pipefail + title="hrr-gpu-${DISPATCH_ID}" + sched="projects/hrr/tests/scripts/hrr_ci_schedule.py" + # Runs already using this title belong to an earlier dispatch. + seen="$(gh run list --workflow hrr-ci.yml --branch "${REF}" --limit 40 \ + --json databaseId,displayTitle,name,event \ + --jq "[.[] | select(.event==\"workflow_dispatch\" and (.displayTitle==\"${title}\" or .name==\"${title}\")) | .databaseId] | join(\" \")" \ + || true)" + set +e + dispatch_out="$(gh workflow run hrr-ci.yml --ref "${REF}" \ + -f gpu_only=true \ + -f "family=${FAMILY}" \ + -f "runner=${RUNNER}" \ + -f "use_container=${USE_CONTAINER}" \ + -f "sdk_version=${SDK_VERSION}" \ + -f "build_workspace=${BUILD_WORKSPACE}" \ + -f "build_run_id=${BUILD_RUN_ID}" \ + -f "build_sha=${BUILD_SHA}" \ + -f "dispatch_id=${DISPATCH_ID}" \ + -f "ignore_test_config=${IGNORE_TEST_CONFIG}" 2>&1)" + dispatch_rc=$? + set -e + printf '%s\n' "${dispatch_out}" + if [ "${dispatch_rc}" -ne 0 ]; then + exit "${dispatch_rc}" + fi + + # gh prints the new run URL once it exists. Prefer that id. A URL + # that was already in the list is the previous attempt. + printed="" + while IFS= read -r line; do + case "${line}" in + *actions/runs/[0-9]*) + printed="${line##*/}" + printed="${printed%%[^0-9]*}" + break + ;; + esac + done < "${GITHUB_WORKSPACE}/gpu-run-id.txt" + echo "run_id=${run_id}" >> "${GITHUB_OUTPUT}" + + action="poll" + while true; do + job_status="$(gh run view "${run_id}" --json jobs \ + --jq "[.jobs[] | select(.name==\"Integration GPU (${FAMILY})\")][0].status // empty" \ + || true)" + run_status="$(gh run view "${run_id}" --json status --jq .status || true)" + action="$(python3 "${sched}" --elapsed "${SECONDS}" \ + --status "${job_status}" --run-status "${run_status}")" + if [ "${action}" != "poll" ]; then + break + fi + sleep 20 + done + + if [ "${action}" = "skip" ]; then + gh run cancel "${run_id}" || true + echo "::notice title=Integration skipped::${FAMILY} was not scheduled within 30 minutes" + python3 "${sched}" --write-skip "integration-${FAMILY}.xml" \ + --message "not scheduled within 30 minutes" + echo "conclusion=skipped" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + exec_deadline=$((SECONDS + 40 * 60)) + while [ "${SECONDS}" -lt "${exec_deadline}" ]; do + run_status="$(gh run view "${run_id}" --json status --jq .status || true)" + if [ "${run_status}" = "completed" ]; then + break + fi + sleep 20 + done + run_status="$(gh run view "${run_id}" --json status --jq .status)" + if [ "${run_status}" != "completed" ]; then + echo "::error::GPU integration ${FAMILY} did not finish" + echo "conclusion=failure" >> "${GITHUB_OUTPUT}" + exit 1 + fi + job_conclusion="$(gh run view "${run_id}" --json jobs \ + --jq "[.jobs[] | select(.name==\"Integration GPU (${FAMILY})\")][0].conclusion // empty" \ + || true)" + run_conclusion="$(gh run view "${run_id}" --json conclusion --jq .conclusion)" + if [ -z "${job_conclusion}" ] || [ "${job_conclusion}" = "skipped" ]; then + job_conclusion="${run_conclusion}" + fi + echo "conclusion=${job_conclusion}" >> "${GITHUB_OUTPUT}" + + - name: Download GPU results + if: ${{ success() && steps.gpu.outputs.conclusion != 'skipped' && steps.gpu.outputs.conclusion != '' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: results-integration-${{ matrix.family }} + path: . + run-id: ${{ steps.gpu.outputs.run_id }} + github-token: ${{ github.token }} + + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: results-integration-${{ matrix.family }} + path: integration-${{ matrix.family }}.xml + if-no-files-found: ignore + retention-days: 3 + + - name: Propagate GPU result + if: ${{ success() && steps.gpu.outputs.conclusion != 'success' && steps.gpu.outputs.conclusion != 'skipped' }} + run: | + echo "::error::GPU integration ${{ matrix.family }} finished ${{ steps.gpu.outputs.conclusion }}" + exit 1 + + - name: Cancel unfinished GPU run + if: cancelled() || failure() + env: + GH_TOKEN: ${{ github.token }} + run: | + if [ -f gpu-run-id.txt ]; then + id="$(cat gpu-run-id.txt)" + status="$(gh run view "${id}" --json status --jq .status || true)" + if [ -n "${status}" ] && [ "${status}" != "completed" ]; then + gh run cancel "${id}" || true + fi + fi + + # --------------------------------------------------------------------------- + # The GPU suite for one family. Started by the integration job above via + # workflow_dispatch, so its queue can be cancelled without cancelling the PR. + # --------------------------------------------------------------------------- + integration_gpu: + # A skipped job does not evaluate its name, so an expression here is shown + # as text on the pull request. The run name carries the family. + name: Integration GPU + if: ${{ github.event.inputs.gpu_only == 'true' }} + # A pull request does not define this input. Keep a real label here so the + # skipped job still parses; gpu_only replaces it with the family runner. + runs-on: ${{ github.event.inputs.gpu_only == 'true' && github.event.inputs.runner || 'ubuntu-24.04' }} + permissions: + actions: read + contents: read + # The suite runs in about 90 seconds. These are scarce single-GPU machines + # and the default timeout is six hours, so a workload that deadlocks on one + # part would hold a runner for the rest of the day and publish no log to + # say why. Fail instead, while the log still gets uploaded. Queue time is + # not part of this budget; the dispatcher owns that. + timeout-minutes: 30 + # The OSSCI pod pools (gfx90a, gfx950) expose /dev/kfd only to a job with + # a container spec: run bare there and hipGetDeviceCount returns 100 with + # no devices. Every other pool is a bare machine and gets no container, + # since a null image runs the job directly on the runner. + container: + image: ${{ github.event.inputs.use_container == 'true' && 'ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98' || null }} + options: >- + --ipc host + --group-add video + --device /dev/kfd + --device /dev/dri + --group-add 110 + --group-add 993 + --group-add 992 + --ulimit memlock=-1:-1 + --security-opt seccomp=unconfined + --env-file /etc/podinfo/gha-gpu-isolation-settings + --user 0:0 + defaults: + run: + shell: bash + steps: + # Only the test runner: this job compiles nothing and takes its binaries + # from the build artifact. It has to come first, because checkout cleans + # the workspace and would delete an artifact unpacked before it. + # build_sha is the parent run's commit. Without it, this checkout follows + # the branch named in the dispatch and can pick up a newer config than + # the binaries. + - name: Checkout test runner + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.build_sha || github.sha }} + sparse-checkout: | + projects/hrr/tests/scripts/run_catch2.py + projects/hrr/tests/scripts/hrr_test_config.py + projects/hrr/tests/scripts/hrr_ci_schedule.py + projects/hrr/tests/config/integration.yaml + sparse-checkout-cone-mode: false + set-safe-directory: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + # Invoked through "python -m" and by script path throughout: on some of + # these self-hosted machines the console-script wrappers cannot execute, + # and the system Python has no ensurepip to build a virtualenv with. + - name: Install ROCm SDK + env: + SDK_VERSION: ${{ inputs.sdk_version }} + run: | + # devel as well as libraries: rocm-sdk init refuses to run without it. + # The version the build job used, not whatever the index has now. + python -m pip install --no-cache-dir \ + --index-url "${ROCM_INDEX_URL}" "rocm[libraries,devel]${SDK_VERSION:+==${SDK_VERSION}}" + # Run as a module where possible. Falling back to the script means + # locating it: some of these machines install into a --user bin that + # is not on PATH, leaving command -v empty. + if ! python -m rocm_sdk init; then + cli="$(command -v rocm-sdk || true)" + if [ -z "${cli}" ]; then + cli="$(python -c 'import site;print(site.USER_BASE)')/bin/rocm-sdk" + fi + python "${cli}" init + fi + echo "ROCM_SDK=$(python -c 'import _rocm_sdk_devel,os;print(os.path.dirname(_rocm_sdk_devel.__file__))')" >> "$GITHUB_ENV" + + # Unpacked under build/ so the tree matches the build machine's layout. + # The suite spawns each workload by re-execing HRR_TEST_EXE, which CMake + # bakes in as an absolute path, so the binary has to sit where it was + # built or every workload dies in execvp with 127. + - name: Download build + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: hrr-build + path: build + run-id: ${{ inputs.build_run_id }} + github-token: ${{ github.token }} + + - name: Restore executable bits + run: chmod +x build/hrr/playback/hrr-playback build/hrr/tests/*/hrr-*tests + + # In a container the workspace is /__w/..., while HRR_TEST_EXE, + # HRR_PLAYBACK_EXE and the SDK RUNPATH were baked on the build machine. + # The tests re-exec /proc/self/exe, because a symlink to the checkout + # created on the container root can sit on a noexec mount: test -x + # succeeds and execve returns EACCES, which the suite reports as 127. + # The tool-cache link is what makes that RUNPATH resolve when a child + # cannot use LD_LIBRARY_PATH. ln -sfn will not replace a real directory. + - name: Recreate build path + env: + BUILD_WORKSPACE: ${{ inputs.build_workspace }} + run: | + # Never delete a real directory. On a shared host the baked path can + # be another runner's checkout. Only create the link when the path + # is missing or is already a symlink. + link_into() { + local path="$1" target="$2" + if [ -e "${path}" ] && [ ! -L "${path}" ]; then + echo "::error::${path} exists and is not a symlink; not deleting it" + exit 1 + fi + mkdir -p "$(dirname "${path}")" + ln -sfn "${target}" "${path}" + } + if [ -n "${BUILD_WORKSPACE}" ] && [ "${BUILD_WORKSPACE}" != "${PWD}" ]; then + link_into "${BUILD_WORKSPACE}" "${PWD}" + fi + # Build-machine SDK path -> this container's tool cache. + if [ -d /__w/_tool ]; then + link_into /home/runner/_work/_tool /__w/_tool + fi + exe="${BUILD_WORKSPACE:-${PWD}}/build/hrr/tests/integration/hrr-integration-tests" + if [ ! -x "${exe}" ]; then + echo "::error::baked test binary is not executable: ${exe} (PWD=${PWD})" + ls -ld "${BUILD_WORKSPACE}" "${PWD}/build/hrr/tests/integration" || true + exit 1 + fi + + # CMake gives the binaries a RUNPATH to the SDK's lib directory, where + # the plain amdhip64 lives. LD_LIBRARY_PATH is searched before RUNPATH, so + # putting the capture build first here is what selects it. + - name: Run integration suite + env: + HRR_TEST_IGNORE_CONFIG: ${{ inputs.ignore_test_config && '1' || '' }} + run: | + # Use a PyYAML that is already importable. Forcing 6.0.1 uninstalls + # 6.0.3 on runners whose site-packages is read-only. + python3 -c "import yaml" || python3 -m pip install --no-cache-dir PyYAML==6.0.1 + # hipamd/lib holds the amdhip64 carrying the capture shims and must + # precede the SDK, which ships a plain build under the same soname. + # rocr-install/lib is the ROCr built from this tree, which hipamd/lib's + # amdhip64 was compiled against; it must precede the SDK's ROCr. + # Keep the runner's existing path: setup-python puts libpython there, + # and replacing it makes python3 fail to start on some GPU machines. + # rocm_sysdeps/lib holds the librocm_sysdeps_* libraries that rocr-install's + # libhsa-runtime64 links (drm, numa, elf); it has no RUNPATH of its own. + export LD_LIBRARY_PATH="${PWD}/build/clr/hipamd/lib:${PWD}/build/rocr-install/lib:${ROCM_SDK}/lib:${ROCM_SDK}/lib/rocm_sysdeps/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + # A labelled runner with no visible device is an infrastructure miss. + # Skip the suite and leave a skipped result so the check stays green. + # Exit 1 is "confirmed no device". Any other probe failure (broken + # ctypes, missing libpython) is not a device count and must not + # skip the GPU suite. + set +e + python3 - <<'PY' + import sys + + try: + import ctypes + hip = ctypes.CDLL("libamdhip64.so.7") + count = ctypes.c_int() + status = hip.hipGetDeviceCount(ctypes.byref(count)) + except Exception as exc: + print(f"GPU probe unavailable: {type(exc).__name__}: {exc}", file=sys.stderr) + sys.exit(2) + print(f"hipGetDeviceCount: status={status}, devices={count.value}") + sys.exit(0 if status == 0 and count.value > 0 else 1) + PY + probe_rc=$? + set -e + if [ "$probe_rc" -eq 1 ]; then + echo "::notice title=Integration skipped::GPU runner has no visible ROCm-capable device" + python3 projects/hrr/tests/scripts/hrr_ci_schedule.py \ + --write-skip "integration-${{ inputs.family }}.xml" \ + --message "GPU runner has no visible ROCm-capable device" + exit 0 + fi + # Keep passing logs to one result line per case. The wrapper captures + # Catch2 and subprocess output, and only emits it when the suite fails. + # Time out inside the job's 30 minutes, so a hang prints the cases + # that finished instead of being cancelled with no output. + # + # Which cases run is the suite config, not a selector written here. + # It lists the hidden [hrr] drivers (T0-T4, the kernel-argument + # regression, the HSA first-touch case) and leaves out the direct + # workloads they spawn, T5, and [hrr-repro]. --arch auto reads the + # GPU this machine actually has, so a rule can name one architecture + # of a wildcard pool. HRR_TEST_IGNORE_CONFIG runs the skipped cases. + python3 projects/hrr/tests/scripts/run_catch2.py \ + --xml integration-${{ inputs.family }}.xml --timeout 1200 \ + --config projects/hrr/tests/config/integration.yaml \ + --os linux --arch auto \ + -- ./build/hrr/tests/integration/hrr-integration-tests + + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: results-integration-${{ inputs.family }} + path: integration-${{ inputs.family }}.xml + retention-days: 3 + + # --------------------------------------------------------------------------- + # Per-test reporting for every job above. Collected centrally because the + # publishing action is a container action and so cannot run on the Windows + # runners. + # --------------------------------------------------------------------------- + report: + name: "Test results" + needs: [gate, unit, integration] + if: ${{ always() && github.event.inputs.gpu_only != 'true' && needs.gate.outputs.run == 'true' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + checks: write + pull-requests: write + steps: + - name: Checkout summary script + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + projects/hrr/tests/scripts/summarize_junit.py + sparse-checkout-cone-mode: false + set-safe-directory: true + + - name: Download all results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: results-* + merge-multiple: true + + - name: Platform / family table + if: always() + run: | + # Written first so it sits above EnricoMi's suite dump on the job + # summary and on the HRR test results check page. + python3 projects/hrr/tests/scripts/summarize_junit.py \ + --dir . \ + --summary "${GITHUB_STEP_SUMMARY}" + + - name: Publish + # Runs even when the table step exits 1 on unreadable XML. + if: always() + uses: EnricoMi/publish-unit-test-result-action@d0a4676d0e0b938bc201470d88276b7c74c712b3 # v2.24.0 + with: + files: "*.xml" + check_name: "HRR test results" + comment_mode: "off" + # Fork pull requests get a read-only token, and creating a check run + # then fails with "Resource not accessible by integration". They keep + # the job summary, which needs no token. + check_run: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork) }} + job_summary: true + report_individual_runs: true + deduplicate_classes_by_file_name: true + + # --------------------------------------------------------------------------- + # Single required-check target. Passes when the path gate skipped everything + # and when every job that ran succeeded; fails on any failure or cancellation, + # and on a skip the gate does not explain. Fork PRs skip build and integration + # on purpose, so those skips are accepted there. + # --------------------------------------------------------------------------- + hrr_ci_summary: + name: "HRR CI Summary" + if: ${{ always() && github.event.inputs.gpu_only != 'true' }} + needs: [gate, codegen, unit, ci_config, build, integration, report] + runs-on: ubuntu-24.04 + steps: + - name: Evaluate + env: + NEEDS_JSON: ${{ toJSON(needs) }} + IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork }} + run: | + python3 - <<'PY' + import json, os, sys + + needs = json.loads(os.environ["NEEDS_JSON"]) + result = {name: job["result"] for name, job in needs.items()} + print(json.dumps(result, indent=2)) + + if result["gate"] != "success": + print("::error::path gate did not complete: " + result["gate"]) + sys.exit(1) + if needs["gate"]["outputs"].get("run") != "true": + print("HRR CI skipped: the pull request changes no HRR path.") + sys.exit(0) + + fork = os.environ["IS_FORK"] == "true" + optional = {"build", "integration"} if fork else set() + bad = [n for n, r in result.items() + if r not in ("success",) and not (r == "skipped" and n in optional)] + if bad: + print("::error::HRR CI jobs not successful: " + ", ".join(sorted(bad))) + sys.exit(1) + PY From c9a77deeefe65344acd320b848a32abc9ab73303 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 12 Oct 2026 12:01:38 +0200 Subject: [PATCH 02/89] ci(hrr): address review comments on the HRR CI workflow - Gate: list previous_filename for renamed files so moving a file out of an HRR path still counts as an HRR change. - Dispatcher: select the dispatched job by its real name, "Integration GPU", so job status and conclusion are read instead of falling back to the run. - GPU probe: skip only for hipErrorNoDevice (100) or a zero device count; any other hipGetDeviceCount status continues to the suite and fails visibly. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 49b9290eb442..ad1170dd4e68 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -122,9 +122,11 @@ jobs: if [ -n "${PR_NUMBER}" ]; then reason="the pull request changes an HRR path" # The files API lists at most 3000 files; a PR that large is treated - # as relevant rather than guessed at. + # as relevant rather than guessed at. A rename lists only the new + # path in filename, so previous_filename is listed as well: moving a + # file out of an HRR path is an HRR change. gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files" \ - --jq '.[].filename' > changed-files.txt + --jq '.[] | .filename, (.previous_filename // empty)' > changed-files.txt if ! python3 - changed-files.txt <<'PY' import sys @@ -770,7 +772,7 @@ jobs: action="poll" while true; do job_status="$(gh run view "${run_id}" --json jobs \ - --jq "[.jobs[] | select(.name==\"Integration GPU (${FAMILY})\")][0].status // empty" \ + --jq "[.jobs[] | select(.name==\"Integration GPU\")][0].status // empty" \ || true)" run_status="$(gh run view "${run_id}" --json status --jq .status || true)" action="$(python3 "${sched}" --elapsed "${SECONDS}" \ @@ -805,7 +807,7 @@ jobs: exit 1 fi job_conclusion="$(gh run view "${run_id}" --json jobs \ - --jq "[.jobs[] | select(.name==\"Integration GPU (${FAMILY})\")][0].conclusion // empty" \ + --jq "[.jobs[] | select(.name==\"Integration GPU\")][0].conclusion // empty" \ || true)" run_conclusion="$(gh run view "${run_id}" --json conclusion --jq .conclusion)" if [ -z "${job_conclusion}" ] || [ "${job_conclusion}" = "skipped" ]; then @@ -1011,8 +1013,9 @@ jobs: export LD_LIBRARY_PATH="${PWD}/build/clr/hipamd/lib:${PWD}/build/rocr-install/lib:${ROCM_SDK}/lib:${ROCM_SDK}/lib/rocm_sysdeps/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" # A labelled runner with no visible device is an infrastructure miss. # Skip the suite and leave a skipped result so the check stays green. - # Exit 1 is "confirmed no device". Any other probe failure (broken - # ctypes, missing libpython) is not a device count and must not + # Exit 1 is "confirmed no device": hipErrorNoDevice (100) or a count + # of zero. Any other probe failure (broken ctypes, missing libpython, + # an init or ABI error status) is not a device count and must not # skip the GPU suite. set +e python3 - <<'PY' @@ -1027,7 +1030,11 @@ jobs: print(f"GPU probe unavailable: {type(exc).__name__}: {exc}", file=sys.stderr) sys.exit(2) print(f"hipGetDeviceCount: status={status}, devices={count.value}") - sys.exit(0 if status == 0 and count.value > 0 else 1) + if status == 0 and count.value > 0: + sys.exit(0) + # hipErrorNoDevice. Other statuses fall through to the suite, which + # then fails visibly instead of being reported as a skip. + sys.exit(1 if status == 100 or (status == 0 and count.value == 0) else 3) PY probe_rc=$? set -e From d0044bc9b1e483b7ce75b24bd0e8d67c6a9d1a33 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 12 Oct 2026 13:42:32 +0200 Subject: [PATCH 03/89] ci(hrr): probe for a GPU with the SDK runtime, not the in-tree ROCr CLR returns hipErrorNoDevice when hsa_init fails, so probing through the in-tree ROCr made a broken ROCr look like a runner without a GPU and skip the suite green. Probe with the SDK's amdhip64 and ROCr; the suite still runs against the in-tree ones and now fails if they do not initialise. Also reword the SDK pin comment: it no longer protects ROCr. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index ad1170dd4e68..077b6f2d6f1e 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -427,8 +427,10 @@ jobs: outputs: workspace: ${{ github.workspace }} # The nightly index moves every day. The integration jobs install exactly - # this version so the capture amdhip64 never runs against a newer ROCr - # than the one it was built with, however long a GPU queue is. + # this version so the toolchain (comgr, the device libraries) and the + # sysdeps sonames the in-tree ROCr and CLR use never drift from what they + # were built with, however long a GPU queue is. ROCr itself comes from + # this tree, not the SDK. sdk_version: ${{ steps.sdk.outputs.version }} runs-on: azure-linux-scale-rocm env: @@ -1010,15 +1012,22 @@ jobs: # and replacing it makes python3 fail to start on some GPU machines. # rocm_sysdeps/lib holds the librocm_sysdeps_* libraries that rocr-install's # libhsa-runtime64 links (drm, numa, elf); it has no RUNPATH of its own. - export LD_LIBRARY_PATH="${PWD}/build/clr/hipamd/lib:${PWD}/build/rocr-install/lib:${ROCM_SDK}/lib:${ROCM_SDK}/lib/rocm_sysdeps/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + base_ld="${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + export LD_LIBRARY_PATH="${PWD}/build/clr/hipamd/lib:${PWD}/build/rocr-install/lib:${ROCM_SDK}/lib:${ROCM_SDK}/lib/rocm_sysdeps/lib${base_ld}" + # The probe below uses the SDK's own amdhip64 and ROCr instead. CLR + # also returns hipErrorNoDevice when hsa_init fails, so probing with + # the in-tree ROCr would turn a broken in-tree ROCr into a green skip. + probe_ld="${ROCM_SDK}/lib:${ROCM_SDK}/lib/rocm_sysdeps/lib${base_ld}" # A labelled runner with no visible device is an infrastructure miss. + # The probe runs against the SDK's runtime, so only the machine can + # make it report no device; the suite below runs against this tree's. # Skip the suite and leave a skipped result so the check stays green. # Exit 1 is "confirmed no device": hipErrorNoDevice (100) or a count # of zero. Any other probe failure (broken ctypes, missing libpython, # an init or ABI error status) is not a device count and must not # skip the GPU suite. set +e - python3 - <<'PY' + LD_LIBRARY_PATH="${probe_ld}" python3 - <<'PY' import sys try: From d0cffe5ede4024d5973cb436deb4fcad992b80b9 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 13 Oct 2026 12:34:03 +0200 Subject: [PATCH 04/89] ci(hrr): move HRR CI logic into tested Python scripts The HRR workflow carried its path gate, generated-source checks, GPU dispatch and summary as inline shell. Move each into a script under .github/scripts with pytest tests, as the other rocm-systems CI scripts do: - hrr_ci_configure.py decides whether the CPU checks and the GPU suite run, and on which GPU families. Families follow TheRock's trigger policy (amdgpu_family_matrix.py) and its ci:gfx... labels; ci:hrr-capture asks for the GPU suite on any pull request. - hrr_check_sources.py checks the generated HRR sources, API-ID compatibility with the base commit, and the API-matrix expectations. - hrr_capture_build.py builds an amdhip64 with HRR capture compiled in on top of the ROCm artifacts that rocm-systems' TheRock CI built for a recent develop commit, then builds the HRR suite against it. This is interim until TheRock offers a capture-enabled build variant. - hrr_ci_report.py writes one summary of every HRR job and the per-platform, per-family results table. Signed-off-by: Daniel Olsson --- .github/scripts/hrr_capture_build.py | 285 +++++++++++++++ .github/scripts/hrr_check_sources.py | 201 +++++++++++ .github/scripts/hrr_ci_configure.py | 330 ++++++++++++++++++ .github/scripts/hrr_ci_report.py | 163 +++++++++ .../scripts/tests/hrr_capture_build_test.py | 153 ++++++++ .../scripts/tests/hrr_check_sources_test.py | 160 +++++++++ .../scripts/tests/hrr_ci_configure_test.py | 204 +++++++++++ .github/scripts/tests/hrr_ci_report_test.py | 90 +++++ 8 files changed, 1586 insertions(+) create mode 100644 .github/scripts/hrr_capture_build.py create mode 100644 .github/scripts/hrr_check_sources.py create mode 100644 .github/scripts/hrr_ci_configure.py create mode 100644 .github/scripts/hrr_ci_report.py create mode 100644 .github/scripts/tests/hrr_capture_build_test.py create mode 100644 .github/scripts/tests/hrr_check_sources_test.py create mode 100644 .github/scripts/tests/hrr_ci_configure_test.py create mode 100644 .github/scripts/tests/hrr_ci_report_test.py diff --git a/.github/scripts/hrr_capture_build.py b/.github/scripts/hrr_capture_build.py new file mode 100644 index 000000000000..1d232f147755 --- /dev/null +++ b/.github/scripts/hrr_capture_build.py @@ -0,0 +1,285 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Build an amdhip64 with HRR capture compiled in, for the HRR GPU suite. + +Interim: TheRock has no capture-enabled build yet. Once rocm-systems CI can +request TheRock's hrr-capture build variant, hrr-gpu.yml calls TheRock's +workflows instead and this script is deleted. + +Release builds compile capture out (HIP_HRR_CAPTURE_ENABLED undefined). This +script installs the ROCm artifacts that rocm-systems' own TheRock CI built for +a recent develop commit, rebuilds only CLR from this checkout with capture +enabled, installs it over those artifacts, and builds projects/hrr on top. The +GPU jobs then install the same artifacts plus this amdhip64, so build and test +use the same libraries, and the binaries' RUNPATH needs no LD_LIBRARY_PATH. + + fetch-base Pick the base run and install the build-time artifacts. + build Build CLR with capture, install it into the base, build HRR. + verify Check the suite's case list and the embedded GPU targets. + install-base On a GPU machine: install the run-time artifacts of the + recorded base run and the capture-enabled amdhip64. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Callable, Sequence + +Runner = Callable[[Sequence[str]], None] + +BASE_REPO = "ROCm/rocm-systems" +BASE_WORKFLOW = "therock-multi-arch-ci.yml" +BASE_BRANCH = "develop" +# The generic (target-neutral) artifacts are the same in every family group. +BASE_ARTIFACT_GROUP = "gfx94X-dcgpu" + +# Artifacts needed to build CLR and projects/hrr. +BUILD_ARTIFACTS = ( + "amd-llvm_run", "amd-llvm_lib", "amd-llvm_dev", + "base_run", "base_lib", "base_dev", + "sysdeps_lib", "sysdeps_dev", + "core-runtime_run", "core-runtime_lib", "core-runtime_dev", + "core-hip_run", "core-hip_lib", "core-hip_dev", +) + +# Artifacts the GPU suite loads at run time. +RUNTIME_ARTIFACTS = ( + "amd-llvm_lib", + "base_run", "base_lib", + "sysdeps_lib", + "core-runtime_run", "core-runtime_lib", + "core-hip_lib", +) + +CAPTURE_LIB_GLOB = "libamdhip64.so*" + +# What the GPU suite runs. The host unit suite is built and run by hrr-ci.yml. +HRR_TARGETS = ("hrr-playback", "hrr-integration-tests", "hrr-test-dispatch-tool") + +# Uploaded by hrr-gpu.yml's build job, relative to --build. +EXECUTABLES = ( + "hrr/playback/hrr-playback", + "hrr/tests/integration/hrr-integration-tests", +) + + +def run(cmd: Sequence[str]) -> None: + print("+ " + " ".join(str(c) for c in cmd), flush=True) + subprocess.run([str(c) for c in cmd], check=True) + + +def pick_base_run(finder_json: dict) -> dict: + """The base run from find_latest_artifacts.py --json output.""" + if finder_json.get("status") != "found" or not finder_json.get("results"): + raise SystemExit(f"no {BASE_BRANCH} run of {BASE_WORKFLOW} has the needed artifacts") + result = finder_json["results"][0] + return { + "repo": result["github_repository_name"], + "run_id": str(result["workflow_run_id"]), + "commit": result["git_commit_sha"], + "run_url": result["workflow_run_html_url"], + "artifact_group": result["artifact_group"], + } + + +def find_base_run(therock: Path) -> dict: + cmd = [ + sys.executable, therock / "build_tools" / "find_latest_artifacts.py", + "--repo", BASE_REPO, + "--workflow", BASE_WORKFLOW, + "--branch", BASE_BRANCH, + "--artifact-group", BASE_ARTIFACT_GROUP, + "--json", + ] + for artifact in BUILD_ARTIFACTS: + cmd += ["--require-artifact", f"{artifact}_generic"] + print("+ " + " ".join(str(c) for c in cmd), flush=True) + # Only stdout is captured: it carries the JSON, stderr the finder's log. + out = subprocess.run([str(c) for c in cmd], check=True, stdout=subprocess.PIPE, text=True) + return pick_base_run(json.loads(out.stdout)) + + +def fetch_command(therock: Path, base: dict, dist: Path, artifacts: Sequence[str]) -> list: + return [ + sys.executable, therock / "build_tools" / "fetch_artifacts.py", + "--run-id", base["run_id"], + "--run-github-repo", base["repo"], + "--artifact-group", base["artifact_group"], + "--output-dir", dist, + "--flatten", + *artifacts, + ] + + +def clr_configure_command(src: Path, build: Path, dist: Path) -> list: + # HIP_HRR_CAPTURE_ENABLED is what compiles capture into amdhip64; CLR has + # no CMake option for it. + return [ + "cmake", "-S", src / "projects" / "clr", "-B", build / "clr", "-GNinja", "-Wno-dev", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CXX_FLAGS=-DHIP_HRR_CAPTURE_ENABLED", + f"-DCMAKE_PREFIX_PATH={dist}", + f"-DCMAKE_INSTALL_PREFIX={dist}", + # The artifacts use lib/; GNUInstallDirs would pick lib64 on this image. + "-DCMAKE_INSTALL_LIBDIR=lib", + # libhiprtc sets no RUNPATH of its own; comgr and ROCr sit next to it. + "-DCMAKE_INSTALL_RPATH=$ORIGIN", + f"-DROCM_PATH={dist}", + f"-DHIP_COMMON_DIR={src / 'projects' / 'hip'}", + "-DHIP_PLATFORM=amd", + "-DCLR_BUILD_HIP=ON", + "-DCLR_BUILD_OCL=OFF", + "-DHIPCC_BIN_DIR=", + ] + + +def hrr_configure_command(src: Path, build: Path, dist: Path, archs: str) -> list: + llvm = dist / "lib" / "llvm" + # One compiler for C, C++ and HIP, as TheRock builds HRR: host objects from + # the system compiler are not position independent, and the HIP link is. + return [ + "cmake", "-S", src / "projects" / "hrr", "-B", build / "hrr", "-GNinja", "-Wno-dev", + "-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_PREFIX_PATH={dist}", + f"-DROCM_PATH={dist}", + f"-DCMAKE_C_COMPILER={llvm / 'bin' / 'clang'}", + f"-DCMAKE_CXX_COMPILER={llvm / 'bin' / 'clang++'}", + f"-DCMAKE_HIP_COMPILER={llvm / 'bin' / 'clang++'}", + f"-DCMAKE_HIP_FLAGS=--rocm-device-lib-path={llvm / 'amdgcn' / 'bitcode'}", + f"-DGPU_TARGETS={archs}", + f"-DCMAKE_HIP_ARCHITECTURES={archs}", + "-DHRR_BUILD_PLAYBACK=ON", + "-DHRR_BUILD_TESTS=ON", + "-DHRR_BUILD_INTEGRATION_TESTS=ON", + ] + + +def remove_amdhip64(lib_dir: Path) -> None: + """Remove the release amdhip64 so only the capture-enabled one remains.""" + for path in lib_dir.glob(CAPTURE_LIB_GLOB): + path.unlink() + + +def copy_amdhip64(src_dir: Path, dst_dir: Path) -> list[str]: + """Copy libamdhip64.so* from src_dir to dst_dir, keeping symlinks.""" + dst_dir.mkdir(parents=True, exist_ok=True) + copied = [] + for path in sorted(src_dir.glob(CAPTURE_LIB_GLOB)): + target = dst_dir / path.name + if target.is_symlink() or target.exists(): + target.unlink() + if path.is_symlink(): + os.symlink(os.readlink(path), target) + else: + shutil.copy2(path, target) + copied.append(path.name) + if not copied: + raise SystemExit(f"no {CAPTURE_LIB_GLOB} in {src_dir}") + return copied + + +def embedded_targets(objdump_output: str) -> set[str]: + return set(re.findall(r"\bgfx[0-9a-f]+\b", objdump_output)) + + +def missing_targets(archs: str, objdump_output: str) -> list[str]: + found = embedded_targets(objdump_output) + return [a for a in archs.split(";") if a and a not in found] + + +def cmd_fetch_base(args: argparse.Namespace, runner: Runner = run) -> int: + base = json.loads(args.base_json.read_text()) if args.base_json.is_file() else None + if base is None: + base = find_base_run(args.therock) + args.base_json.parent.mkdir(parents=True, exist_ok=True) + args.base_json.write_text(json.dumps(base, indent=2) + "\n") + print(f"Base: {base['repo']} run {base['run_id']} at {base['commit']}") + runner(fetch_command(args.therock, base, args.dist, BUILD_ARTIFACTS)) + return 0 + + +def cmd_build(args: argparse.Namespace, runner: Runner = run) -> int: + runner(clr_configure_command(args.src, args.build, args.dist)) + runner(["cmake", "--build", args.build / "clr"]) + remove_amdhip64(args.dist / "lib") + runner(["cmake", "--install", args.build / "clr"]) + copy_amdhip64(args.dist / "lib", args.capture_lib) + runner(hrr_configure_command(args.src, args.build, args.dist, args.archs)) + runner(["cmake", "--build", args.build / "hrr", "--target", *HRR_TARGETS]) + return 0 + + +def cmd_verify(args: argparse.Namespace) -> int: + exe = args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests" + run([ + sys.executable, args.src / "projects/hrr/tests/scripts/hrr_test_config.py", "check", + "--binary", exe, + "--config", args.src / "projects/hrr/tests/config/integration.yaml", + ]) + objdump = args.dist / "lib" / "llvm" / "bin" / "llvm-objdump" + out = subprocess.run( + [str(objdump), "--offloading", str(exe)], check=True, capture_output=True, text=True + ).stdout + missing = missing_targets(args.archs, out) + print("embedded: " + " ".join(sorted(embedded_targets(out)))) + if missing: + print(f"::error::missing from the fat binary: {' '.join(missing)}") + return 1 + return 0 + + +def restore_executable_bits(build: Path) -> None: + """Artifact downloads drop the executable bit.""" + for rel in EXECUTABLES: + path = build / rel + path.chmod(path.stat().st_mode | 0o111) + + +def cmd_install_base(args: argparse.Namespace, runner: Runner = run) -> int: + base = json.loads(args.base_json.read_text()) + print(f"Base: {base['repo']} run {base['run_id']} at {base['commit']}") + runner(fetch_command(args.therock, base, args.dist, RUNTIME_ARTIFACTS)) + remove_amdhip64(args.dist / "lib") + copy_amdhip64(args.capture_lib, args.dist / "lib") + restore_executable_bits(args.build) + return 0 + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("command", choices=("fetch-base", "build", "verify", "install-base")) + parser.add_argument("--therock", type=Path, default=Path("therock")) + parser.add_argument("--src", type=Path, default=Path.cwd()) + parser.add_argument("--dist", type=Path, default=Path("dist")) + parser.add_argument("--build", type=Path, default=Path("build")) + parser.add_argument("--capture-lib", type=Path, default=Path("build/capture-lib")) + parser.add_argument("--base-json", type=Path, default=Path("build/base.json")) + parser.add_argument("--archs", default=os.environ.get("HRR_ARCHS", "")) + args = parser.parse_args(argv) + for name in ("therock", "src", "dist", "build", "capture_lib", "base_json"): + setattr(args, name, getattr(args, name).resolve()) + if args.command in ("build", "verify") and not args.archs: + parser.error("--archs (or HRR_ARCHS) is required") + return args + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + return { + "fetch-base": cmd_fetch_base, + "build": cmd_build, + "verify": cmd_verify, + "install-base": cmd_install_base, + }[args.command](args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/hrr_check_sources.py b/.github/scripts/hrr_check_sources.py new file mode 100644 index 000000000000..8d6c0fc15960 --- /dev/null +++ b/.github/scripts/hrr_check_sources.py @@ -0,0 +1,201 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Check the HRR sources that are generated from the HIP API headers. + + generated Regenerate hrr_api_args.h and the capture and playback dispatch + sources and compare them with the checked-in copies. + api-compat Check that the regenerated API IDs stay compatible with the + header on the base commit (IDs are part of the archive format). + api-matrix Re-derive the API class manifest and compare the generated + test expectations with the checked-in header. + all All three, reporting every failure. + +Everything is written under --out; checked-in files are never modified. +""" + +from __future__ import annotations + +import argparse +import difflib +import subprocess +import sys +from pathlib import Path +from typing import Callable, Sequence + +Runner = Callable[[Sequence[str]], int] + +API_TRACE = "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp" +PUBLIC_HEADER = "projects/hip/include/hip/hip_runtime_api.h" +MANUAL_CAPTURE = "projects/clr/hipamd/src/hrr/hip_capture.cpp" +GENERATOR = "projects/hrr/tools/gen_hrr_api_args.py" +COMPAT_CHECKER = "projects/hrr/tools/check_hrr_api_compat.py" +MATRIX_TOOLS = "projects/hrr/tools/api-matrix" +API_ARGS_HEADER = "projects/hrr/include/hrr/hrr_api_args.h" +MATRIX_EXPECTATIONS = "projects/hrr/tests/integration/hrr_api_matrix_expectations.h" + +# Checked-in file -> name of the regenerated copy under --out. +GENERATED_FILES = { + API_ARGS_HEADER: "include/hrr/hrr_api_args.h", + "projects/clr/hipamd/src/hrr/hip_capture_generated.cpp": "hip_capture_generated.cpp", + "projects/hrr/playback/hip_playback_generated.cpp": "hip_playback_generated.cpp", +} + + +def run(cmd: Sequence[str]) -> int: + print("+ " + " ".join(cmd), flush=True) + return subprocess.run(list(cmd)).returncode + + +def error(message: str) -> None: + print(f"::error::{message}", flush=True) + + +def diff_files(expected: Path, actual: Path) -> list[str]: + """Unified diff of two text files, ignoring line-ending differences.""" + a = expected.read_text(encoding="utf-8").splitlines() + b = actual.read_text(encoding="utf-8").splitlines() + return list( + difflib.unified_diff(a, b, fromfile=str(expected), tofile=str(actual), lineterm="") + ) + + +def check_generated(root: Path, out: Path, runner: Runner = run) -> bool: + (out / "include" / "hrr").mkdir(parents=True, exist_ok=True) + rc = runner( + [ + sys.executable, str(root / GENERATOR), + "--input", str(root / API_TRACE), + "--public-header", str(root / PUBLIC_HEADER), + "--output-header", str(out / GENERATED_FILES[API_ARGS_HEADER]), + "--output-capture", str(out / "hip_capture_generated.cpp"), + "--output-playback", str(out / "hip_playback_generated.cpp"), + "--check-hrr-coverage", + "--manual-capture-source", str(root / MANUAL_CAPTURE), + "--silent", + ] + ) + if rc != 0: + error(f"{GENERATOR} failed (exit {rc})") + return False + + stale = [] + for tracked, generated in GENERATED_FILES.items(): + lines = diff_files(root / tracked, out / generated) + if lines: + print("\n".join(lines)) + stale.append(tracked) + if stale: + error( + "checked-in HRR generated sources are stale: " + + ", ".join(stale) + + f". Regenerate them with {GENERATOR} --check-hrr-coverage." + ) + return False + return True + + +def base_header(root: Path, base_sha: str) -> str | None: + """hrr_api_args.h at base_sha, or None when the base has no copy.""" + proc = subprocess.run( + ["git", "-C", str(root), "show", f"{base_sha}:{API_ARGS_HEADER}"], + capture_output=True, + text=True, + ) + return proc.stdout if proc.returncode == 0 else None + + +def resolve_base(root: Path, base_sha: str) -> str: + """The commit to compare API IDs against: base_sha, else the parent commit.""" + if base_sha and set(base_sha) != {"0"}: + return base_sha + return subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD^"], + check=True, capture_output=True, text=True, + ).stdout.strip() + + +def check_api_compat( + root: Path, + out: Path, + base_sha: str, + runner: Runner = run, + read_base: Callable[[Path, str], str | None] = base_header, +) -> bool: + current = out / GENERATED_FILES[API_ARGS_HEADER] + if not current.is_file(): + error("api-compat needs the regenerated header; run the generated check first") + return False + text = read_base(root, base_sha) + if text is None: + print(f"{API_ARGS_HEADER} does not exist at {base_sha}; nothing to compare") + return True + base = out / "base_hrr_api_args.h" + base.write_text(text, encoding="utf-8") + rc = runner( + [ + sys.executable, str(root / COMPAT_CHECKER), + "--base-header", str(base), + "--current-header", str(current), + ] + ) + if rc != 0: + error(f"HRR API IDs are not compatible with {base_sha}; see {COMPAT_CHECKER}") + return False + return True + + +def check_api_matrix(root: Path, out: Path, runner: Runner = run) -> bool: + tools = root / MATRIX_TOOLS + matrix_out = out / "api-matrix" + matrix_out.mkdir(parents=True, exist_ok=True) + manifest = matrix_out / "api_classes.json" + emitted = matrix_out / "hrr_api_matrix_expectations.h" + steps = [ + [sys.executable, str(tools / "derive_manifest.py"), "--quiet", "--output", str(manifest)], + [sys.executable, str(tools / "check_matrix.py"), "--check-workloads"], + [ + sys.executable, str(tools / "check_matrix.py"), + "--manifest", str(manifest), "--emit-cxx", str(emitted), + ], + ] + for step in steps: + rc = runner(step) + if rc != 0: + error(f"{Path(step[1]).name} failed (exit {rc})") + return False + lines = diff_files(root / MATRIX_EXPECTATIONS, emitted) + if lines: + print("\n".join(lines)) + error( + f"{MATRIX_EXPECTATIONS} is stale; run derive_manifest.py, then " + f"check_matrix.py --emit-cxx, in {MATRIX_TOOLS}" + ) + return False + return True + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("check", choices=("generated", "api-compat", "api-matrix", "all")) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--out", type=Path, default=Path("build/hrr-codegen")) + parser.add_argument("--base-sha", default="") + args = parser.parse_args(argv) + + root = args.root.resolve() + out = args.out.resolve() + out.mkdir(parents=True, exist_ok=True) + + ok = True + if args.check in ("generated", "all"): + ok &= check_generated(root, out) + if args.check in ("api-compat", "all"): + ok &= check_api_compat(root, out, resolve_base(root, args.base_sha)) + if args.check in ("api-matrix", "all"): + ok &= check_api_matrix(root, out) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/hrr_ci_configure.py b/.github/scripts/hrr_ci_configure.py new file mode 100644 index 000000000000..c780f7918028 --- /dev/null +++ b/.github/scripts/hrr_ci_configure.py @@ -0,0 +1,330 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Decide which HRR CI jobs run and on which GPU families. + +Writes these GitHub Actions outputs: + + run_cpu "true" when the generated-source, script and unit jobs run. + run_gpu "true" when the capture-enabled build and the GPU suite run. + families JSON list of {"family", "runner", "gfx_targets"} for the GPU suite. + reason One line explaining the decision, shown in the summary. + +The GPU suite needs an amdhip64 with HRR capture compiled in, which only the +HRR GPU workflow builds. It runs automatically when a change touches capture +or replay (CAPTURE_PATHS), on any pull request labelled ci:hrr-capture, on +pushes to develop that touch capture, nightly, and on workflow_dispatch. + +GPU families follow TheRock's trigger policy (amdgpu_family_matrix.py): +pull requests test the presubmit families, pushes the postsubmit families and +the nightly schedule the nightly families, each limited to HRR_FAMILIES. A +ci:gfx... label adds that family, the same label TheRock CI uses. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Iterable + +CAPTURE_LABEL = "ci:hrr-capture" + +# A change under any of these runs the generated-source, script and unit jobs. +CPU_PATHS = ( + "projects/hrr/", + "projects/clr/hipamd/src/hrr/", + "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp", + "projects/hip/include/hip/hip_runtime_api.h", + ".github/workflows/hrr-ci.yml", + ".github/workflows/hrr-gpu.yml", + ".github/scripts/hrr_", + ".github/scripts/tests/hrr_", +) + +# hrr-skills.yml covers these. +EXCLUDED_PATHS = ("projects/hrr/skills/", "projects/hrr/README.md") + +# A change under any of these also runs the capture-enabled build and the GPU +# suite. hip_runtime_api.h is deliberately absent: almost every HIP API change +# touches it, and the generated-source job already checks HRR against it. +CAPTURE_PATHS = ( + "projects/clr/hipamd/src/hrr/", + "projects/clr/hipamd/src/hip_context.cpp", + "projects/clr/hipamd/src/hip_code_object.cpp", + "projects/clr/hipamd/src/hip_code_object.hpp", + "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp", + "projects/hrr/CMakeLists.txt", + "projects/hrr/include/", + "projects/hrr/playback/", + "projects/hrr/tests/CMakeLists.txt", + "projects/hrr/tests/include/", + "projects/hrr/tests/integration/", + "projects/hrr/tests/config/integration.yaml", + "projects/hrr/tests/scripts/run_catch2.py", + "projects/hrr/tests/scripts/hrr_test_config.py", + "projects/hrr/tests/scripts/summarize_junit.py", + ".github/workflows/hrr-ci.yml", + ".github/workflows/hrr-gpu.yml", + ".github/scripts/hrr_capture_build.py", + ".github/scripts/hrr_ci_configure.py", + ".github/scripts/hrr_ci_report.py", +) + +# TheRock family keys the HRR fat binary covers (HRR_ARCHS in hrr-gpu.yml). +# gfx125x is built but only tested on request: its runner pool is not reliably +# online. +HRR_FAMILIES = ( + "gfx90a", + "gfx94x", + "gfx950", + "gfx103x", + "gfx110x", + "gfx1150", + "gfx1151", + "gfx1153", + "gfx120x", + "gfx125x", +) +ON_REQUEST_FAMILIES = ("gfx125x",) +# TheRock tests gfx950 only for submodule bumps; HRR also tests it nightly. +NIGHTLY_EXTRA_FAMILIES = ("gfx950",) +# The T3 tier needs two visible devices, so these families take TheRock's +# multi-GPU runner when it has one. +MULTI_GPU_FAMILIES = ("gfx950",) + +TRIGGER_FOR_EVENT = { + "pull_request": "presubmit", + "push": "postsubmit", + "schedule": "nightly", + "workflow_dispatch": "nightly", +} + +# The pull request files API returns at most this many files. A larger change +# is treated as touching everything rather than guessed at. +MAX_LISTED_FILES = 3000 + + +@dataclass +class Decision: + run_cpu: bool + run_gpu: bool + reason: str + families: list[dict] = field(default_factory=list) + + +def _matches(path: str, prefixes: Iterable[str]) -> bool: + return any(path == p or path.startswith(p) for p in prefixes) + + +def relevant_paths(paths: Iterable[str]) -> list[str]: + return [p for p in paths if not _matches(p, EXCLUDED_PATHS)] + + +def touches_cpu(paths: Iterable[str]) -> bool: + return any(_matches(p, CPU_PATHS) for p in relevant_paths(paths)) + + +def touches_capture(paths: Iterable[str]) -> bool: + return any(_matches(p, CAPTURE_PATHS) for p in relevant_paths(paths)) + + +def labelled_families(labels: Iterable[str], matrix: dict) -> list[str]: + """Family keys named by ci:gfx... labels, e.g. ci:gfx110x or ci:gfx110X-all.""" + wanted = [] + for label in labels: + label = label.strip().lower() + if not label.startswith("ci:gfx"): + continue + name = label[len("ci:"):] + for key, config in matrix.items(): + linux = config.get("linux", {}) + if name in (key.lower(), str(linux.get("family", "")).lower()): + wanted.append(key) + return wanted + + +def runner_for( + linux: dict, pick_weighted: Callable[[list, str], str], multi_gpu: bool = False +) -> str: + if multi_gpu: + labels = linux.get("test-runs-on-multi-gpu-labels") + if labels: + return pick_weighted(labels, "hrr-multi-gpu") + if linux.get("test-runs-on-multi-gpu"): + return linux["test-runs-on-multi-gpu"] + labels = linux.get("test-runs-on-labels") + if labels: + return pick_weighted(labels, "hrr") + return linux.get("test-runs-on", "") + + +def select_families( + trigger: str, + requested: list[str], + labels: list[str], + matrix: dict, + pick_weighted: Callable[[list, str], str], +) -> list[dict]: + """GPU families to test, in HRR_FAMILIES order, skipping any without a runner.""" + if requested: + chosen = set(requested) + else: + chosen = { + key + for key in HRR_FAMILIES + if key not in ON_REQUEST_FAMILIES + and trigger in matrix.get(key, {}).get("linux", {}).get("tests_on_trigger", []) + } + if trigger == "nightly": + chosen |= set(NIGHTLY_EXTRA_FAMILIES) + chosen |= set(labelled_families(labels, matrix)) + + families = [] + for key in HRR_FAMILIES: + if key not in chosen: + continue + linux = matrix.get(key, {}).get("linux", {}) + runner = runner_for(linux, pick_weighted, key in MULTI_GPU_FAMILIES) + if not runner: + continue + families.append( + { + "family": key, + "runner": runner, + "gfx_targets": ",".join(linux.get("fetch-gfx-targets", [])), + } + ) + return families + + +def decide( + event: str, + paths: list[str] | None, + labels: list[str], + is_fork: bool, + requested: list[str], + matrix: dict, + pick_weighted: Callable[[list, str], str], +) -> Decision: + """paths is None when the change set could not be listed in full.""" + trigger = TRIGGER_FOR_EVENT.get(event) + if trigger is None: + return Decision(False, False, f"event {event} does not run HRR CI") + + if event in ("schedule", "workflow_dispatch"): + run_cpu, capture, why = True, True, f"{event} run" + elif paths is None: + run_cpu, capture, why = True, True, "change set too large to list" + else: + run_cpu = touches_cpu(paths) + capture = touches_capture(paths) + why = "the change touches capture or replay" if capture else ( + "the change touches HRR" if run_cpu else "the change does not touch HRR" + ) + + labelled = event == "pull_request" and CAPTURE_LABEL in labels + if labelled and not capture: + run_cpu, capture, why = True, True, f"the pull request is labelled {CAPTURE_LABEL}" + + if not capture: + return Decision(run_cpu, False, why) + if is_fork: + return Decision(run_cpu, False, why + "; GPU suite skipped for a fork pull request") + + families = select_families(trigger, requested, labels, matrix, pick_weighted) + if not families: + return Decision(run_cpu, False, why + "; no GPU family with a runner was selected") + return Decision(run_cpu, True, why, families) + + +def list_pr_files(repo: str, number: str) -> list[str] | None: + out = subprocess.run( + [ + "gh", "api", "--paginate", f"repos/{repo}/pulls/{number}/files", + "--jq", ".[] | .filename, (.previous_filename // empty)", + ], + check=True, capture_output=True, text=True, + ).stdout + files = [line for line in out.splitlines() if line] + return None if len(files) >= MAX_LISTED_FILES else files + + +def list_push_files(repo: str, before: str, after: str) -> list[str] | None: + if not before or set(before) == {"0"}: + return None + out = subprocess.run( + [ + "gh", "api", f"repos/{repo}/compare/{before}...{after}", + "--jq", ".files[] | .filename, (.previous_filename // empty)", + ], + check=True, capture_output=True, text=True, + ).stdout + files = [line for line in out.splitlines() if line] + # The compare API lists at most 300 files. + return None if len(files) >= 300 else files + + +def load_matrix(therock_dir: Path): + sys.path.insert(0, str(therock_dir / "build_tools" / "github_actions")) + import amdgpu_family_matrix # noqa: E402 + + families = amdgpu_family_matrix.get_all_families_for_trigger_types([]) + return families, amdgpu_family_matrix.select_weighted_label + + +def write_outputs(decision: Decision, output_path: str | None) -> None: + lines = [ + f"run_cpu={'true' if decision.run_cpu else 'false'}", + f"run_gpu={'true' if decision.run_gpu else 'false'}", + f"families={json.dumps(decision.families, separators=(',', ':'))}", + f"reason={decision.reason}", + ] + text = "\n".join(lines) + "\n" + if output_path: + with open(output_path, "a", encoding="utf-8") as out: + out.write(text) + sys.stdout.write(text) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--event", required=True) + parser.add_argument("--repo", required=True) + parser.add_argument("--pr-number", default="") + parser.add_argument("--before", default="") + parser.add_argument("--after", default="") + parser.add_argument("--labels", default="[]", help="JSON list of label names") + parser.add_argument("--is-fork", default="false") + parser.add_argument( + "--families", default="", help="Comma-separated family keys (workflow_dispatch)" + ) + parser.add_argument("--therock-dir", type=Path, required=True) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + labels = json.loads(args.labels or "[]") + requested = [f.strip().lower() for f in args.families.split(",") if f.strip()] + + paths: list[str] | None = [] + if args.event == "pull_request" and args.pr_number: + paths = list_pr_files(args.repo, args.pr_number) + elif args.event == "push": + paths = list_push_files(args.repo, args.before, args.after) + + matrix, pick_weighted = load_matrix(args.therock_dir) + decision = decide( + args.event, paths, labels, args.is_fork == "true", requested, matrix, pick_weighted + ) + write_outputs(decision, os.environ.get("GITHUB_OUTPUT")) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/hrr_ci_report.py b/.github/scripts/hrr_ci_report.py new file mode 100644 index 000000000000..2d45e29eca4a --- /dev/null +++ b/.github/scripts/hrr_ci_report.py @@ -0,0 +1,163 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Write the HRR CI summary for a run and decide its overall result. + +The summary has three parts: + + * why this run tested what it did, and which base build the GPU suite used; + * the conclusion of every unit, build and GPU job, so a job that never + produced results (cancelled, timed out, still queued) still has a row; + * the per-suite results table from projects/hrr's summarize_junit.py. + +Exits 1 when any HRR job failed or was cancelled, or a results file could not +be read. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +JOB_PATTERNS = ( + ("Generated sources", re.compile(r"^Generated sources$")), + ("Script tests", re.compile(r"^HRR script tests$")), + ("Unit", re.compile(r"(?:^|/ )Unit \((?P[^)]+)\)$")), + ("Build", re.compile(r"(?:^|/ )Build capture-enabled HIP and HRR$")), + ("Integration", re.compile(r"(?:^|/ )Integration \((?P[^)]+)\)$")), +) +BAD_CONCLUSIONS = {"failure", "cancelled", "timed_out", "startup_failure"} + + +def classify_jobs(jobs: list[dict]) -> list[dict]: + """HRR jobs from the run's job list, as {kind, name, conclusion, url}.""" + rows = [] + for job in jobs: + for kind, pattern in JOB_PATTERNS: + match = pattern.search(job.get("name", "")) + if match: + rows.append( + { + "kind": kind, + "name": match.groupdict().get("name") or "", + "conclusion": job.get("conclusion") or job.get("status") or "unknown", + "url": job.get("html_url", ""), + } + ) + break + return rows + + +def missing_families(families: list[dict], rows: list[dict]) -> list[str]: + seen = {r["name"] for r in rows if r["kind"] == "Integration"} + return [f["family"] for f in families if f["family"] not in seen] + + +def render_header(reason: str, run_gpu: bool, families: list[dict], base: dict | None) -> str: + lines = ["## HRR CI", "", f"- Scope: {reason}."] + if run_gpu: + names = ", ".join(f["family"] for f in families) or "none" + lines.append(f"- GPU families: {names}.") + if base: + lines.append( + f"- Base ROCm artifacts: [{base['repo']} run {base['run_id']}]({base['run_url']})" + f" at `{base['commit'][:12]}`; amdhip64 rebuilt from this commit with capture enabled." + ) + else: + lines.append("- GPU suite: not run.") + return "\n".join(lines) + "\n\n" + + +def render_jobs(rows: list[dict], missing: list[str]) -> str: + if not rows and not missing: + return "" + lines = ["### Jobs", "", "| Job | Target | Conclusion |", "| --- | --- | --- |"] + for row in rows: + conclusion = f"[{row['conclusion']}]({row['url']})" if row["url"] else row["conclusion"] + lines.append(f"| {row['kind']} | {row['name'] or '—'} | {conclusion} |") + for family in missing: + lines.append(f"| Integration | {family} | _did not run_ |") + return "\n".join(lines) + "\n\n" + + +def overall_ok(rows: list[dict], missing: list[str]) -> bool: + return not missing and not any(r["conclusion"] in BAD_CONCLUSIONS for r in rows) + + +def load_summarize_junit(path: Path): + spec = importlib.util.spec_from_file_location("summarize_junit", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def list_jobs(repo: str, run_id: str, attempt: str) -> list[dict]: + out = subprocess.run( + [ + "gh", "api", "--paginate", + f"repos/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", + "--jq", ".jobs[]", + ], + check=True, capture_output=True, text=True, + ).stdout + decoder = json.JSONDecoder() + jobs, pos = [], 0 + while pos < len(out): + while pos < len(out) and out[pos].isspace(): + pos += 1 + if pos >= len(out): + break + job, pos = decoder.raw_decode(out, pos) + jobs.append(job) + return jobs + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--reason", default="") + parser.add_argument("--run-gpu", default="false") + parser.add_argument("--families", default="[]") + parser.add_argument("--results-dir", type=Path, default=Path("results")) + parser.add_argument("--base-json", type=Path) + parser.add_argument( + "--summarize-junit", + type=Path, + default=Path("projects/hrr/tests/scripts/summarize_junit.py"), + ) + parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "")) + parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID", "")) + parser.add_argument("--run-attempt", default=os.environ.get("GITHUB_RUN_ATTEMPT", "1")) + args = parser.parse_args(argv) + + run_gpu = args.run_gpu == "true" + families = json.loads(args.families or "[]") if run_gpu else [] + base = None + if args.base_json and args.base_json.is_file(): + base = json.loads(args.base_json.read_text()) + + rows = classify_jobs(list_jobs(args.repo, args.run_id, args.run_attempt)) + missing = missing_families(families, rows) + + markdown = render_header(args.reason, run_gpu, families, base) + render_jobs(rows, missing) + errors: list[str] = [] + args.results_dir.mkdir(parents=True, exist_ok=True) + markdown += load_summarize_junit(args.summarize_junit).render(args.results_dir, errors) + + sys.stdout.write(markdown) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as out: + out.write(markdown) + for error in errors: + print(f"::error::unreadable JUnit XML {error}") + return 0 if overall_ok(rows, missing) and not errors else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/tests/hrr_capture_build_test.py b/.github/scripts/tests/hrr_capture_build_test.py new file mode 100644 index 000000000000..da144073ce82 --- /dev/null +++ b/.github/scripts/tests/hrr_capture_build_test.py @@ -0,0 +1,153 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for hrr_capture_build.py.""" + +import argparse +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) +import hrr_capture_build as cb + +FINDER_FOUND = { + "status": "found", + "results": [ + { + "github_repository_name": "ROCm/rocm-systems", + "workflow_run_id": "123", + "git_commit_sha": "abc", + "workflow_run_html_url": "https://example/runs/123", + "artifact_group": "gfx94X-dcgpu", + } + ], +} + + +class PickBaseRunTest(unittest.TestCase): + def test_found(self): + base = cb.pick_base_run(FINDER_FOUND) + self.assertEqual(base["run_id"], "123") + self.assertEqual(base["commit"], "abc") + self.assertEqual(base["repo"], "ROCm/rocm-systems") + + def test_not_found(self): + with self.assertRaises(SystemExit): + cb.pick_base_run({"status": "not_found", "results": []}) + + +class CommandTest(unittest.TestCase): + def test_fetch_command_uses_recorded_run(self): + base = cb.pick_base_run(FINDER_FOUND) + cmd = [str(c) for c in cb.fetch_command(Path("/t"), base, Path("/d"), ("core-hip_lib",))] + self.assertIn("--flatten", cmd) + self.assertEqual(cmd[cmd.index("--run-id") + 1], "123") + self.assertEqual(cmd[cmd.index("--run-github-repo") + 1], "ROCm/rocm-systems") + self.assertEqual(cmd[-1], "core-hip_lib") + + def test_clr_compiles_capture_in(self): + cmd = [str(c) for c in cb.clr_configure_command(Path("/s"), Path("/b"), Path("/d"))] + self.assertIn("-DCMAKE_CXX_FLAGS=-DHIP_HRR_CAPTURE_ENABLED", cmd) + self.assertIn("-DCMAKE_INSTALL_PREFIX=/d", cmd) + self.assertIn("-DHIP_COMMON_DIR=/s/projects/hip", cmd) + + def test_hrr_builds_integration_suite_for_archs(self): + cmd = [str(c) for c in cb.hrr_configure_command(Path("/s"), Path("/b"), Path("/d"), "gfx942;gfx950")] + self.assertIn("-DGPU_TARGETS=gfx942;gfx950", cmd) + self.assertIn("-DHRR_BUILD_INTEGRATION_TESTS=ON", cmd) + self.assertIn("-DCMAKE_HIP_COMPILER=/d/lib/llvm/bin/clang++", cmd) + + def test_build_targets_cover_uploaded_executables(self): + names = {Path(rel).name for rel in cb.EXECUTABLES} + self.assertTrue(names <= set(cb.HRR_TARGETS)) + + def test_build_and_runtime_artifacts_are_generic_names(self): + self.assertTrue(set(cb.RUNTIME_ARTIFACTS) <= set(cb.BUILD_ARTIFACTS)) + + +class EmbeddedTargetsTest(unittest.TestCase): + OUTPUT = ( + "OFFLOADING IMAGE [0]:\nkind elf\narch gfx942\ntriple amdgcn-amd-amdhsa\n" + "OFFLOADING IMAGE [1]:\nkind elf\narch gfx1100\n" + ) + + def test_parses_arches(self): + self.assertEqual(cb.embedded_targets(self.OUTPUT), {"gfx942", "gfx1100"}) + + def test_missing(self): + self.assertEqual(cb.missing_targets("gfx942;gfx950;gfx1100", self.OUTPUT), ["gfx950"]) + + def test_none_missing(self): + self.assertEqual(cb.missing_targets("gfx942;gfx1100", self.OUTPUT), []) + + +class LibraryOverlayTest(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + def make_lib(self, directory: Path, real: str, content: str) -> None: + directory.mkdir(parents=True, exist_ok=True) + (directory / real).write_text(content) + os.symlink(real, directory / "libamdhip64.so.7") + os.symlink("libamdhip64.so.7", directory / "libamdhip64.so") + + def test_overlay_replaces_release_library(self): + capture = self.tmp / "capture" + dist = self.tmp / "dist" / "lib" + self.make_lib(capture, "libamdhip64.so.7.2.1", "capture") + self.make_lib(dist, "libamdhip64.so.7.2.0", "release") + (dist / "libhiprtc.so.7").write_text("keep") + + cb.remove_amdhip64(dist) + copied = cb.copy_amdhip64(capture, dist) + + self.assertEqual(sorted(copied), ["libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.7.2.1"]) + self.assertFalse((dist / "libamdhip64.so.7.2.0").exists()) + self.assertEqual((dist / "libamdhip64.so").read_text(), "capture") + self.assertTrue((dist / "libamdhip64.so.7").is_symlink()) + self.assertEqual((dist / "libhiprtc.so.7").read_text(), "keep") + + def test_copy_without_library_fails(self): + (self.tmp / "empty").mkdir() + with self.assertRaises(SystemExit): + cb.copy_amdhip64(self.tmp / "empty", self.tmp / "out") + + +class ExecutableBitsTest(unittest.TestCase): + def test_restores_bits(self): + with tempfile.TemporaryDirectory() as tmp: + build = Path(tmp) + for rel in cb.EXECUTABLES: + (build / rel).parent.mkdir(parents=True, exist_ok=True) + (build / rel).write_text("") + (build / rel).chmod(0o644) + cb.restore_executable_bits(build) + for rel in cb.EXECUTABLES: + self.assertEqual((build / rel).stat().st_mode & 0o777, 0o755) + + +class FetchBaseTest(unittest.TestCase): + def test_reuses_recorded_base(self): + with tempfile.TemporaryDirectory() as tmp: + base_json = Path(tmp) / "base.json" + base_json.write_text(json.dumps(cb.pick_base_run(FINDER_FOUND))) + calls = [] + args = argparse.Namespace( + base_json=base_json, therock=Path("/t"), dist=Path("/d") + ) + cb.cmd_fetch_base(args, runner=calls.append) + cmd = [str(c) for c in calls[0]] + self.assertEqual(cmd[cmd.index("--run-id") + 1], "123") + self.assertIn("core-runtime_dev", cmd) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/hrr_check_sources_test.py b/.github/scripts/tests/hrr_check_sources_test.py new file mode 100644 index 000000000000..3f8a185479e2 --- /dev/null +++ b/.github/scripts/tests/hrr_check_sources_test.py @@ -0,0 +1,160 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for hrr_check_sources.py.""" + +import contextlib +import io +import os +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) +import hrr_check_sources as chk + + +def write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +class Tree(unittest.TestCase): + """A fake checkout whose generator writes whatever self.generated holds.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + base = Path(self._tmp.name) + self.root = base / "src" + self.out = base / "out" + self.out.mkdir() + for tracked in chk.GENERATED_FILES: + write(self.root / tracked, f"// {tracked}\n") + write(self.root / chk.MATRIX_EXPECTATIONS, "expect\n") + self.generated = {t: f"// {t}\n" for t in chk.GENERATED_FILES} + self.emitted = "expect\n" + self.calls = [] + self.rc = {} + + def tearDown(self): + self._tmp.cleanup() + + def runner(self, cmd): + self.calls.append(list(cmd)) + script = Path(cmd[1]).name + if script == "gen_hrr_api_args.py": + for tracked, name in chk.GENERATED_FILES.items(): + write(self.out / name, self.generated[tracked]) + if "--emit-cxx" in cmd: + write(Path(cmd[cmd.index("--emit-cxx") + 1]), self.emitted) + return self.rc.get(script, 0) + + def quiet(self, fn, *args, **kwargs): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + result = fn(*args, **kwargs) + return result, buf.getvalue() + + +class GeneratedTest(Tree): + def test_up_to_date(self): + ok, _ = self.quiet(chk.check_generated, self.root, self.out, self.runner) + self.assertTrue(ok) + self.assertIn("--check-hrr-coverage", self.calls[0]) + + def test_stale_file_is_reported(self): + stale = "projects/hrr/playback/hip_playback_generated.cpp" + self.generated[stale] = "// changed\n" + ok, output = self.quiet(chk.check_generated, self.root, self.out, self.runner) + self.assertFalse(ok) + self.assertIn("::error::checked-in HRR generated sources are stale", output) + self.assertIn(stale, output) + self.assertIn("+// changed", output) + + def test_line_endings_are_ignored(self): + tracked = chk.API_ARGS_HEADER + self.generated[tracked] = f"// {tracked}\r\n" + ok, _ = self.quiet(chk.check_generated, self.root, self.out, self.runner) + self.assertTrue(ok) + + def test_generator_failure(self): + self.rc["gen_hrr_api_args.py"] = 2 + ok, output = self.quiet(chk.check_generated, self.root, self.out, self.runner) + self.assertFalse(ok) + self.assertIn("failed (exit 2)", output) + + +class ApiCompatTest(Tree): + def regenerate(self): + self.quiet(chk.check_generated, self.root, self.out, self.runner) + self.calls.clear() + + def test_compares_against_base(self): + self.regenerate() + ok, _ = self.quiet( + chk.check_api_compat, self.root, self.out, "abc", + self.runner, lambda root, sha: "base header\n", + ) + self.assertTrue(ok) + cmd = self.calls[0] + self.assertEqual(Path(cmd[1]).name, "check_hrr_api_compat.py") + base = Path(cmd[cmd.index("--base-header") + 1]) + self.assertEqual(base.read_text(), "base header\n") + + def test_incompatible_ids_fail(self): + self.regenerate() + self.rc["check_hrr_api_compat.py"] = 1 + ok, output = self.quiet( + chk.check_api_compat, self.root, self.out, "abc", + self.runner, lambda root, sha: "base\n", + ) + self.assertFalse(ok) + self.assertIn("not compatible with abc", output) + + def test_missing_base_header_passes(self): + self.regenerate() + ok, output = self.quiet( + chk.check_api_compat, self.root, self.out, "abc", + self.runner, lambda root, sha: None, + ) + self.assertTrue(ok) + self.assertEqual(self.calls, []) + self.assertIn("does not exist at abc", output) + + def test_needs_regenerated_header(self): + ok, output = self.quiet( + chk.check_api_compat, self.root, self.out, "abc", + self.runner, lambda root, sha: "x", + ) + self.assertFalse(ok) + self.assertIn("run the generated check first", output) + + +class ApiMatrixTest(Tree): + def test_up_to_date(self): + ok, _ = self.quiet(chk.check_api_matrix, self.root, self.out, self.runner) + self.assertTrue(ok) + self.assertEqual(len(self.calls), 3) + + def test_stale_expectations(self): + self.emitted = "different\n" + ok, output = self.quiet(chk.check_api_matrix, self.root, self.out, self.runner) + self.assertFalse(ok) + self.assertIn("hrr_api_matrix_expectations.h is stale", output) + + def test_step_failure_stops(self): + self.rc["derive_manifest.py"] = 3 + ok, output = self.quiet(chk.check_api_matrix, self.root, self.out, self.runner) + self.assertFalse(ok) + self.assertEqual(len(self.calls), 1) + self.assertIn("derive_manifest.py failed (exit 3)", output) + + +class ResolveBaseTest(unittest.TestCase): + def test_explicit_sha(self): + self.assertEqual(chk.resolve_base(Path("."), "1234abcd"), "1234abcd") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/hrr_ci_configure_test.py b/.github/scripts/tests/hrr_ci_configure_test.py new file mode 100644 index 000000000000..40348f53b41f --- /dev/null +++ b/.github/scripts/tests/hrr_ci_configure_test.py @@ -0,0 +1,204 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for hrr_ci_configure.py.""" + +import os +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) +import hrr_ci_configure as cfg + +MATRIX = { + "gfx94x": { + "linux": { + "family": "gfx94X-dcgpu", + "test-runs-on": "gpu-94x", + "test-runs-on-labels": [{"label": "gpu-94x-a", "count": 1}], + "fetch-gfx-targets": ["gfx942"], + "tests_on_trigger": ["presubmit", "postsubmit", "nightly"], + } + }, + "gfx90a": { + "linux": { + "family": "gfx90a", + "test-runs-on": "gpu-90a", + "fetch-gfx-targets": ["gfx90a"], + "tests_on_trigger": ["postsubmit", "nightly"], + } + }, + "gfx110x": { + "linux": { + "family": "gfx110X-all", + "test-runs-on": "gpu-110x", + "fetch-gfx-targets": ["gfx1100", "gfx1101"], + "tests_on_trigger": ["nightly"], + } + }, + "gfx950": { + "linux": { + "family": "gfx950-dcgpu", + "test-runs-on": "gpu-950-1", + "test-runs-on-multi-gpu": "gpu-950-8", + "fetch-gfx-targets": ["gfx950"], + "tests_on_trigger": ["submodule_bump"], + } + }, + "gfx1150": { + "linux": { + "family": "gfx1150", + "test-runs-on": "", + "tests_on_trigger": ["nightly"], + } + }, + "gfx125x": { + "linux": { + "family": "gfx125X-dcgpu", + "test-runs-on": "gpu-125x", + "fetch-gfx-targets": ["gfx1250"], + "tests_on_trigger": ["nightly"], + } + }, +} + + +def pick_first(labels, _context): + return labels[0]["label"] + + +def decide(event, paths, labels=(), is_fork=False, requested=()): + return cfg.decide( + event, paths, list(labels), is_fork, list(requested), MATRIX, pick_first + ) + + +def family_keys(decision): + return [f["family"] for f in decision.families] + + +class PathClassificationTest(unittest.TestCase): + def test_playback_change_is_capture(self): + self.assertTrue(cfg.touches_capture(["projects/hrr/playback/hip_playback.cpp"])) + + def test_capture_source_is_capture(self): + self.assertTrue(cfg.touches_capture(["projects/clr/hipamd/src/hrr/hip_capture.cpp"])) + + def test_public_header_is_cpu_only(self): + paths = ["projects/hip/include/hip/hip_runtime_api.h"] + self.assertTrue(cfg.touches_cpu(paths)) + self.assertFalse(cfg.touches_capture(paths)) + + def test_unit_test_is_cpu_only(self): + paths = ["projects/hrr/tests/unit/hrr_format_test.cc"] + self.assertTrue(cfg.touches_cpu(paths)) + self.assertFalse(cfg.touches_capture(paths)) + + def test_skills_are_excluded(self): + paths = ["projects/hrr/skills/hrr/SKILL.md", "projects/hrr/README.md"] + self.assertFalse(cfg.touches_cpu(paths)) + + def test_unrelated_change(self): + self.assertFalse(cfg.touches_cpu(["projects/rocminfo/rocminfo.cc"])) + + def test_prefix_must_match_whole_component(self): + self.assertFalse(cfg.touches_cpu(["projects/hrrx/foo.cc"])) + + +class DecideTest(unittest.TestCase): + def test_unrelated_pull_request_runs_nothing(self): + d = decide("pull_request", ["projects/rocminfo/rocminfo.cc"]) + self.assertFalse(d.run_cpu) + self.assertFalse(d.run_gpu) + + def test_cpu_only_pull_request(self): + d = decide("pull_request", ["projects/hrr/tests/unit/hrr_format_test.cc"]) + self.assertTrue(d.run_cpu) + self.assertFalse(d.run_gpu) + + def test_capture_pull_request_uses_presubmit_families(self): + d = decide("pull_request", ["projects/clr/hipamd/src/hrr/hip_capture.cpp"]) + self.assertTrue(d.run_gpu) + self.assertEqual(family_keys(d), ["gfx94x"]) + self.assertEqual(d.families[0]["runner"], "gpu-94x-a") + self.assertEqual(d.families[0]["gfx_targets"], "gfx942") + + def test_rename_out_of_capture_counts(self): + # The files API lists previous_filename alongside filename. + d = decide("pull_request", ["projects/other/x.cpp", "projects/hrr/playback/x.cpp"]) + self.assertTrue(d.run_gpu) + + def test_label_requests_gpu_on_unrelated_pull_request(self): + d = decide("pull_request", ["projects/rocminfo/x.cc"], labels=[cfg.CAPTURE_LABEL]) + self.assertTrue(d.run_cpu) + self.assertTrue(d.run_gpu) + + def test_family_label_adds_family(self): + d = decide( + "pull_request", + ["projects/hrr/playback/x.cpp"], + labels=["ci:gfx110X-all", "ci:gfx90a"], + ) + self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx110x"]) + + def test_fork_skips_gpu(self): + d = decide("pull_request", ["projects/hrr/playback/x.cpp"], is_fork=True) + self.assertTrue(d.run_cpu) + self.assertFalse(d.run_gpu) + self.assertIn("fork", d.reason) + + def test_push_uses_postsubmit_families(self): + d = decide("push", ["projects/hrr/playback/x.cpp"]) + self.assertEqual(family_keys(d), ["gfx90a", "gfx94x"]) + + def test_push_without_capture_change_has_no_gpu(self): + d = decide("push", ["projects/hrr/tests/unit/x.cc"]) + self.assertTrue(d.run_cpu) + self.assertFalse(d.run_gpu) + + def test_unlisted_change_set_runs_everything(self): + d = decide("push", None) + self.assertTrue(d.run_cpu) + self.assertTrue(d.run_gpu) + + def test_nightly_skips_on_request_and_runnerless_families(self): + d = decide("schedule", []) + self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950", "gfx110x"]) + + def test_gfx950_uses_multi_gpu_runner(self): + d = decide("schedule", []) + runners = {f["family"]: f["runner"] for f in d.families} + self.assertEqual(runners["gfx950"], "gpu-950-8") + + def test_gfx950_not_on_pull_requests_by_default(self): + d = decide("pull_request", ["projects/hrr/playback/x.cpp"]) + self.assertNotIn("gfx950", family_keys(d)) + + def test_dispatch_with_families(self): + d = decide("workflow_dispatch", [], requested=["gfx125x", "gfx1150"]) + # gfx1150 has no runner in this matrix. + self.assertEqual(family_keys(d), ["gfx125x"]) + + def test_unknown_event(self): + d = decide("merge_group", []) + self.assertFalse(d.run_cpu) + self.assertFalse(d.run_gpu) + + +class OutputTest(unittest.TestCase): + def test_write_outputs(self): + import tempfile + + d = cfg.Decision(True, True, "why", [{"family": "gfx94x", "runner": "r", "gfx_targets": "gfx942"}]) + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "out" + cfg.write_outputs(d, str(out)) + text = out.read_text() + self.assertIn("run_cpu=true\n", text) + self.assertIn('families=[{"family":"gfx94x","runner":"r","gfx_targets":"gfx942"}]\n', text) + self.assertIn("reason=why\n", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/tests/hrr_ci_report_test.py b/.github/scripts/tests/hrr_ci_report_test.py new file mode 100644 index 000000000000..ef67b10d9e41 --- /dev/null +++ b/.github/scripts/tests/hrr_ci_report_test.py @@ -0,0 +1,90 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for hrr_ci_report.py.""" + +import os +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) +import hrr_ci_report as rpt + +REPO_ROOT = Path(__file__).resolve().parents[3] + +JOBS = [ + {"name": "Configure", "conclusion": "success", "html_url": "u0"}, + {"name": "Generated sources", "conclusion": "success", "html_url": "c0"}, + {"name": "Unit (ubuntu-24.04)", "conclusion": "success", "html_url": "u1"}, + {"name": "Unit (windows-2022)", "conclusion": "failure", "html_url": "u2"}, + {"name": "gpu / Build capture-enabled HIP and HRR", "conclusion": "success", "html_url": "u3"}, + {"name": "gpu / Integration (gfx94x)", "conclusion": "success", "html_url": "u4"}, + {"name": "gpu / Integration (gfx950)", "conclusion": "cancelled", "html_url": "u5"}, +] + +FAMILIES = [{"family": "gfx94x"}, {"family": "gfx950"}, {"family": "gfx110x"}] + + +class ClassifyTest(unittest.TestCase): + def test_picks_hrr_jobs(self): + rows = rpt.classify_jobs(JOBS) + self.assertEqual( + [(r["kind"], r["name"]) for r in rows], + [ + ("Generated sources", ""), + ("Unit", "ubuntu-24.04"), + ("Unit", "windows-2022"), + ("Build", ""), + ("Integration", "gfx94x"), + ("Integration", "gfx950"), + ], + ) + + def test_missing_family(self): + rows = rpt.classify_jobs(JOBS) + self.assertEqual(rpt.missing_families(FAMILIES, rows), ["gfx110x"]) + + def test_queued_job_reports_status(self): + rows = rpt.classify_jobs([{"name": "gpu / Integration (gfx90a)", "status": "queued"}]) + self.assertEqual(rows[0]["conclusion"], "queued") + + +class OverallTest(unittest.TestCase): + def test_failure_or_cancel_fails(self): + self.assertFalse(rpt.overall_ok(rpt.classify_jobs(JOBS), [])) + + def test_all_green(self): + rows = rpt.classify_jobs([JOBS[1], JOBS[2], JOBS[5]]) + self.assertTrue(rpt.overall_ok(rows, [])) + + def test_missing_family_fails(self): + rows = rpt.classify_jobs([JOBS[5]]) + self.assertFalse(rpt.overall_ok(rows, ["gfx950"])) + + +class RenderTest(unittest.TestCase): + def test_header_names_base(self): + base = {"repo": "ROCm/rocm-systems", "run_id": "7", "run_url": "https://r/7", "commit": "0123456789abcdef"} + text = rpt.render_header("the change touches capture or replay", True, FAMILIES, base) + self.assertIn("GPU families: gfx94x, gfx950, gfx110x.", text) + self.assertIn("[ROCm/rocm-systems run 7](https://r/7) at `0123456789ab`", text) + + def test_header_without_gpu(self): + self.assertIn("GPU suite: not run.", rpt.render_header("why", False, [], None)) + + def test_jobs_table_lists_missing(self): + rows = rpt.classify_jobs(JOBS) + text = rpt.render_jobs(rows, ["gfx110x"]) + self.assertIn("| Integration | gfx950 | [cancelled](u5) |", text) + self.assertIn("| Integration | gfx110x | _did not run_ |", text) + + def test_summarize_junit_loads(self): + module = rpt.load_summarize_junit( + REPO_ROOT / "projects/hrr/tests/scripts/summarize_junit.py" + ) + self.assertTrue(callable(module.render)) + + +if __name__ == "__main__": + unittest.main() From 1491254661a614e37c7f89af2c1af829dc66fe3e Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 13 Oct 2026 12:34:11 +0200 Subject: [PATCH 05/89] ci(hrr): split HRR CI into an entry workflow and a GPU workflow hrr-ci.yml now runs a configure job (hrr_ci_configure.py), the generated-source, script and host unit jobs, calls hrr-gpu.yml with workflow_call when the GPU suite should run, and ends with one HRR CI Summary table across platforms and GPU families. It also runs nightly. hrr-gpu.yml builds the capture-enabled amdhip64 and the HRR suite in TheRock's build image from TheRock CI artifacts, then runs the suite per GPU family in TheRock's test image with TheRock's container options. That removes the self-dispatch and polling, the nightly-wheel and apt -dev build dependencies, the ccache in the Actions cache, and the path symlinks and LD_LIBRARY_PATH overrides on the GPU machines. The unit job installs one pinned ROCm SDK version, and every checkout disables persist-credentials. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 1256 +++++---------------------------- .github/workflows/hrr-gpu.yml | 188 +++++ 2 files changed, 384 insertions(+), 1060 deletions(-) create mode 100644 .github/workflows/hrr-gpu.yml diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 077b6f2d6f1e..6ab4ec0fba09 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -1,1174 +1,310 @@ -name: HRR CI +# HRR CI: generated-source checks, the HRR helper-script tests and the host +# unit suite, plus the GPU capture/replay suite (hrr-gpu.yml) for changes that +# touch capture or replay. .github/scripts/hrr_ci_configure.py decides what +# runs; "HRR CI Summary" reports every platform and GPU family in one table. -# A GPU-only dispatch names the run after its id so the PR job can find it. -# Every other event keeps a stable title. -run-name: ${{ github.event.inputs.gpu_only == 'true' && format('hrr-gpu-{0}', github.event.inputs.dispatch_id) || format('HRR CI ({0})', github.event_name) }} +name: HRR CI -# Pull requests always start the workflow. A workflow skipped by a top-level -# "paths" filter creates no check at all, so a required "HRR CI Summary" would -# wait forever on every PR that does not touch HRR. The gate job below does the -# path filtering instead, and the summary job always reports. -# -# Pushes to develop run the same suites for one reason beyond post-merge signal: -# a pull request can only restore caches from its own ref and from the base -# branch, so without a develop run every PR would start with a cold ccache. on: pull_request: + types: [opened, synchronize, reopened, labeled] push: - branches: - - develop - paths: - - "projects/hrr/**" - - "!projects/hrr/skills/**" - - "projects/clr/hipamd/src/hrr/**" - - "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp" - - "projects/hip/include/hip/hip_runtime_api.h" - - ".github/workflows/hrr-ci.yml" - - ".github/actions/rocprofiler-sdk-util/resolve-ci-config/**" + branches: [develop] + schedule: + # Nightly: capture-enabled build and the GPU suite on every HRR family. + - cron: "0 3 * * *" workflow_dispatch: inputs: - ignore_test_config: - description: "Run cases the YAML config would skip" - type: boolean - default: false - # The fields below are set by the integration dispatcher. A new workflow - # file cannot be dispatched until it is on the default branch, so the - # GPU job lives in this file and the PR run starts it with gpu_only. - gpu_only: - description: "Run one integration family against an existing build" - type: boolean - default: false - family: - description: "Integration family (gpu_only)" + families: + description: "Comma-separated GPU families (TheRock keys, e.g. gfx94x,gfx110x). Empty: the nightly set." type: string default: "" - runner: - description: "Runner label (gpu_only)" - type: string - default: "" - use_container: - description: "Run inside the GPU container (gpu_only)" + ignore_test_config: + description: "Run the cases the suite config skips" type: boolean default: false - sdk_version: - description: "ROCm SDK version the build used (gpu_only)" - type: string - default: "" - build_workspace: - description: "Absolute workspace path baked into the binaries (gpu_only)" - type: string - default: "" - build_run_id: - description: "Run id whose hrr-build artifact to download (gpu_only)" - type: string - default: "" - # workflow_dispatch allows at most ten inputs; this list is full. - build_sha: - description: "Commit the build compiled; the GPU job checks this out (gpu_only)" - type: string - default: "" - dispatch_id: - description: "Unique id the dispatcher uses to find this run (gpu_only)" - type: string - default: "" - -concurrency: - # Each GPU dispatch has its own group. Sharing the PR group would queue the - # families behind each other, and the schedule budget would skip them. - group: ${{ github.event.inputs.gpu_only == 'true' && format('hrr-ci-gpu-{0}', github.event.inputs.dispatch_id) || format('hrr-ci-{0}', github.event.pull_request.number || github.ref) }} - # Never cancel a develop run: it is what saves the ccache for later PRs. - cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + env: - # One fat binary covers every family that has a test runner, so the GPU jobs - # download a single artifact instead of rebuilding per architecture. - # - # Every arch a pool can hand out has to be listed, not one representative per - # family: linux-gfx110X-gpu-rocm and linux-gfx120X-gpu-rocm are wildcard - # pools, and a machine whose arch is missing from the binary fails every - # workload with hipErrorInvalidImage (200) rather than skipping. gfx1250 is - # carried so the binary is ready once its runner is enabled below. - HRR_ARCHS: "gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1153;gfx1200;gfx1201;gfx1250" + # The unit suite only needs HIP headers and the hip CMake package. Pinned so + # a run does not depend on the day's nightly. ROCM_INDEX_URL: "https://rocm.nightlies.amd.com/whl-multi-arch/" + ROCM_SDK_VERSION: "10.1.0a20260822" jobs: - # --------------------------------------------------------------------------- - # Path gate. The "on:" block cannot filter pull requests by path (see above), - # so this job lists the files the pull request changes and sets run=false when - # none is an HRR path. Every job below is then skipped, not failed, and - # "HRR CI Summary" still reports. Pushes and manual runs always run. - # --------------------------------------------------------------------------- - gate: - name: "HRR paths changed?" + configure: + name: Configure runs-on: ubuntu-24.04 permissions: contents: read pull-requests: read outputs: - run: ${{ steps.decide.outputs.run }} + run_cpu: ${{ steps.decide.outputs.run_cpu }} + run_gpu: ${{ steps.decide.outputs.run_gpu }} + families: ${{ steps.decide.outputs.families }} + reason: ${{ steps.decide.outputs.reason }} steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: .github/scripts + + - name: Checkout TheRock family matrix + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ROCm/TheRock + # Keep in sync with the TheRock ref in therock-multi-arch-ci.yml. + ref: 05add1921fd03b0649f054ec2a7c54d162c0cd99 # 2026-10-08 + path: therock + sparse-checkout: build_tools/github_actions + + # Current runner labels, as TheRock's setup_multi_arch.yml loads them. + - name: Checkout TheRock CI runner config + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + continue-on-error: true + with: + persist-credentials: false + repository: ROCm/therock-ci-config + ref: main + path: ci-config + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + - name: Decide id: decide env: GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} + CI_CONFIG_PATH: ci-config + EVENT: ${{ github.event_name }} REPO: ${{ github.repository }} - run: | - run=true - reason="not a pull request" - if [ -n "${PR_NUMBER}" ]; then - reason="the pull request changes an HRR path" - # The files API lists at most 3000 files; a PR that large is treated - # as relevant rather than guessed at. A rename lists only the new - # path in filename, so previous_filename is listed as well: moving a - # file out of an HRR path is an HRR change. - gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files" \ - --jq '.[] | .filename, (.previous_filename // empty)' > changed-files.txt - if ! python3 - changed-files.txt <<'PY' - import sys - - INCLUDE = ( - "projects/hrr/", - "projects/clr/hipamd/src/hrr/", - "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp", - "projects/hip/include/hip/hip_runtime_api.h", - ".github/workflows/hrr-ci.yml", - ".github/actions/rocprofiler-sdk-util/resolve-ci-config/", - ) - EXCLUDE = ("projects/hrr/skills/",) - - files = [line.strip() for line in open(sys.argv[1]) if line.strip()] - relevant = len(files) >= 3000 or any( - f.startswith(INCLUDE) and not f.startswith(EXCLUDE) for f in files - ) - sys.exit(0 if relevant else 1) - PY - then - run=false - reason="the pull request changes no HRR path" - fi - fi - echo "run=${run}" >> "$GITHUB_OUTPUT" - echo "HRR CI: run=${run} (${reason})" | tee -a "$GITHUB_STEP_SUMMARY" + PR_NUMBER: ${{ github.event.pull_request.number }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.sha }} + LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }} + IS_FORK: ${{ github.event.pull_request.head.repo.fork || false }} + FAMILIES: ${{ inputs.families }} + run: >- + python .github/scripts/hrr_ci_configure.py + --event "${EVENT}" --repo "${REPO}" --pr-number "${PR_NUMBER}" + --before "${BEFORE}" --after "${AFTER}" --labels "${LABELS}" + --is-fork "${IS_FORK}" --families "${FAMILIES}" + --therock-dir therock - # --------------------------------------------------------------------------- - # Generated-source and wire-compatibility gate. This is intentionally - # host-only and separate from the CLR build: an API-header PR must get a fast - # result even when no GPU job runs. - # --------------------------------------------------------------------------- codegen: - name: "Generated sources" - needs: [gate] - if: ${{ github.event.inputs.gpu_only != 'true' && needs.gate.outputs.run == 'true' }} + name: Generated sources + needs: configure + if: ${{ needs.configure.outputs.run_cpu == 'true' }} runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + persist-credentials: false + # The API-ID check reads hrr_api_args.h at the base commit. fetch-depth: 0 + filter: blob:none sparse-checkout: | + .github/scripts projects/hrr projects/hip/include projects/clr/hipamd/include projects/clr/hipamd/src/hrr - set-safe-directory: true - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" - # api_matrix.yaml is read with PyYAML by the matrix scripts below. - # Install only when yaml does not import. A version pin makes pip - # uninstall a newer copy, which fails when site-packages is read-only. - name: Install PyYAML - run: python3 -c "import yaml" || python3 -m pip install --no-cache-dir PyYAML==6.0.1 - - - name: Generate and compare - run: | - mkdir -p build/hrr-codegen/include/hrr - python3 projects/hrr/tools/gen_hrr_api_args.py \ - --input projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp \ - --public-header projects/hip/include/hip/hip_runtime_api.h \ - --output-header build/hrr-codegen/include/hrr/hrr_api_args.h \ - --output-capture build/hrr-codegen/hip_capture_generated.cpp \ - --output-playback build/hrr-codegen/hip_playback_generated.cpp \ - --check-hrr-coverage \ - --manual-capture-source projects/clr/hipamd/src/hrr/hip_capture.cpp \ - --silent + run: python -m pip install --no-cache-dir PyYAML==6.0.2 - failed=0 - diff -u projects/hrr/include/hrr/hrr_api_args.h \ - build/hrr-codegen/include/hrr/hrr_api_args.h || failed=1 - diff -u projects/clr/hipamd/src/hrr/hip_capture_generated.cpp \ - build/hrr-codegen/hip_capture_generated.cpp || failed=1 - diff -u projects/hrr/playback/hip_playback_generated.cpp \ - build/hrr-codegen/hip_playback_generated.cpp || failed=1 - if [ "${failed}" -ne 0 ]; then - echo "::error::checked-in HRR generated sources are stale; run projects/hrr/tools/gen_hrr_api_args.py --check-hrr-coverage" - exit 1 - fi - - - name: Check API ID compatibility + - name: Check generated sources, API IDs and the API matrix env: BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} - run: | - if [ -z "${BASE_SHA}" ] || [[ "${BASE_SHA}" =~ ^0+$ ]]; then - BASE_SHA="$(git rev-parse HEAD^)" - fi - git show "${BASE_SHA}:projects/hrr/include/hrr/hrr_api_args.h" \ - > build/hrr-codegen/base_hrr_api_args.h - python3 projects/hrr/tools/check_hrr_api_compat.py \ - --base-header build/hrr-codegen/base_hrr_api_args.h \ - --current-header build/hrr-codegen/include/hrr/hrr_api_args.h - - # Everything is written under build/hrr-codegen: the derived manifest and - # the regenerated header are compared with the checked-in header, never - # written over it. - - name: Check API matrix - run: | - tools=projects/hrr/tools/api-matrix - out=build/hrr-codegen/api-matrix - tracked=projects/hrr/tests/integration/hrr_api_matrix_expectations.h - mkdir -p "${out}" - - python3 "${tools}/derive_manifest.py" --quiet \ - --output "${out}/api_classes.json" - python3 "${tools}/check_matrix.py" --check-workloads - python3 "${tools}/check_matrix.py" \ - --manifest "${out}/api_classes.json" \ - --emit-cxx "${out}/hrr_api_matrix_expectations.h" - - if ! diff -u "${tracked}" "${out}/hrr_api_matrix_expectations.h"; then - echo "::error::${tracked} is stale; run derive_manifest.py, then check_matrix.py --emit-cxx, in projects/hrr/tools/api-matrix" - exit 1 - fi - - # --------------------------------------------------------------------------- - # Host-only unit suite. These 23 cases link nothing but libstdc++/libc: they - # include for types but issue no HIP calls, so they need - # neither a GPU nor the capture library and can run on any platform. - # --------------------------------------------------------------------------- - unit: - name: "Unit (${{ matrix.os }})" - needs: [gate] - if: ${{ github.event.inputs.gpu_only != 'true' && needs.gate.outputs.run == 'true' }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04, windows-2022] - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - sparse-checkout: | - projects/hrr - projects/hip/include - projects/clr/hipamd/include - projects/clr/hipamd/src - .github/workflows - set-safe-directory: true - - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: "3.12" - - - name: Install ROCm SDK (headers and toolchain only) - shell: bash - run: | - pip install --no-cache-dir --index-url "${ROCM_INDEX_URL}" "rocm[devel]" - rocm-sdk init - echo "ROCM_SDK=$(python -c 'import _rocm_sdk_devel,os;print(os.path.dirname(_rocm_sdk_devel.__file__))')" >> "$GITHUB_ENV" - - - name: Configure - shell: bash - run: | - extra=() - if [ "${RUNNER_OS}" = "Windows" ]; then - # The runner builds with MinGW, whose libstdc++/libgcc/winpthread - # DLLs are not on PATH for later steps, so a dynamically linked - # binary fails to start at all. Link them in instead. - extra+=(-DCMAKE_EXE_LINKER_FLAGS=-static) - fi - # Declarations of the HIP API come from this tree, ahead of the SDK's: - # the nightly SDK lags the tree, so an API added here is not in its - # headers yet. Only headers: the SDK still supplies the runtime library. - # cygpath -m gives the drive-letter form a native cmake accepts. - tree="$(cygpath -m "${PWD}" 2>/dev/null || echo "${PWD}")" - cmake -S projects/hrr -B build -GNinja -Wno-dev \ - "${extra[@]}" \ - "-DHRR_HIP_INCLUDE_DIRS=${tree}/projects/hip/include;${tree}/projects/clr/hipamd/include" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="${ROCM_SDK}" \ - -DROCM_PATH="${ROCM_SDK}" \ - -DHRR_BUILD_PLAYBACK=ON \ - -DHRR_BUILD_TESTS=ON \ - -DHRR_BUILD_INTEGRATION_TESTS=OFF - - - name: Build - run: cmake --build build -j - - - name: Assert the expected unit cases are present - shell: bash - run: | - # A case behind an #if leaves the binary with no signal at all -- not - # failed, not skipped, absent -- so a fully green suite can hide it. - # Assert the compiled set against the YAML before trusting the run. - python3 -c "import yaml" || python3 -m pip install --no-cache-dir PyYAML==6.0.1 - if [ "${RUNNER_OS}" = "Windows" ]; then - PATH="$(cygpath -u "${ROCM_SDK}")/bin:${PATH}" - export PATH - fi - shopt -s nullglob - bin=(build/tests/unit/hrr-unit-tests*) - if [ ${#bin[@]} -eq 0 ]; then - echo "::error::no unit binary under build/tests/unit" - exit 1 - fi - python3 projects/hrr/tests/scripts/hrr_test_config.py lint \ - projects/hrr/tests/config/unit.yaml \ - projects/hrr/tests/config/integration.yaml \ - --workflow .github/workflows/hrr-ci.yml - python3 projects/hrr/tests/scripts/hrr_test_config.py check \ - --binary "${bin[0]}" \ - --config projects/hrr/tests/config/unit.yaml - - - name: Run unit suite - shell: bash - env: - HRR_TEST_IGNORE_CONFIG: ${{ inputs.ignore_test_config && '1' || '' }} - run: | - # Named hrr-unit-tests.exe on Windows. Resolved with a bash glob - # rather than find, which on a Windows runner resolves to FIND.EXE. - if [ "${RUNNER_OS}" = "Windows" ]; then - # The recovery cases spawn hrr-playback.exe, which loads the SDK's - # HIP DLLs. Without them on PATH it dies with STATUS_DLL_NOT_FOUND - # before running anything. - PATH="$(cygpath -u "${ROCM_SDK}")/bin:${PATH}" - export PATH - fi - shopt -s nullglob - bin=(build/tests/unit/hrr-unit-tests*) - if [ ${#bin[@]} -eq 0 ]; then - echo "::error::no unit binary under build/tests/unit" - ls -R build/tests || true - exit 1 - fi - ls -l "${bin[0]}" - python3 projects/hrr/tests/scripts/run_catch2.py \ - --xml unit-${{ matrix.os }}.xml \ - --config projects/hrr/tests/config/unit.yaml \ - --arch none \ - -- "${bin[0]}" - - # The helpers the suites below rely on (JUnit wrapper, summary table, - # suite config). They are POSIX-process code; once is enough. - # test_api_matrix.py imports PyYAML, which setup-python does not provide. - - name: Test the CI helper scripts - if: ${{ runner.os == 'Linux' }} - run: | - python3 -c "import yaml" || python3 -m pip install --no-cache-dir PyYAML==6.0.1 - python3 -m unittest discover -s projects/hrr/tests/scripts -p 'test_*.py' -v - - - name: Upload results - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: results-unit-${{ matrix.os }} - path: unit-${{ matrix.os }}.xml - retention-days: 3 + run: python .github/scripts/hrr_check_sources.py all --base-sha "${BASE_SHA}" - # --------------------------------------------------------------------------- - # Resolve the GPU runner labels from the shared CI config, the same way the - # rocprofiler-sdk workflows do. - # --------------------------------------------------------------------------- - ci_config: - name: "Resolve runners" - needs: [gate] - # Also runs for a gpu_only dispatch. The integration matrix below reads - # these outputs while that job decides to skip, and a skipped resolver - # makes that evaluation fail. - if: ${{ needs.gate.outputs.run == 'true' }} + scripts: + name: HRR script tests + needs: configure + if: ${{ needs.configure.outputs.run_cpu == 'true' }} runs-on: ubuntu-24.04 - outputs: - gfx90a_runner: ${{ steps.runner.outputs.gfx90a_runner }} - gfx94x_runner: ${{ steps.runner.outputs.gfx94x_runner }} - gfx950_runner: ${{ steps.runner.outputs.gfx950_runner }} - gfx103x_runner: ${{ steps.runner.outputs.gfx103x_runner }} - gfx110x_runner: ${{ steps.runner.outputs.gfx110x_runner }} - gfx120x_runner: ${{ steps.runner.outputs.gfx120x_runner }} - gfx1151_runner: ${{ steps.runner.outputs.gfx1151_runner }} - steps: - - name: Checkout workflow actions - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - sparse-checkout: .github/actions/rocprofiler-sdk-util/resolve-ci-config - set-safe-directory: true - - - name: Resolve runner labels - id: runner - uses: ./.github/actions/rocprofiler-sdk-util/resolve-ci-config - - # --------------------------------------------------------------------------- - # Builds CLR (which brings in-tree ROCr with it) to get an amdhip64 carrying - # the HRR capture shims, then projects/hrr against the SDK. The capture build - # is substituted at run time via LD_LIBRARY_PATH, not linked here. Produces - # one multi-architecture artifact consumed by every GPU job below. - # --------------------------------------------------------------------------- - build: - name: "Build (multi-arch)" - needs: [gate] - if: ${{ github.event.inputs.gpu_only != 'true' && needs.gate.outputs.run == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} - # The integration binaries embed absolute paths into this workspace, so a - # containerised GPU job has to recreate it (see "Recreate build path"). - outputs: - workspace: ${{ github.workspace }} - # The nightly index moves every day. The integration jobs install exactly - # this version so the toolchain (comgr, the device libraries) and the - # sysdeps sonames the in-tree ROCr and CLR use never drift from what they - # were built with, however long a GPU queue is. ROCr itself comes from - # this tree, not the SDK. - sdk_version: ${{ steps.sdk.outputs.version }} - runs-on: azure-linux-scale-rocm - env: - CCACHE_DIR: ${{ github.workspace }}/.ccache - # Hash the compiler by content, not mtime: the SDK wheel is reinstalled - # every run, so an mtime check would miss on every single object. - CCACHE_COMPILERCHECK: content - CCACHE_MAXSIZE: "5G" steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + persist-credentials: false sparse-checkout: | + .github/scripts + .github/workflows/hrr-gpu.yml projects/hrr - projects/clr - projects/hip - projects/rocr-runtime - set-safe-directory: true - - # ROCclr pulls in OpenGL through ROCclrHSA.cmake even with CLR_BUILD_OCL - # off, so the GL development packages are required to configure CLR. - - name: Install system dependencies - run: | - sudo apt-get update -q - sudo apt-get install -y --no-install-recommends \ - ninja-build pkg-config libnuma-dev libelf-dev libdrm-dev \ - libgl1-mesa-dev libglx-dev mesa-common-dev \ - libclang-rt-18-dev ccache xxd - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" - - name: Install ROCm SDK - id: sdk - run: | - # CppHeaderParser is required by hipamd/src/CMakeLists.txt. - pip install --no-cache-dir CppHeaderParser - pip install --no-cache-dir --index-url "${ROCM_INDEX_URL}" "rocm[libraries,devel]" - rocm-sdk init - echo "version=$(python -c 'import importlib.metadata as m; print(m.version("rocm"))')" >> "$GITHUB_OUTPUT" - echo "ROCM_SDK=$(python -c 'import _rocm_sdk_devel,os;print(os.path.dirname(_rocm_sdk_devel.__file__))')" >> "$GITHUB_ENV" - - # Nothing survived between runs before this: the artifact fan-out only - # reuses a build WITHIN a run. ccache is the house convention (15 - # workflows) and is carried across runs by actions/cache (9 workflows) - # because azure-linux-scale-rocm is ephemeral and keeps no state. - # - # This cache is a SPEED optimisation only and can never make a stale - # binary look fresh. The key exists to pick a good starting point; it - # is ccache itself that decides reuse, and it keys on the preprocessed - # source plus the command line plus the compiler contents. A changed - # input is therefore a miss by construction -- restoring a stale cache - # costs misses, never a wrong object. Do not replace this with a - # "skip the build if the key matches" scheme: that trades a correctness - # property for the same speed. - # - # Restore everywhere, save only from a push to develop. A pull request can - # restore its own caches and the base branch's but never write to - # develop's scope, and a per-run entry of up to 5 GB from every PR run - # would evict other workflows' caches. One develop entry per push seeds - # every PR instead. - - name: Restore ccache - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ${{ github.workspace }}/.ccache - key: hrr-ccache-${{ runner.os }}-${{ github.run_id }} - restore-keys: | - hrr-ccache-${{ runner.os }}- + - name: Install test requirements + run: python -m pip install --no-cache-dir PyYAML==6.0.2 pytest==8.4.2 - - name: Zero ccache statistics - run: ccache --zero-stats + - name: Lint the suite configs + run: >- + python projects/hrr/tests/scripts/hrr_test_config.py lint + projects/hrr/tests/config/unit.yaml + projects/hrr/tests/config/integration.yaml + --workflow .github/workflows/hrr-gpu.yml - # CLR is built against this tree's ROCr, never the SDK's: CLR and ROCr - # change together in this repo, and the nightly's ROCr lags (a CLR that - # calls a new hsa_amd_* API does not compile against its headers). - # find_package(hsa-runtime64) finds this install first via - # CMAKE_PREFIX_PATH below. The SDK still supplies the compiler, comgr - # and device libraries. - # - # libdrm, libdrm_amdgpu, libnuma and libelf come from the SDK's bundled - # sysdeps (soname librocm_sysdeps_*), not the apt packages above. The GPU - # machines and the no-ROCm container do not have the system ones, so a - # ROCr linked against them fails to load there (libdrm.so.2). The SDK's - # own ROCr links these same sysdeps. - - name: Build ROCr from this tree - run: | - sysdeps="${ROCM_SDK}/lib/rocm_sysdeps" - export PKG_CONFIG_PATH="${sysdeps}/lib/pkgconfig" - cmake -S projects/rocr-runtime -B build/rocr -GNinja -Wno-dev \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="${sysdeps};${ROCM_SDK}" \ - "-DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/build/rocr-install" - cmake --build build/rocr -j - cmake --install build/rocr - - # -DHIP_HRR_CAPTURE_ENABLED is what compiles capture into amdhip64 - # (hip_capture.cpp); the tree has no CMake option for it. CI always sets - # it, so capture is always on in the build under test. - - name: Build amdhip64 with the HRR capture shims - run: | - cmake -S projects/clr -B build/clr -GNinja -Wno-dev \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - "-DCMAKE_CXX_FLAGS=-DHIP_HRR_CAPTURE_ENABLED -I${GITHUB_WORKSPACE}/build/rocr-install/include -I${GITHUB_WORKSPACE}/build/rocr-install/include/hsa" \ - -DCMAKE_BUILD_TYPE=Release \ - "-DCMAKE_PREFIX_PATH=${GITHUB_WORKSPACE}/build/rocr-install;${ROCM_SDK}" \ - -DROCM_PATH="${ROCM_SDK}" \ - -DHIP_COMMON_DIR="${GITHUB_WORKSPACE}/projects/hip" \ - -DHIP_PLATFORM=amd \ - -DCLR_BUILD_HIP=ON \ - -DCLR_BUILD_OCL=OFF - cmake --build build/clr --target amdhip64 -j - - - name: Build projects/hrr for every testable architecture - run: | - # C and C++ only. CMAKE_HIP_COMPILER_LAUNCHER is deliberately NOT - # set: HIP device compilation fans one TU out to a code object per - # --offload-arch and bundles them, and ccache's handling of that - # multi-output bundling is not something this workflow has proven. - # A wrong hit there would mean testing a binary that does not match - # its source, which is exactly the failure mode this cache is - # required not to introduce. The C++ half (CLR/amdhip64, the bulk of - # the wall clock) is cached; extend to HIP only after a run proves - # the arch-embedding check still passes on a fully cached build. - # Host C++ is compiled against this tree's public HIP headers (and the - # hip_version.h the CLR build above generated) ahead of the SDK's, so - # an API added to the tree builds before any nightly declares it. HIP - # device code keeps the SDK's headers. The SDK's libamdhip64 is only - # linked, never run: the integration jobs put build/clr first. - cmake -S projects/hrr -B build/hrr -GNinja -Wno-dev \ - "-DHRR_HIP_INCLUDE_DIRS=${GITHUB_WORKSPACE}/projects/hip/include;${GITHUB_WORKSPACE}/projects/clr/hipamd/include;${GITHUB_WORKSPACE}/build/clr/hipamd/include" \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="${ROCM_SDK}" \ - -DROCM_PATH="${ROCM_SDK}" \ - -DCMAKE_HIP_COMPILER="${ROCM_SDK}/bin/amdclang++" \ - -DCMAKE_HIP_FLAGS="--rocm-device-lib-path=${ROCM_SDK}/lib/llvm/amdgcn/bitcode" \ - -DGPU_TARGETS="${HRR_ARCHS}" \ - -DCMAKE_HIP_ARCHITECTURES="${HRR_ARCHS}" \ - -DHRR_BUILD_PLAYBACK=ON \ - -DHRR_BUILD_TESTS=ON \ - -DHRR_BUILD_INTEGRATION_TESTS=ON - cmake --build build/hrr -j - - - name: ccache statistics - if: always() - run: ccache --show-stats --verbose || ccache --show-stats - - - name: Assert the expected integration cases are present - run: | - # Listing needs no GPU: Catch2 registers cases in static initialisers - # and --list-tests never calls into HIP. Checked here rather than in - # the integration jobs so a compiled-out case fails once, straight - # away, instead of nine times after the fan-out. - python3 -c "import yaml" || python3 -m pip install --no-cache-dir PyYAML==6.0.1 - python3 projects/hrr/tests/scripts/hrr_test_config.py check \ - --binary build/hrr/tests/integration/hrr-integration-tests \ - --config projects/hrr/tests/config/integration.yaml - - - name: Confirm every architecture is embedded - run: | - found=$("${ROCM_SDK}"/lib/llvm/bin/llvm-objdump --offloading \ - build/hrr/tests/integration/hrr-integration-tests \ - | grep -o 'gfx[0-9a-z]*' | sort -u | tr '\n' ' ') - echo "embedded: ${found}" - for arch in ${HRR_ARCHS//;/ }; do - case " ${found} " in - *" ${arch} "*) ;; - *) echo "::error::${arch} missing from the fat binary"; exit 1 ;; - esac - done - - - name: Save ccache - if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/develop' }} - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ${{ github.workspace }}/.ccache - key: hrr-ccache-${{ runner.os }}-${{ github.run_id }} + - name: Run tests + run: python -m pytest projects/hrr/tests/scripts .github/scripts/tests/hrr_*_test.py - - name: Upload build - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: hrr-build - path: | - build/hrr - build/clr/hipamd/lib - build/rocr-install/lib - retention-days: 3 - - # --------------------------------------------------------------------------- - # Integration check for every family that has a test runner. - # - # This job is the PR check. It runs on a hosted runner and dispatches the - # GPU work as its own workflow run. timeout-minutes does not include queue - # time, and cancelling one job inside this run would cancel the run, so a - # family that has no runner within 30 minutes is cancelled over there and - # reported green here. A device-less runner is handled inside the GPU job - # the same way: skip the suite, upload a skipped result, exit 0. A push to - # develop uses this same result. A suite failure or the execution timeout - # still fails the job. - # --------------------------------------------------------------------------- - integration: - name: "Integration (${{ matrix.family }})" - needs: [gate, ci_config, build] - if: ${{ github.event.inputs.gpu_only != 'true' && needs.gate.outputs.run == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} - runs-on: ubuntu-24.04 - permissions: - actions: write - contents: read - # 30 minutes of queue, then the GPU job's own 30 minute execution cap, - # plus time to publish the JUnit. - timeout-minutes: 80 + unit: + name: Unit (${{ matrix.os }}) + needs: configure + if: ${{ needs.configure.outputs.run_cpu == 'true' }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - - { family: gfx90a, container: true, runner: "${{ needs.ci_config.outputs.gfx90a_runner }}" } - # gfx94x uses the resolved 1-GPU runner: the 8-GPU gfx942 pool - # exposes /dev/kfd only to a job with a container spec. - - { family: gfx94x, runner: "${{ needs.ci_config.outputs.gfx94x_runner }}" } - # The T3 tier needs two visible devices and there is no 2-GPU pool, - # so gfx950 takes an 8-GPU machine; every other family skips T3. - # Do not pin HIP_VISIBLE_DEVICES: on the CCS pool fixed indices can - # be empty even though the job has GPUs. - - { family: gfx950, container: true, runner: "linux-gfx950-8gpu-ccs-ossci-rocm" } - - { family: gfx103x, runner: "${{ needs.ci_config.outputs.gfx103x_runner }}" } - - { family: gfx110x, runner: "${{ needs.ci_config.outputs.gfx110x_runner }}" } - - { family: gfx120x, runner: "${{ needs.ci_config.outputs.gfx120x_runner }}" } - - { family: gfx1151, runner: "${{ needs.ci_config.outputs.gfx1151_runner }}" } - # Not exposed by resolve-ci-config; taken from TheRock's family matrix. - - { family: gfx1150, runner: "linux-gfx1150-gpu-rocm" } - - { family: gfx1153, runner: "linux-gfx1153-gpu-rocm" } - # gfx1250 (MI455) is withheld until linux-mi455-gpu-rocm is healthy. - # If it is wanted sooner, add it as a separate non-blocking job, as - # therock-ci.yml does, so the report job never waits on that queue: - # - # - { family: gfx1250, runner: "linux-mi455-gpu-rocm" } - steps: - - name: Checkout schedule helper - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - sparse-checkout: projects/hrr/tests/scripts/hrr_ci_schedule.py - sparse-checkout-cone-mode: false - set-safe-directory: true - - - name: Dispatch GPU integration - id: gpu - env: - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - # A branch, not a SHA: workflow_dispatch selects its workflow file - # from this ref. The GPU job then checks out BUILD_SHA, which is - # the commit this run compiled (the pull request's merge commit, or - # the pushed commit). The branch tip can move while the build runs. - REF: ${{ github.head_ref || github.ref_name }} - BUILD_SHA: ${{ github.sha }} - FAMILY: ${{ matrix.family }} - RUNNER: ${{ matrix.runner }} - USE_CONTAINER: ${{ matrix.container && 'true' || 'false' }} - SDK_VERSION: ${{ needs.build.outputs.sdk_version }} - BUILD_WORKSPACE: ${{ needs.build.outputs.workspace }} - BUILD_RUN_ID: ${{ github.run_id }} - IGNORE_TEST_CONFIG: ${{ inputs.ignore_test_config && 'true' || 'false' }} - # run_attempt changes on "Rerun failed jobs". The run id does not, so - # without it the previous GPU run is still listed under this title. - DISPATCH_ID: ${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.family }} - run: | - set -euo pipefail - title="hrr-gpu-${DISPATCH_ID}" - sched="projects/hrr/tests/scripts/hrr_ci_schedule.py" - # Runs already using this title belong to an earlier dispatch. - seen="$(gh run list --workflow hrr-ci.yml --branch "${REF}" --limit 40 \ - --json databaseId,displayTitle,name,event \ - --jq "[.[] | select(.event==\"workflow_dispatch\" and (.displayTitle==\"${title}\" or .name==\"${title}\")) | .databaseId] | join(\" \")" \ - || true)" - set +e - dispatch_out="$(gh workflow run hrr-ci.yml --ref "${REF}" \ - -f gpu_only=true \ - -f "family=${FAMILY}" \ - -f "runner=${RUNNER}" \ - -f "use_container=${USE_CONTAINER}" \ - -f "sdk_version=${SDK_VERSION}" \ - -f "build_workspace=${BUILD_WORKSPACE}" \ - -f "build_run_id=${BUILD_RUN_ID}" \ - -f "build_sha=${BUILD_SHA}" \ - -f "dispatch_id=${DISPATCH_ID}" \ - -f "ignore_test_config=${IGNORE_TEST_CONFIG}" 2>&1)" - dispatch_rc=$? - set -e - printf '%s\n' "${dispatch_out}" - if [ "${dispatch_rc}" -ne 0 ]; then - exit "${dispatch_rc}" - fi - - # gh prints the new run URL once it exists. Prefer that id. A URL - # that was already in the list is the previous attempt. - printed="" - while IFS= read -r line; do - case "${line}" in - *actions/runs/[0-9]*) - printed="${line##*/}" - printed="${printed%%[^0-9]*}" - break - ;; - esac - done < "${GITHUB_WORKSPACE}/gpu-run-id.txt" - echo "run_id=${run_id}" >> "${GITHUB_OUTPUT}" - - action="poll" - while true; do - job_status="$(gh run view "${run_id}" --json jobs \ - --jq "[.jobs[] | select(.name==\"Integration GPU\")][0].status // empty" \ - || true)" - run_status="$(gh run view "${run_id}" --json status --jq .status || true)" - action="$(python3 "${sched}" --elapsed "${SECONDS}" \ - --status "${job_status}" --run-status "${run_status}")" - if [ "${action}" != "poll" ]; then - break - fi - sleep 20 - done - - if [ "${action}" = "skip" ]; then - gh run cancel "${run_id}" || true - echo "::notice title=Integration skipped::${FAMILY} was not scheduled within 30 minutes" - python3 "${sched}" --write-skip "integration-${FAMILY}.xml" \ - --message "not scheduled within 30 minutes" - echo "conclusion=skipped" >> "${GITHUB_OUTPUT}" - exit 0 - fi - - exec_deadline=$((SECONDS + 40 * 60)) - while [ "${SECONDS}" -lt "${exec_deadline}" ]; do - run_status="$(gh run view "${run_id}" --json status --jq .status || true)" - if [ "${run_status}" = "completed" ]; then - break - fi - sleep 20 - done - run_status="$(gh run view "${run_id}" --json status --jq .status)" - if [ "${run_status}" != "completed" ]; then - echo "::error::GPU integration ${FAMILY} did not finish" - echo "conclusion=failure" >> "${GITHUB_OUTPUT}" - exit 1 - fi - job_conclusion="$(gh run view "${run_id}" --json jobs \ - --jq "[.jobs[] | select(.name==\"Integration GPU\")][0].conclusion // empty" \ - || true)" - run_conclusion="$(gh run view "${run_id}" --json conclusion --jq .conclusion)" - if [ -z "${job_conclusion}" ] || [ "${job_conclusion}" = "skipped" ]; then - job_conclusion="${run_conclusion}" - fi - echo "conclusion=${job_conclusion}" >> "${GITHUB_OUTPUT}" - - - name: Download GPU results - if: ${{ success() && steps.gpu.outputs.conclusion != 'skipped' && steps.gpu.outputs.conclusion != '' }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: results-integration-${{ matrix.family }} - path: . - run-id: ${{ steps.gpu.outputs.run_id }} - github-token: ${{ github.token }} - - - name: Upload results - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: results-integration-${{ matrix.family }} - path: integration-${{ matrix.family }}.xml - if-no-files-found: ignore - retention-days: 3 - - - name: Propagate GPU result - if: ${{ success() && steps.gpu.outputs.conclusion != 'success' && steps.gpu.outputs.conclusion != 'skipped' }} - run: | - echo "::error::GPU integration ${{ matrix.family }} finished ${{ steps.gpu.outputs.conclusion }}" - exit 1 - - - name: Cancel unfinished GPU run - if: cancelled() || failure() - env: - GH_TOKEN: ${{ github.token }} - run: | - if [ -f gpu-run-id.txt ]; then - id="$(cat gpu-run-id.txt)" - status="$(gh run view "${id}" --json status --jq .status || true)" - if [ -n "${status}" ] && [ "${status}" != "completed" ]; then - gh run cancel "${id}" || true - fi - fi - - # --------------------------------------------------------------------------- - # The GPU suite for one family. Started by the integration job above via - # workflow_dispatch, so its queue can be cancelled without cancelling the PR. - # --------------------------------------------------------------------------- - integration_gpu: - # A skipped job does not evaluate its name, so an expression here is shown - # as text on the pull request. The run name carries the family. - name: Integration GPU - if: ${{ github.event.inputs.gpu_only == 'true' }} - # A pull request does not define this input. Keep a real label here so the - # skipped job still parses; gpu_only replaces it with the family runner. - runs-on: ${{ github.event.inputs.gpu_only == 'true' && github.event.inputs.runner || 'ubuntu-24.04' }} - permissions: - actions: read - contents: read - # The suite runs in about 90 seconds. These are scarce single-GPU machines - # and the default timeout is six hours, so a workload that deadlocks on one - # part would hold a runner for the rest of the day and publish no log to - # say why. Fail instead, while the log still gets uploaded. Queue time is - # not part of this budget; the dispatcher owns that. - timeout-minutes: 30 - # The OSSCI pod pools (gfx90a, gfx950) expose /dev/kfd only to a job with - # a container spec: run bare there and hipGetDeviceCount returns 100 with - # no devices. Every other pool is a bare machine and gets no container, - # since a null image runs the job directly on the runner. - container: - image: ${{ github.event.inputs.use_container == 'true' && 'ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98' || null }} - options: >- - --ipc host - --group-add video - --device /dev/kfd - --device /dev/dri - --group-add 110 - --group-add 993 - --group-add 992 - --ulimit memlock=-1:-1 - --security-opt seccomp=unconfined - --env-file /etc/podinfo/gha-gpu-isolation-settings - --user 0:0 + - os: ubuntu-24.04 + exe: hrr-unit-tests + linker_flags: "" + - os: windows-2022 + exe: hrr-unit-tests.exe + # The MinGW runtime DLLs are not on PATH in later steps. + linker_flags: "-static" defaults: run: shell: bash steps: - # Only the test runner: this job compiles nothing and takes its binaries - # from the build artifact. It has to come first, because checkout cleans - # the workspace and would delete an artifact unpacked before it. - # build_sha is the parent run's commit. Without it, this checkout follows - # the branch named in the dispatch and can pick up a newer config than - # the binaries. - - name: Checkout test runner + - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ inputs.build_sha || github.sha }} + persist-credentials: false sparse-checkout: | - projects/hrr/tests/scripts/run_catch2.py - projects/hrr/tests/scripts/hrr_test_config.py - projects/hrr/tests/scripts/hrr_ci_schedule.py - projects/hrr/tests/config/integration.yaml - sparse-checkout-cone-mode: false - set-safe-directory: true + projects/hrr + projects/hip/include + projects/clr/hipamd/include - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" - # Invoked through "python -m" and by script path throughout: on some of - # these self-hosted machines the console-script wrappers cannot execute, - # and the system Python has no ensurepip to build a virtualenv with. - - name: Install ROCm SDK - env: - SDK_VERSION: ${{ inputs.sdk_version }} + - name: Install the ROCm SDK run: | - # devel as well as libraries: rocm-sdk init refuses to run without it. - # The version the build job used, not whatever the index has now. - python -m pip install --no-cache-dir \ - --index-url "${ROCM_INDEX_URL}" "rocm[libraries,devel]${SDK_VERSION:+==${SDK_VERSION}}" - # Run as a module where possible. Falling back to the script means - # locating it: some of these machines install into a --user bin that - # is not on PATH, leaving command -v empty. - if ! python -m rocm_sdk init; then - cli="$(command -v rocm-sdk || true)" - if [ -z "${cli}" ]; then - cli="$(python -c 'import site;print(site.USER_BASE)')/bin/rocm-sdk" - fi - python "${cli}" init - fi - echo "ROCM_SDK=$(python -c 'import _rocm_sdk_devel,os;print(os.path.dirname(_rocm_sdk_devel.__file__))')" >> "$GITHUB_ENV" + python -m pip install --no-cache-dir PyYAML==6.0.2 + python -m pip install --no-cache-dir --index-url "${ROCM_INDEX_URL}" "rocm[devel]==${ROCM_SDK_VERSION}" + rocm-sdk init + echo "ROCM_SDK=$(rocm-sdk path --root)" >> "${GITHUB_ENV}" + rocm-sdk path --bin >> "${GITHUB_PATH}" - # Unpacked under build/ so the tree matches the build machine's layout. - # The suite spawns each workload by re-execing HRR_TEST_EXE, which CMake - # bakes in as an absolute path, so the binary has to sit where it was - # built or every workload dies in execvp with 127. - - name: Download build - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: hrr-build - path: build - run-id: ${{ inputs.build_run_id }} - github-token: ${{ github.token }} + # This tree's HIP headers come first: an API added here is not in the + # pinned SDK's headers yet. The SDK supplies the hip CMake package. + - name: Configure + env: + LINKER_FLAGS: ${{ matrix.linker_flags }} + run: >- + cmake -S projects/hrr -B build -GNinja -Wno-dev + -DCMAKE_BUILD_TYPE=Release + "-DCMAKE_PREFIX_PATH=${ROCM_SDK}" + "-DROCM_PATH=${ROCM_SDK}" + "-DHRR_HIP_INCLUDE_DIRS=${PWD}/projects/hip/include;${PWD}/projects/clr/hipamd/include" + "-DCMAKE_EXE_LINKER_FLAGS=${LINKER_FLAGS}" + -DHRR_BUILD_PLAYBACK=ON + -DHRR_BUILD_TESTS=ON + -DHRR_BUILD_INTEGRATION_TESTS=OFF - - name: Restore executable bits - run: chmod +x build/hrr/playback/hrr-playback build/hrr/tests/*/hrr-*tests + - name: Build + run: cmake --build build -j - # In a container the workspace is /__w/..., while HRR_TEST_EXE, - # HRR_PLAYBACK_EXE and the SDK RUNPATH were baked on the build machine. - # The tests re-exec /proc/self/exe, because a symlink to the checkout - # created on the container root can sit on a noexec mount: test -x - # succeeds and execve returns EACCES, which the suite reports as 127. - # The tool-cache link is what makes that RUNPATH resolve when a child - # cannot use LD_LIBRARY_PATH. ln -sfn will not replace a real directory. - - name: Recreate build path + # A case compiled out by an #if leaves no failed or skipped result, so + # compare the compiled case list with the suite config first. + - name: Check the compiled case list env: - BUILD_WORKSPACE: ${{ inputs.build_workspace }} - run: | - # Never delete a real directory. On a shared host the baked path can - # be another runner's checkout. Only create the link when the path - # is missing or is already a symlink. - link_into() { - local path="$1" target="$2" - if [ -e "${path}" ] && [ ! -L "${path}" ]; then - echo "::error::${path} exists and is not a symlink; not deleting it" - exit 1 - fi - mkdir -p "$(dirname "${path}")" - ln -sfn "${target}" "${path}" - } - if [ -n "${BUILD_WORKSPACE}" ] && [ "${BUILD_WORKSPACE}" != "${PWD}" ]; then - link_into "${BUILD_WORKSPACE}" "${PWD}" - fi - # Build-machine SDK path -> this container's tool cache. - if [ -d /__w/_tool ]; then - link_into /home/runner/_work/_tool /__w/_tool - fi - exe="${BUILD_WORKSPACE:-${PWD}}/build/hrr/tests/integration/hrr-integration-tests" - if [ ! -x "${exe}" ]; then - echo "::error::baked test binary is not executable: ${exe} (PWD=${PWD})" - ls -ld "${BUILD_WORKSPACE}" "${PWD}/build/hrr/tests/integration" || true - exit 1 - fi + EXE: ${{ matrix.exe }} + run: >- + python projects/hrr/tests/scripts/hrr_test_config.py check + --binary "build/tests/unit/${EXE}" + --config projects/hrr/tests/config/unit.yaml - # CMake gives the binaries a RUNPATH to the SDK's lib directory, where - # the plain amdhip64 lives. LD_LIBRARY_PATH is searched before RUNPATH, so - # putting the capture build first here is what selects it. - - name: Run integration suite + - name: Run unit suite env: + EXE: ${{ matrix.exe }} + OS: ${{ matrix.os }} HRR_TEST_IGNORE_CONFIG: ${{ inputs.ignore_test_config && '1' || '' }} - run: | - # Use a PyYAML that is already importable. Forcing 6.0.1 uninstalls - # 6.0.3 on runners whose site-packages is read-only. - python3 -c "import yaml" || python3 -m pip install --no-cache-dir PyYAML==6.0.1 - # hipamd/lib holds the amdhip64 carrying the capture shims and must - # precede the SDK, which ships a plain build under the same soname. - # rocr-install/lib is the ROCr built from this tree, which hipamd/lib's - # amdhip64 was compiled against; it must precede the SDK's ROCr. - # Keep the runner's existing path: setup-python puts libpython there, - # and replacing it makes python3 fail to start on some GPU machines. - # rocm_sysdeps/lib holds the librocm_sysdeps_* libraries that rocr-install's - # libhsa-runtime64 links (drm, numa, elf); it has no RUNPATH of its own. - base_ld="${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - export LD_LIBRARY_PATH="${PWD}/build/clr/hipamd/lib:${PWD}/build/rocr-install/lib:${ROCM_SDK}/lib:${ROCM_SDK}/lib/rocm_sysdeps/lib${base_ld}" - # The probe below uses the SDK's own amdhip64 and ROCr instead. CLR - # also returns hipErrorNoDevice when hsa_init fails, so probing with - # the in-tree ROCr would turn a broken in-tree ROCr into a green skip. - probe_ld="${ROCM_SDK}/lib:${ROCM_SDK}/lib/rocm_sysdeps/lib${base_ld}" - # A labelled runner with no visible device is an infrastructure miss. - # The probe runs against the SDK's runtime, so only the machine can - # make it report no device; the suite below runs against this tree's. - # Skip the suite and leave a skipped result so the check stays green. - # Exit 1 is "confirmed no device": hipErrorNoDevice (100) or a count - # of zero. Any other probe failure (broken ctypes, missing libpython, - # an init or ABI error status) is not a device count and must not - # skip the GPU suite. - set +e - LD_LIBRARY_PATH="${probe_ld}" python3 - <<'PY' - import sys - - try: - import ctypes - hip = ctypes.CDLL("libamdhip64.so.7") - count = ctypes.c_int() - status = hip.hipGetDeviceCount(ctypes.byref(count)) - except Exception as exc: - print(f"GPU probe unavailable: {type(exc).__name__}: {exc}", file=sys.stderr) - sys.exit(2) - print(f"hipGetDeviceCount: status={status}, devices={count.value}") - if status == 0 and count.value > 0: - sys.exit(0) - # hipErrorNoDevice. Other statuses fall through to the suite, which - # then fails visibly instead of being reported as a skip. - sys.exit(1 if status == 100 or (status == 0 and count.value == 0) else 3) - PY - probe_rc=$? - set -e - if [ "$probe_rc" -eq 1 ]; then - echo "::notice title=Integration skipped::GPU runner has no visible ROCm-capable device" - python3 projects/hrr/tests/scripts/hrr_ci_schedule.py \ - --write-skip "integration-${{ inputs.family }}.xml" \ - --message "GPU runner has no visible ROCm-capable device" - exit 0 - fi - # Keep passing logs to one result line per case. The wrapper captures - # Catch2 and subprocess output, and only emits it when the suite fails. - # Time out inside the job's 30 minutes, so a hang prints the cases - # that finished instead of being cancelled with no output. - # - # Which cases run is the suite config, not a selector written here. - # It lists the hidden [hrr] drivers (T0-T4, the kernel-argument - # regression, the HSA first-touch case) and leaves out the direct - # workloads they spawn, T5, and [hrr-repro]. --arch auto reads the - # GPU this machine actually has, so a rule can name one architecture - # of a wildcard pool. HRR_TEST_IGNORE_CONFIG runs the skipped cases. - python3 projects/hrr/tests/scripts/run_catch2.py \ - --xml integration-${{ inputs.family }}.xml --timeout 1200 \ - --config projects/hrr/tests/config/integration.yaml \ - --os linux --arch auto \ - -- ./build/hrr/tests/integration/hrr-integration-tests + run: >- + python projects/hrr/tests/scripts/run_catch2.py + --xml "unit-${OS}.xml" + --config projects/hrr/tests/config/unit.yaml + --arch none + -- "build/tests/unit/${EXE}" - name: Upload results - if: always() + if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: results-integration-${{ inputs.family }} - path: integration-${{ inputs.family }}.xml - retention-days: 3 - - # --------------------------------------------------------------------------- - # Per-test reporting for every job above. Collected centrally because the - # publishing action is a container action and so cannot run on the Windows - # runners. - # --------------------------------------------------------------------------- - report: - name: "Test results" - needs: [gate, unit, integration] - if: ${{ always() && github.event.inputs.gpu_only != 'true' && needs.gate.outputs.run == 'true' }} + name: results-unit-${{ matrix.os }} + path: unit-${{ matrix.os }}.xml + if-no-files-found: ignore + retention-days: 7 + + gpu: + name: GPU + needs: configure + if: ${{ needs.configure.outputs.run_gpu == 'true' }} + uses: ./.github/workflows/hrr-gpu.yml + # No secrets: the GPU workflow reads public artifacts with the job token. + with: + families: ${{ needs.configure.outputs.families }} + ignore_test_config: ${{ inputs.ignore_test_config || false }} + + summary: + name: HRR CI Summary + needs: [configure, codegen, scripts, unit, gpu] + if: ${{ always() && (needs.configure.outputs.run_cpu == 'true' || needs.configure.outputs.run_gpu == 'true') }} runs-on: ubuntu-24.04 permissions: + actions: read contents: read - checks: write - pull-requests: write steps: - - name: Checkout summary script + - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + persist-credentials: false sparse-checkout: | + .github/scripts/hrr_ci_report.py projects/hrr/tests/scripts/summarize_junit.py sparse-checkout-cone-mode: false - set-safe-directory: true - - name: Download all results + - name: Download results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: results-* merge-multiple: true + path: results - - name: Platform / family table - if: always() - run: | - # Written first so it sits above EnricoMi's suite dump on the job - # summary and on the HRR test results check page. - python3 projects/hrr/tests/scripts/summarize_junit.py \ - --dir . \ - --summary "${GITHUB_STEP_SUMMARY}" - - - name: Publish - # Runs even when the table step exits 1 on unreadable XML. - if: always() - uses: EnricoMi/publish-unit-test-result-action@d0a4676d0e0b938bc201470d88276b7c74c712b3 # v2.24.0 + - name: Download base build record + if: ${{ needs.configure.outputs.run_gpu == 'true' }} + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - files: "*.xml" - check_name: "HRR test results" - comment_mode: "off" - # Fork pull requests get a read-only token, and creating a check run - # then fails with "Resource not accessible by integration". They keep - # the job summary, which needs no token. - check_run: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork) }} - job_summary: true - report_individual_runs: true - deduplicate_classes_by_file_name: true + name: hrr-base-${{ github.sha }} + path: base - # --------------------------------------------------------------------------- - # Single required-check target. Passes when the path gate skipped everything - # and when every job that ran succeeded; fails on any failure or cancellation, - # and on a skip the gate does not explain. Fork PRs skip build and integration - # on purpose, so those skips are accepted there. - # --------------------------------------------------------------------------- - hrr_ci_summary: - name: "HRR CI Summary" - if: ${{ always() && github.event.inputs.gpu_only != 'true' }} - needs: [gate, codegen, unit, ci_config, build, integration, report] - runs-on: ubuntu-24.04 - steps: - - name: Evaluate + - name: Summarize env: - NEEDS_JSON: ${{ toJSON(needs) }} - IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork }} - run: | - python3 - <<'PY' - import json, os, sys - - needs = json.loads(os.environ["NEEDS_JSON"]) - result = {name: job["result"] for name, job in needs.items()} - print(json.dumps(result, indent=2)) - - if result["gate"] != "success": - print("::error::path gate did not complete: " + result["gate"]) - sys.exit(1) - if needs["gate"]["outputs"].get("run") != "true": - print("HRR CI skipped: the pull request changes no HRR path.") - sys.exit(0) - - fork = os.environ["IS_FORK"] == "true" - optional = {"build", "integration"} if fork else set() - bad = [n for n, r in result.items() - if r not in ("success",) and not (r == "skipped" and n in optional)] - if bad: - print("::error::HRR CI jobs not successful: " + ", ".join(sorted(bad))) - sys.exit(1) - PY + GH_TOKEN: ${{ github.token }} + REASON: ${{ needs.configure.outputs.reason }} + RUN_GPU: ${{ needs.configure.outputs.run_gpu }} + FAMILIES: ${{ needs.configure.outputs.families }} + run: >- + python3 .github/scripts/hrr_ci_report.py + --reason "${REASON}" --run-gpu "${RUN_GPU}" --families "${FAMILIES}" + --results-dir results --base-json base/base.json diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml new file mode 100644 index 000000000000..e1f87983f701 --- /dev/null +++ b/.github/workflows/hrr-gpu.yml @@ -0,0 +1,188 @@ +# HRR GPU suite: build an amdhip64 with HRR capture compiled in, then run the +# HRR capture/replay integration suite on each requested GPU family. +# +# Called by hrr-ci.yml, which decides when this runs and on which families. +# +# Interim: the build job and .github/scripts/hrr_capture_build.py exist only +# until TheRock provides an hrr-capture build variant. After that, this file +# calls TheRock's multi-arch workflows with that variant instead. + +name: HRR GPU + +on: + workflow_call: + inputs: + families: + description: JSON list of {family, runner, gfx_targets} from hrr_ci_configure.py + type: string + required: true + ignore_test_config: + description: Run the cases the suite config skips + type: boolean + default: false + +permissions: + contents: read + +env: + # Every architecture the integration binary is built for. Must match + # targets.arch in projects/hrr/tests/config/*.yaml (hrr_test_config.py lint). + HRR_ARCHS: "gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1153;gfx1200;gfx1201;gfx1250" + +jobs: + build: + name: Build capture-enabled HIP and HRR + runs-on: ubuntu-24.04 + timeout-minutes: 90 + container: + image: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:cf4f6d9909056906e4a1c0bc4c245658a0af4c6a6174082a86adfa2d518b7aeb # 2026-08-18 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + .github/scripts + projects/clr + projects/hip + projects/hrr + + - name: Checkout TheRock artifact tools + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ROCm/TheRock + # Keep in sync with the TheRock ref in therock-multi-arch-ci.yml. + ref: 05add1921fd03b0649f054ec2a7c54d162c0cd99 # 2026-10-08 + path: therock + sparse-checkout: | + build_tools + cmake/therock_amdgpu_targets.cmake + requirements.txt + sparse-checkout-cone-mode: false + + - name: Install TheRock Python requirements + run: python3 -m pip install --no-cache-dir -r therock/requirements.txt + + - name: Install base ROCm artifacts + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 .github/scripts/hrr_capture_build.py fetch-base + + - name: Build + run: python3 .github/scripts/hrr_capture_build.py build + + - name: Verify + run: python3 .github/scripts/hrr_capture_build.py verify + + - name: Upload base build record + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hrr-base-${{ github.sha }} + path: build/base.json + if-no-files-found: error + retention-days: 7 + + - name: Upload build + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hrr-capture-${{ github.sha }} + path: | + build/base.json + build/capture-lib/ + build/hrr/playback/hrr-playback + build/hrr/tests/integration/hrr-integration-tests + build/hrr/tests/integration/*.so + if-no-files-found: error + retention-days: 7 + + test: + name: Integration (${{ matrix.family }}) + needs: build + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.families) }} + # The image and options TheRock's test_component.yml uses for GPU tests. + # Running in a container also gives this job the build job's workspace + # path, which the test binaries have compiled in. + container: + image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98 + options: >- + --ipc host + --user 0:0 + --ulimit memlock=-1:-1 + --ulimit nofile=1048576:1048576 + --security-opt seccomp=unconfined + --group-add video + --device /dev/kfd + --device /dev/dri + --group-add 993 + --group-add 992 + --group-add 110 + --env-file /etc/podinfo/gha-gpu-isolation-settings + -e ROCR_VISIBLE_DEVICES + -e KUBE_CPU_REQUEST + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + .github/scripts + projects/hrr/tests/config + projects/hrr/tests/scripts + + - name: Checkout TheRock artifact tools + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ROCm/TheRock + ref: 05add1921fd03b0649f054ec2a7c54d162c0cd99 # 2026-10-08 + path: therock + sparse-checkout: | + build_tools + cmake/therock_amdgpu_targets.cmake + requirements.txt + sparse-checkout-cone-mode: false + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install Python requirements + run: python -m pip install --no-cache-dir -r therock/requirements.txt + + - name: Download build + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: hrr-capture-${{ github.sha }} + path: build + + - name: Install base ROCm artifacts and the capture-enabled amdhip64 + run: python .github/scripts/hrr_capture_build.py install-base + + - name: Run integration suite + env: + FAMILY: ${{ matrix.family }} + HRR_TEST_IGNORE_CONFIG: ${{ inputs.ignore_test_config && '1' || '' }} + # --arch auto finds this machine's GPU with the installed rocminfo. + ROCM_SDK: ${{ github.workspace }}/dist + run: >- + python projects/hrr/tests/scripts/run_catch2.py + --xml "integration-${FAMILY}.xml" + --timeout 1200 + --config projects/hrr/tests/config/integration.yaml + --os linux --arch auto + -- build/hrr/tests/integration/hrr-integration-tests + + - name: Upload results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: results-integration-${{ matrix.family }} + path: integration-${{ matrix.family }}.xml + if-no-files-found: ignore + retention-days: 7 From e9a89d25990b6aac7d5e37d983ce3ad5d7fab157 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 13 Oct 2026 12:41:43 +0200 Subject: [PATCH 06/89] ci(hrr): check out amdhip.def.in for the script tests gen_hrr_api_args.py reads the Windows export list when it is imported, so pytest stopped at collection without it. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 6ab4ec0fba09..5c680e89587e 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -142,10 +142,13 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + # Cone mode also checks out projects/clr/hipamd/src/amdhip.def.in, + # which gen_hrr_api_args.py reads on import. sparse-checkout: | .github/scripts .github/workflows/hrr-gpu.yml projects/hrr + projects/clr/hipamd/src/hrr - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: From 453bdbc333d5cbf7a739d23653b30047b2e8906a Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 13 Oct 2026 12:41:43 +0200 Subject: [PATCH 07/89] ci(hrr): pass Windows paths to cmake for the in-tree HIP headers Git Bash's ${PWD} is an MSYS path that a native cmake cannot resolve, so the Windows unit build fell back to the pinned SDK's older HIP headers. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 5c680e89587e..872ad244b32d 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -210,19 +210,21 @@ jobs: # This tree's HIP headers come first: an API added here is not in the # pinned SDK's headers yet. The SDK supplies the hip CMake package. + # On Windows, cygpath -m gives the drive-letter form a native cmake accepts. - name: Configure env: LINKER_FLAGS: ${{ matrix.linker_flags }} - run: >- - cmake -S projects/hrr -B build -GNinja -Wno-dev - -DCMAKE_BUILD_TYPE=Release - "-DCMAKE_PREFIX_PATH=${ROCM_SDK}" - "-DROCM_PATH=${ROCM_SDK}" - "-DHRR_HIP_INCLUDE_DIRS=${PWD}/projects/hip/include;${PWD}/projects/clr/hipamd/include" - "-DCMAKE_EXE_LINKER_FLAGS=${LINKER_FLAGS}" - -DHRR_BUILD_PLAYBACK=ON - -DHRR_BUILD_TESTS=ON - -DHRR_BUILD_INTEGRATION_TESTS=OFF + run: | + tree="$(cygpath -m "${PWD}" 2>/dev/null || echo "${PWD}")" + cmake -S projects/hrr -B build -GNinja -Wno-dev \ + -DCMAKE_BUILD_TYPE=Release \ + "-DCMAKE_PREFIX_PATH=${ROCM_SDK}" \ + "-DROCM_PATH=${ROCM_SDK}" \ + "-DHRR_HIP_INCLUDE_DIRS=${tree}/projects/hip/include;${tree}/projects/clr/hipamd/include" \ + "-DCMAKE_EXE_LINKER_FLAGS=${LINKER_FLAGS}" \ + -DHRR_BUILD_PLAYBACK=ON \ + -DHRR_BUILD_TESTS=ON \ + -DHRR_BUILD_INTEGRATION_TESTS=OFF - name: Build run: cmake --build build -j From 0dc2f9c21515ec61c2805ed7f8575cd0d2b8ee65 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 13 Oct 2026 13:18:43 +0200 Subject: [PATCH 08/89] ci(hrr): move the HRR CI scripts into projects/hrr The scripts change with HRR, so they now sit with the other HRR helpers, where hrr-reviewers own them. Only the two workflows stay under .github. - hrr_ci_configure.py, hrr_capture_build.py and hrr_ci_report.py move to projects/hrr/tests/scripts, beside run_catch2.py and summarize_junit.py. hrr_ci_report.py imports summarize_junit as a sibling module. - hrr_check_sources.py moves to projects/hrr/tools, beside gen_hrr_api_args.py and check_hrr_api_compat.py. - Their tests become test_.py in projects/hrr/tests/scripts, so the script-test job runs one pytest over that directory. The workflows' sparse checkouts and the configure script's path lists follow the new locations. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 20 +++++++------- .github/workflows/hrr-gpu.yml | 16 +++++------- projects/hrr/tests/README.md | 8 +++--- .../hrr/tests}/scripts/hrr_capture_build.py | 2 +- .../hrr/tests}/scripts/hrr_ci_configure.py | 8 +++--- .../hrr/tests}/scripts/hrr_ci_report.py | 19 +++----------- .../tests/scripts/test_hrr_capture_build.py | 9 ++++--- .../tests/scripts/test_hrr_check_sources.py | 10 +++---- .../tests/scripts/test_hrr_ci_configure.py | 10 +++---- .../hrr/tests/scripts/test_hrr_ci_report.py | 26 ++++++++++++------- .../hrr/tools}/hrr_check_sources.py | 0 11 files changed, 61 insertions(+), 67 deletions(-) rename {.github => projects/hrr/tests}/scripts/hrr_capture_build.py (99%) rename {.github => projects/hrr/tests}/scripts/hrr_ci_configure.py (98%) rename {.github => projects/hrr/tests}/scripts/hrr_ci_report.py (90%) rename .github/scripts/tests/hrr_capture_build_test.py => projects/hrr/tests/scripts/test_hrr_capture_build.py (96%) rename .github/scripts/tests/hrr_check_sources_test.py => projects/hrr/tests/scripts/test_hrr_check_sources.py (96%) rename .github/scripts/tests/hrr_ci_configure_test.py => projects/hrr/tests/scripts/test_hrr_ci_configure.py (97%) rename .github/scripts/tests/hrr_ci_report_test.py => projects/hrr/tests/scripts/test_hrr_ci_report.py (80%) rename {.github/scripts => projects/hrr/tools}/hrr_check_sources.py (100%) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 872ad244b32d..70503463b576 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -1,7 +1,8 @@ # HRR CI: generated-source checks, the HRR helper-script tests and the host # unit suite, plus the GPU capture/replay suite (hrr-gpu.yml) for changes that -# touch capture or replay. .github/scripts/hrr_ci_configure.py decides what -# runs; "HRR CI Summary" reports every platform and GPU family in one table. +# touch capture or replay. projects/hrr/tests/scripts/hrr_ci_configure.py +# decides what runs; "HRR CI Summary" reports every platform and GPU family in +# one table. name: HRR CI @@ -54,7 +55,8 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - sparse-checkout: .github/scripts + sparse-checkout: projects/hrr/tests/scripts/hrr_ci_configure.py + sparse-checkout-cone-mode: false - name: Checkout TheRock family matrix uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -94,7 +96,7 @@ jobs: IS_FORK: ${{ github.event.pull_request.head.repo.fork || false }} FAMILIES: ${{ inputs.families }} run: >- - python .github/scripts/hrr_ci_configure.py + python projects/hrr/tests/scripts/hrr_ci_configure.py --event "${EVENT}" --repo "${REPO}" --pr-number "${PR_NUMBER}" --before "${BEFORE}" --after "${AFTER}" --labels "${LABELS}" --is-fork "${IS_FORK}" --families "${FAMILIES}" @@ -114,7 +116,6 @@ jobs: fetch-depth: 0 filter: blob:none sparse-checkout: | - .github/scripts projects/hrr projects/hip/include projects/clr/hipamd/include @@ -130,7 +131,7 @@ jobs: - name: Check generated sources, API IDs and the API matrix env: BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} - run: python .github/scripts/hrr_check_sources.py all --base-sha "${BASE_SHA}" + run: python projects/hrr/tools/hrr_check_sources.py all --base-sha "${BASE_SHA}" scripts: name: HRR script tests @@ -145,7 +146,6 @@ jobs: # Cone mode also checks out projects/clr/hipamd/src/amdhip.def.in, # which gen_hrr_api_args.py reads on import. sparse-checkout: | - .github/scripts .github/workflows/hrr-gpu.yml projects/hrr projects/clr/hipamd/src/hrr @@ -165,7 +165,7 @@ jobs: --workflow .github/workflows/hrr-gpu.yml - name: Run tests - run: python -m pytest projects/hrr/tests/scripts .github/scripts/tests/hrr_*_test.py + run: python -m pytest projects/hrr/tests/scripts unit: name: Unit (${{ matrix.os }}) @@ -284,7 +284,7 @@ jobs: with: persist-credentials: false sparse-checkout: | - .github/scripts/hrr_ci_report.py + projects/hrr/tests/scripts/hrr_ci_report.py projects/hrr/tests/scripts/summarize_junit.py sparse-checkout-cone-mode: false @@ -310,6 +310,6 @@ jobs: RUN_GPU: ${{ needs.configure.outputs.run_gpu }} FAMILIES: ${{ needs.configure.outputs.families }} run: >- - python3 .github/scripts/hrr_ci_report.py + python3 projects/hrr/tests/scripts/hrr_ci_report.py --reason "${REASON}" --run-gpu "${RUN_GPU}" --families "${FAMILIES}" --results-dir results --base-json base/base.json diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index e1f87983f701..d34c6fdf5f0e 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -3,9 +3,9 @@ # # Called by hrr-ci.yml, which decides when this runs and on which families. # -# Interim: the build job and .github/scripts/hrr_capture_build.py exist only -# until TheRock provides an hrr-capture build variant. After that, this file -# calls TheRock's multi-arch workflows with that variant instead. +# Interim: the build job and projects/hrr/tests/scripts/hrr_capture_build.py +# exist only until TheRock provides an hrr-capture build variant. After that, +# this file calls TheRock's multi-arch workflows with that variant instead. name: HRR GPU @@ -42,7 +42,6 @@ jobs: with: persist-credentials: false sparse-checkout: | - .github/scripts projects/clr projects/hip projects/hrr @@ -67,13 +66,13 @@ jobs: - name: Install base ROCm artifacts env: GITHUB_TOKEN: ${{ github.token }} - run: python3 .github/scripts/hrr_capture_build.py fetch-base + run: python3 projects/hrr/tests/scripts/hrr_capture_build.py fetch-base - name: Build - run: python3 .github/scripts/hrr_capture_build.py build + run: python3 projects/hrr/tests/scripts/hrr_capture_build.py build - name: Verify - run: python3 .github/scripts/hrr_capture_build.py verify + run: python3 projects/hrr/tests/scripts/hrr_capture_build.py verify - name: Upload base build record uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -131,7 +130,6 @@ jobs: with: persist-credentials: false sparse-checkout: | - .github/scripts projects/hrr/tests/config projects/hrr/tests/scripts @@ -162,7 +160,7 @@ jobs: path: build - name: Install base ROCm artifacts and the capture-enabled amdhip64 - run: python .github/scripts/hrr_capture_build.py install-base + run: python projects/hrr/tests/scripts/hrr_capture_build.py install-base - name: Run integration suite env: diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index 2b4a4f382398..d5e05d46ded2 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -125,9 +125,11 @@ PyYAML (used by the api-matrix tools), not ROCm: python3 -m unittest discover -s projects/hrr/tests/scripts -p 'test_*.py' -v ``` -They cover `run_catch2.py`, `summarize_junit.py`, and `hrr_test_config.py`, plus -the tools in `../tools`: the HRR API ID compatibility check, the playback-handler default -in the generator, and the [api-matrix](../tools/api-matrix/README.md) scripts. +They cover `run_catch2.py`, `summarize_junit.py`, `hrr_test_config.py`, and the +HRR CI scripts (`hrr_ci_configure.py`, `hrr_capture_build.py`, `hrr_ci_report.py`), +plus the tools in `../tools`: the HRR API ID compatibility check, the +generated-source checks (`hrr_check_sources.py`), the playback-handler default in +the generator, and the [api-matrix](../tools/api-matrix/README.md) scripts. ## Configuring where a case runs diff --git a/.github/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py similarity index 99% rename from .github/scripts/hrr_capture_build.py rename to projects/hrr/tests/scripts/hrr_capture_build.py index 1d232f147755..fcbe997da500 100644 --- a/.github/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -5,7 +5,7 @@ Interim: TheRock has no capture-enabled build yet. Once rocm-systems CI can request TheRock's hrr-capture build variant, hrr-gpu.yml calls TheRock's -workflows instead and this script is deleted. +workflows instead and this script and test_hrr_capture_build.py are deleted. Release builds compile capture out (HIP_HRR_CAPTURE_ENABLED undefined). This script installs the ROCm artifacts that rocm-systems' own TheRock CI built for diff --git a/.github/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py similarity index 98% rename from .github/scripts/hrr_ci_configure.py rename to projects/hrr/tests/scripts/hrr_ci_configure.py index c780f7918028..4ac2f7dcf1b7 100644 --- a/.github/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -42,8 +42,6 @@ "projects/hip/include/hip/hip_runtime_api.h", ".github/workflows/hrr-ci.yml", ".github/workflows/hrr-gpu.yml", - ".github/scripts/hrr_", - ".github/scripts/tests/hrr_", ) # hrr-skills.yml covers these. @@ -68,11 +66,11 @@ "projects/hrr/tests/scripts/run_catch2.py", "projects/hrr/tests/scripts/hrr_test_config.py", "projects/hrr/tests/scripts/summarize_junit.py", + "projects/hrr/tests/scripts/hrr_capture_build.py", + "projects/hrr/tests/scripts/hrr_ci_configure.py", + "projects/hrr/tests/scripts/hrr_ci_report.py", ".github/workflows/hrr-ci.yml", ".github/workflows/hrr-gpu.yml", - ".github/scripts/hrr_capture_build.py", - ".github/scripts/hrr_ci_configure.py", - ".github/scripts/hrr_ci_report.py", ) # TheRock family keys the HRR fat binary covers (HRR_ARCHS in hrr-gpu.yml). diff --git a/.github/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py similarity index 90% rename from .github/scripts/hrr_ci_report.py rename to projects/hrr/tests/scripts/hrr_ci_report.py index 2d45e29eca4a..d8a9308dd0d1 100644 --- a/.github/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -8,7 +8,7 @@ * why this run tested what it did, and which base build the GPU suite used; * the conclusion of every unit, build and GPU job, so a job that never produced results (cancelled, timed out, still queued) still has a row; - * the per-suite results table from projects/hrr's summarize_junit.py. + * the per-suite results table from summarize_junit.py. Exits 1 when any HRR job failed or was cancelled, or a results file could not be read. @@ -17,7 +17,6 @@ from __future__ import annotations import argparse -import importlib.util import json import os import re @@ -25,6 +24,8 @@ import sys from pathlib import Path +import summarize_junit + JOB_PATTERNS = ( ("Generated sources", re.compile(r"^Generated sources$")), ("Script tests", re.compile(r"^HRR script tests$")), @@ -90,13 +91,6 @@ def overall_ok(rows: list[dict], missing: list[str]) -> bool: return not missing and not any(r["conclusion"] in BAD_CONCLUSIONS for r in rows) -def load_summarize_junit(path: Path): - spec = importlib.util.spec_from_file_location("summarize_junit", path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - def list_jobs(repo: str, run_id: str, attempt: str) -> list[dict]: out = subprocess.run( [ @@ -125,11 +119,6 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--families", default="[]") parser.add_argument("--results-dir", type=Path, default=Path("results")) parser.add_argument("--base-json", type=Path) - parser.add_argument( - "--summarize-junit", - type=Path, - default=Path("projects/hrr/tests/scripts/summarize_junit.py"), - ) parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "")) parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID", "")) parser.add_argument("--run-attempt", default=os.environ.get("GITHUB_RUN_ATTEMPT", "1")) @@ -147,7 +136,7 @@ def main(argv: list[str] | None = None) -> int: markdown = render_header(args.reason, run_gpu, families, base) + render_jobs(rows, missing) errors: list[str] = [] args.results_dir.mkdir(parents=True, exist_ok=True) - markdown += load_summarize_junit(args.summarize_junit).render(args.results_dir, errors) + markdown += summarize_junit.render(args.results_dir, errors) sys.stdout.write(markdown) summary = os.environ.get("GITHUB_STEP_SUMMARY") diff --git a/.github/scripts/tests/hrr_capture_build_test.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py similarity index 96% rename from .github/scripts/tests/hrr_capture_build_test.py rename to projects/hrr/tests/scripts/test_hrr_capture_build.py index da144073ce82..7a691307fed1 100644 --- a/.github/scripts/tests/hrr_capture_build_test.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -1,6 +1,6 @@ -# Copyright Advanced Micro Devices, Inc. +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT - """Tests for hrr_capture_build.py.""" import argparse @@ -11,8 +11,9 @@ import unittest from pathlib import Path -sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) -import hrr_capture_build as cb +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import hrr_capture_build as cb # noqa: E402 FINDER_FOUND = { "status": "found", diff --git a/.github/scripts/tests/hrr_check_sources_test.py b/projects/hrr/tests/scripts/test_hrr_check_sources.py similarity index 96% rename from .github/scripts/tests/hrr_check_sources_test.py rename to projects/hrr/tests/scripts/test_hrr_check_sources.py index 3f8a185479e2..934bcc019917 100644 --- a/.github/scripts/tests/hrr_check_sources_test.py +++ b/projects/hrr/tests/scripts/test_hrr_check_sources.py @@ -1,18 +1,18 @@ -# Copyright Advanced Micro Devices, Inc. +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT - """Tests for hrr_check_sources.py.""" import contextlib import io -import os import sys import tempfile import unittest from pathlib import Path -sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) -import hrr_check_sources as chk +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) + +import hrr_check_sources as chk # noqa: E402 def write(path: Path, text: str) -> None: diff --git a/.github/scripts/tests/hrr_ci_configure_test.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py similarity index 97% rename from .github/scripts/tests/hrr_ci_configure_test.py rename to projects/hrr/tests/scripts/test_hrr_ci_configure.py index 40348f53b41f..7cc422b6bd40 100644 --- a/.github/scripts/tests/hrr_ci_configure_test.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -1,15 +1,15 @@ -# Copyright Advanced Micro Devices, Inc. +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT - """Tests for hrr_ci_configure.py.""" -import os import sys import unittest from pathlib import Path -sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) -import hrr_ci_configure as cfg +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import hrr_ci_configure as cfg # noqa: E402 MATRIX = { "gfx94x": { diff --git a/.github/scripts/tests/hrr_ci_report_test.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py similarity index 80% rename from .github/scripts/tests/hrr_ci_report_test.py rename to projects/hrr/tests/scripts/test_hrr_ci_report.py index ef67b10d9e41..35136dd18e40 100644 --- a/.github/scripts/tests/hrr_ci_report_test.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -1,17 +1,20 @@ -# Copyright Advanced Micro Devices, Inc. +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT - """Tests for hrr_ci_report.py.""" +import contextlib +import io import os import sys +import tempfile import unittest from pathlib import Path +from unittest import mock -sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) -import hrr_ci_report as rpt +sys.path.insert(0, str(Path(__file__).resolve().parent)) -REPO_ROOT = Path(__file__).resolve().parents[3] +import hrr_ci_report as rpt # noqa: E402 JOBS = [ {"name": "Configure", "conclusion": "success", "html_url": "u0"}, @@ -79,11 +82,14 @@ def test_jobs_table_lists_missing(self): self.assertIn("| Integration | gfx950 | [cancelled](u5) |", text) self.assertIn("| Integration | gfx110x | _did not run_ |", text) - def test_summarize_junit_loads(self): - module = rpt.load_summarize_junit( - REPO_ROOT / "projects/hrr/tests/scripts/summarize_junit.py" - ) - self.assertTrue(callable(module.render)) + def test_main_appends_results_table(self): + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(rpt, "list_jobs", return_value=JOBS[:3]), \ + mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ + contextlib.redirect_stdout(io.StringIO()) as out: + rc = rpt.main(["--reason", "why", "--results-dir", tmp]) + self.assertEqual(rc, 0) + self.assertIn("## HRR results by platform", out.getvalue()) if __name__ == "__main__": diff --git a/.github/scripts/hrr_check_sources.py b/projects/hrr/tools/hrr_check_sources.py similarity index 100% rename from .github/scripts/hrr_check_sources.py rename to projects/hrr/tools/hrr_check_sources.py From 4fe51eaf5ec8c530dcc561c7e201e38d85e3516d Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 13 Oct 2026 13:19:27 +0200 Subject: [PATCH 09/89] ci(hrr): authenticate artifact downloads and skip GPU core dumps in tests The test job's base-artifact install ran without a token, so on self-hosted runners that share an IP it hit GitHub's unauthenticated rate limit. It now passes the job token, as the build job does. A regions case faults the GPU on purpose, and ROCr then wrote a gpucore file of about 160 MB into the workspace. HSA_DISABLE_COREDUMP_ON_EXCEPTION turns that off for the suite; the case still passes. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-gpu.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index d34c6fdf5f0e..199b19adf886 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -160,6 +160,8 @@ jobs: path: build - name: Install base ROCm artifacts and the capture-enabled amdhip64 + env: + GITHUB_TOKEN: ${{ github.token }} run: python projects/hrr/tests/scripts/hrr_capture_build.py install-base - name: Run integration suite @@ -168,6 +170,9 @@ jobs: HRR_TEST_IGNORE_CONFIG: ${{ inputs.ignore_test_config && '1' || '' }} # --arch auto finds this machine's GPU with the installed rocminfo. ROCM_SDK: ${{ github.workspace }}/dist + # A regions case faults the GPU on purpose; without this ROCr writes a + # gpucore file of about 160 MB into the workspace. + HSA_DISABLE_COREDUMP_ON_EXCEPTION: "1" run: >- python projects/hrr/tests/scripts/run_catch2.py --xml "integration-${FAMILY}.xml" From a824c56addc758ad0be599703bf972b6f03b58cd Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 10:12:22 +0200 Subject: [PATCH 10/89] ci(hrr): build ROCr from the pull request in the interim GPU build The capture-enabled amdhip64 was built against the ROCr of a develop run, so a pull request that changes ROCr and HRR together was tested against the old ROCr. Build projects/rocr-runtime from the checkout first, the way TheRock does (its sysdeps, lib/, the same RUNPATH), replace the base's ROCr with it and build CLR on top. The GPU jobs install both libraries over the base, and verify and install-base check with ldd that the suite loads exactly one copy of each, from the installed tree. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-gpu.yml | 10 +- .../hrr/tests/scripts/hrr_capture_build.py | 153 ++++++++++++++---- .../tests/scripts/test_hrr_capture_build.py | 125 ++++++++++++-- 3 files changed, 235 insertions(+), 53 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 199b19adf886..30e8e45d860e 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -1,5 +1,6 @@ -# HRR GPU suite: build an amdhip64 with HRR capture compiled in, then run the -# HRR capture/replay integration suite on each requested GPU family. +# HRR GPU suite: build ROCr and an amdhip64 with HRR capture compiled in from +# this checkout, then run the HRR capture/replay integration suite on each +# requested GPU family. # # Called by hrr-ci.yml, which decides when this runs and on which families. # @@ -45,6 +46,7 @@ jobs: projects/clr projects/hip projects/hrr + projects/rocr-runtime - name: Checkout TheRock artifact tools uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -88,7 +90,7 @@ jobs: name: hrr-capture-${{ github.sha }} path: | build/base.json - build/capture-lib/ + build/runtime-lib/ build/hrr/playback/hrr-playback build/hrr/tests/integration/hrr-integration-tests build/hrr/tests/integration/*.so @@ -159,7 +161,7 @@ jobs: name: hrr-capture-${{ github.sha }} path: build - - name: Install base ROCm artifacts and the capture-enabled amdhip64 + - name: Install base ROCm artifacts and the runtime libraries built here env: GITHUB_TOKEN: ${{ github.token }} run: python projects/hrr/tests/scripts/hrr_capture_build.py install-base diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index fcbe997da500..4b9b1885e6dc 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -9,16 +9,19 @@ Release builds compile capture out (HIP_HRR_CAPTURE_ENABLED undefined). This script installs the ROCm artifacts that rocm-systems' own TheRock CI built for -a recent develop commit, rebuilds only CLR from this checkout with capture -enabled, installs it over those artifacts, and builds projects/hrr on top. The -GPU jobs then install the same artifacts plus this amdhip64, so build and test -use the same libraries, and the binaries' RUNPATH needs no LD_LIBRARY_PATH. +a recent develop commit, then builds ROCr and CLR (with capture enabled) from +this checkout over them, and builds projects/hrr on top. The compiler, comgr, +the device libraries, rocprofiler-register and the sysdeps stay prebuilt. The +GPU jobs install the same artifacts plus these two runtime libraries, so build +and test use the same libraries, and the binaries' RUNPATH needs no +LD_LIBRARY_PATH. fetch-base Pick the base run and install the build-time artifacts. - build Build CLR with capture, install it into the base, build HRR. - verify Check the suite's case list and the embedded GPU targets. + build Build ROCr, then CLR with capture, into the base; build HRR. + verify Check the suite's case list, the embedded GPU targets and + which runtime libraries the suite loads. install-base On a GPU machine: install the run-time artifacts of the - recorded base run and the capture-enabled amdhip64. + recorded base run and the runtime libraries built here. """ from __future__ import annotations @@ -41,7 +44,7 @@ # The generic (target-neutral) artifacts are the same in every family group. BASE_ARTIFACT_GROUP = "gfx94X-dcgpu" -# Artifacts needed to build CLR and projects/hrr. +# Artifacts needed to build ROCr, CLR and projects/hrr. BUILD_ARTIFACTS = ( "amd-llvm_run", "amd-llvm_lib", "amd-llvm_dev", "base_run", "base_lib", "base_dev", @@ -59,7 +62,20 @@ "core-hip_lib", ) -CAPTURE_LIB_GLOB = "libamdhip64.so*" +# The libraries built from this checkout that replace the base's copies. +RUNTIME_LIB_GLOBS = ("libamdhip64.so*", "libhsa-runtime64.so*") + +# Everything ROCr installs into the base, relative to --dist, so its build +# replaces the base's ROCr files instead of mixing with them. +ROCR_INSTALLED = ( + "include/hsa", + "include/hsakmt", + "lib/cmake/hsa-runtime64", + "lib/cmake/hsakmt", + "lib/libhsakmt.a", + "lib/pkgconfig/libhsakmt.pc", + "lib/libhsa-runtime64.so*", +) # What the GPU suite runs. The host unit suite is built and run by hrr-ci.yml. HRR_TARGETS = ("hrr-playback", "hrr-integration-tests", "hrr-test-dispatch-tool") @@ -119,6 +135,28 @@ def fetch_command(therock: Path, base: dict, dist: Path, artifacts: Sequence[str ] +def rocr_configure_command(src: Path, build: Path, dist: Path) -> list: + sysdeps = dist / "lib" / "rocm_sysdeps" + cmake_dir = dist / "lib" / "llvm" / "lib" / "cmake" + # The sysdeps provide libelf, libdrm and libnuma, as in TheRock. + return [ + "env", f"PKG_CONFIG_PATH={sysdeps / 'lib' / 'pkgconfig'}", + "cmake", "-S", src / "projects" / "rocr-runtime", "-B", build / "rocr", + "-GNinja", "-Wno-dev", + "-DCMAKE_BUILD_TYPE=Release", + "-DBUILD_SHARED_LIBS=ON", + f"-DCMAKE_PREFIX_PATH={dist};{sysdeps}", + f"-DCMAKE_INSTALL_PREFIX={dist}", + "-DCMAKE_INSTALL_LIBDIR=lib", + # TheRock's libhsa-runtime64 RUNPATH. + "-DCMAKE_INSTALL_RPATH=$ORIGIN;$ORIGIN/rocm_sysdeps/lib", + # The blit and trap-handler kernels need the base's clang; ROCr's own + # hint (${CMAKE_PREFIX_PATH}/llvm) does not work with a list. + f"-DClang_DIR={cmake_dir / 'clang'}", + f"-DLLVM_DIR={cmake_dir / 'llvm'}", + ] + + def clr_configure_command(src: Path, build: Path, dist: Path) -> list: # HIP_HRR_CAPTURE_ENABLED is what compiles capture into amdhip64; CLR has # no CMake option for it. @@ -162,30 +200,70 @@ def hrr_configure_command(src: Path, build: Path, dist: Path, archs: str) -> lis ] -def remove_amdhip64(lib_dir: Path) -> None: - """Remove the release amdhip64 so only the capture-enabled one remains.""" - for path in lib_dir.glob(CAPTURE_LIB_GLOB): - path.unlink() +def remove_paths(root: Path, patterns: Sequence[str]) -> None: + for pattern in patterns: + for path in root.glob(pattern): + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + +def remove_runtime_libs(lib_dir: Path) -> None: + """Remove the base's runtime libraries so only the ones built here remain.""" + remove_paths(lib_dir, RUNTIME_LIB_GLOBS) -def copy_amdhip64(src_dir: Path, dst_dir: Path) -> list[str]: - """Copy libamdhip64.so* from src_dir to dst_dir, keeping symlinks.""" + +def copy_runtime_libs(src_dir: Path, dst_dir: Path) -> list[str]: + """Copy libamdhip64.so* and libhsa-runtime64.so*, keeping symlinks.""" dst_dir.mkdir(parents=True, exist_ok=True) copied = [] - for path in sorted(src_dir.glob(CAPTURE_LIB_GLOB)): - target = dst_dir / path.name - if target.is_symlink() or target.exists(): - target.unlink() - if path.is_symlink(): - os.symlink(os.readlink(path), target) - else: - shutil.copy2(path, target) - copied.append(path.name) - if not copied: - raise SystemExit(f"no {CAPTURE_LIB_GLOB} in {src_dir}") + for pattern in RUNTIME_LIB_GLOBS: + found = sorted(src_dir.glob(pattern)) + if not found: + raise SystemExit(f"no {pattern} in {src_dir}") + for path in found: + target = dst_dir / path.name + if target.is_symlink() or target.exists(): + target.unlink() + if path.is_symlink(): + os.symlink(os.readlink(path), target) + else: + shutil.copy2(path, target) + copied.append(path.name) return copied +def linkage_errors(ldd_output: str, lib_dir: Path) -> list[str]: + """Each runtime library must resolve to exactly one file, in lib_dir.""" + resolved: dict[str, set[str]] = {} + for match in re.finditer(r"^\s*(\S+) => (\S+)", ldd_output, re.MULTILINE): + name, path = match.groups() + for pattern in RUNTIME_LIB_GLOBS: + stem = pattern.rstrip("*") + if name.startswith(stem): + resolved.setdefault(stem, set()).add(path) + errors = [] + for pattern in RUNTIME_LIB_GLOBS: + stem = pattern.rstrip("*") + paths = resolved.get(stem, set()) + if len(paths) != 1: + errors.append(f"{stem} resolves to {sorted(paths) or 'nothing'}") + elif Path(next(iter(paths))).parent != lib_dir: + errors.append(f"{stem} resolves to {next(iter(paths))}, not into {lib_dir}") + return errors + + +def check_linkage(binary: Path, lib_dir: Path) -> int: + out = subprocess.run(["ldd", str(binary)], check=True, capture_output=True, text=True).stdout + errors = linkage_errors(out, lib_dir) + for error in errors: + print(f"::error::{binary.name}: {error}") + if not errors: + print(f"{binary.name} loads the runtime libraries from {lib_dir}") + return 1 if errors else 0 + + def embedded_targets(objdump_output: str) -> set[str]: return set(re.findall(r"\bgfx[0-9a-f]+\b", objdump_output)) @@ -207,11 +285,16 @@ def cmd_fetch_base(args: argparse.Namespace, runner: Runner = run) -> int: def cmd_build(args: argparse.Namespace, runner: Runner = run) -> int: + runner(rocr_configure_command(args.src, args.build, args.dist)) + runner(["cmake", "--build", args.build / "rocr"]) + remove_paths(args.dist, ROCR_INSTALLED) + runner(["cmake", "--install", args.build / "rocr"]) + # CLR finds the ROCr just installed through its hsa-runtime64 package. runner(clr_configure_command(args.src, args.build, args.dist)) runner(["cmake", "--build", args.build / "clr"]) - remove_amdhip64(args.dist / "lib") + remove_paths(args.dist / "lib", ("libamdhip64.so*",)) runner(["cmake", "--install", args.build / "clr"]) - copy_amdhip64(args.dist / "lib", args.capture_lib) + copy_runtime_libs(args.dist / "lib", args.runtime_lib) runner(hrr_configure_command(args.src, args.build, args.dist, args.archs)) runner(["cmake", "--build", args.build / "hrr", "--target", *HRR_TARGETS]) return 0 @@ -233,7 +316,7 @@ def cmd_verify(args: argparse.Namespace) -> int: if missing: print(f"::error::missing from the fat binary: {' '.join(missing)}") return 1 - return 0 + return check_linkage(exe, args.dist / "lib") def restore_executable_bits(build: Path) -> None: @@ -247,10 +330,12 @@ def cmd_install_base(args: argparse.Namespace, runner: Runner = run) -> int: base = json.loads(args.base_json.read_text()) print(f"Base: {base['repo']} run {base['run_id']} at {base['commit']}") runner(fetch_command(args.therock, base, args.dist, RUNTIME_ARTIFACTS)) - remove_amdhip64(args.dist / "lib") - copy_amdhip64(args.capture_lib, args.dist / "lib") + remove_runtime_libs(args.dist / "lib") + copy_runtime_libs(args.runtime_lib, args.dist / "lib") restore_executable_bits(args.build) - return 0 + return check_linkage( + args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests", args.dist / "lib" + ) def parse_args(argv: list[str] | None = None) -> argparse.Namespace: @@ -260,11 +345,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--src", type=Path, default=Path.cwd()) parser.add_argument("--dist", type=Path, default=Path("dist")) parser.add_argument("--build", type=Path, default=Path("build")) - parser.add_argument("--capture-lib", type=Path, default=Path("build/capture-lib")) + parser.add_argument("--runtime-lib", type=Path, default=Path("build/runtime-lib")) parser.add_argument("--base-json", type=Path, default=Path("build/base.json")) parser.add_argument("--archs", default=os.environ.get("HRR_ARCHS", "")) args = parser.parse_args(argv) - for name in ("therock", "src", "dist", "build", "capture_lib", "base_json"): + for name in ("therock", "src", "dist", "build", "runtime_lib", "base_json"): setattr(args, name, getattr(args, name).resolve()) if args.command in ("build", "verify") and not args.archs: parser.error("--archs (or HRR_ARCHS) is required") diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index 7a691307fed1..7c7d1a6ae08e 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -56,6 +56,16 @@ def test_clr_compiles_capture_in(self): self.assertIn("-DCMAKE_INSTALL_PREFIX=/d", cmd) self.assertIn("-DHIP_COMMON_DIR=/s/projects/hip", cmd) + def test_rocr_installs_like_therock(self): + cmd = [str(c) for c in cb.rocr_configure_command(Path("/s"), Path("/b"), Path("/d"))] + self.assertEqual(cmd[:2], ["env", "PKG_CONFIG_PATH=/d/lib/rocm_sysdeps/lib/pkgconfig"]) + self.assertEqual(cmd[cmd.index("-S") + 1], "/s/projects/rocr-runtime") + self.assertIn("-DCMAKE_PREFIX_PATH=/d;/d/lib/rocm_sysdeps", cmd) + self.assertIn("-DCMAKE_INSTALL_PREFIX=/d", cmd) + self.assertIn("-DCMAKE_INSTALL_LIBDIR=lib", cmd) + self.assertIn("-DCMAKE_INSTALL_RPATH=$ORIGIN;$ORIGIN/rocm_sysdeps/lib", cmd) + self.assertIn("-DClang_DIR=/d/lib/llvm/lib/cmake/clang", cmd) + def test_hrr_builds_integration_suite_for_archs(self): cmd = [str(c) for c in cb.hrr_configure_command(Path("/s"), Path("/b"), Path("/d"), "gfx942;gfx950")] self.assertIn("-DGPU_TARGETS=gfx942;gfx950", cmd) @@ -94,32 +104,117 @@ def setUp(self): def tearDown(self): self._tmp.cleanup() - def make_lib(self, directory: Path, real: str, content: str) -> None: + def make_lib(self, directory: Path, stem: str, soname: str, real: str, content: str) -> None: directory.mkdir(parents=True, exist_ok=True) (directory / real).write_text(content) - os.symlink(real, directory / "libamdhip64.so.7") - os.symlink("libamdhip64.so.7", directory / "libamdhip64.so") + os.symlink(real, directory / soname) + os.symlink(soname, directory / stem) + + def make_runtime(self, directory: Path, version: str, content: str) -> None: + self.make_lib(directory, "libamdhip64.so", "libamdhip64.so.7", + f"libamdhip64.so.7.{version}", content) + self.make_lib(directory, "libhsa-runtime64.so", "libhsa-runtime64.so.1", + f"libhsa-runtime64.so.1.{version}", content) - def test_overlay_replaces_release_library(self): - capture = self.tmp / "capture" + def test_overlay_replaces_base_libraries(self): + built = self.tmp / "built" dist = self.tmp / "dist" / "lib" - self.make_lib(capture, "libamdhip64.so.7.2.1", "capture") - self.make_lib(dist, "libamdhip64.so.7.2.0", "release") + self.make_runtime(built, "2.1", "pr") + self.make_runtime(dist, "2.0", "base") (dist / "libhiprtc.so.7").write_text("keep") - cb.remove_amdhip64(dist) - copied = cb.copy_amdhip64(capture, dist) + cb.remove_runtime_libs(dist) + copied = cb.copy_runtime_libs(built, dist) - self.assertEqual(sorted(copied), ["libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.7.2.1"]) + self.assertEqual(len(copied), 6) self.assertFalse((dist / "libamdhip64.so.7.2.0").exists()) - self.assertEqual((dist / "libamdhip64.so").read_text(), "capture") - self.assertTrue((dist / "libamdhip64.so.7").is_symlink()) + self.assertFalse((dist / "libhsa-runtime64.so.1.2.0").exists()) + self.assertEqual((dist / "libamdhip64.so").read_text(), "pr") + self.assertEqual((dist / "libhsa-runtime64.so").read_text(), "pr") + self.assertTrue((dist / "libhsa-runtime64.so.1").is_symlink()) self.assertEqual((dist / "libhiprtc.so.7").read_text(), "keep") - def test_copy_without_library_fails(self): - (self.tmp / "empty").mkdir() + def test_copy_without_rocr_fails(self): + built = self.tmp / "built" + self.make_lib(built, "libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.7.2.1", "pr") with self.assertRaises(SystemExit): - cb.copy_amdhip64(self.tmp / "empty", self.tmp / "out") + cb.copy_runtime_libs(built, self.tmp / "out") + + def test_remove_rocr_install(self): + dist = self.tmp / "dist" + (dist / "include" / "hsa").mkdir(parents=True) + (dist / "include" / "hsa" / "hsa.h").write_text("") + (dist / "include" / "hip").mkdir(parents=True) + self.make_runtime(dist / "lib", "2.0", "base") + cb.remove_paths(dist, cb.ROCR_INSTALLED) + self.assertFalse((dist / "include" / "hsa").exists()) + self.assertTrue((dist / "include" / "hip").exists()) + self.assertFalse(list((dist / "lib").glob("libhsa-runtime64.so*"))) + self.assertTrue((dist / "lib" / "libamdhip64.so.7").exists()) + + +class LinkageTest(unittest.TestCase): + LIB = Path("/w/dist/lib") + + def ldd(self, hsa: str, extra: str = "") -> str: + return ( + "\tlinux-vdso.so.1 (0x00007ffd)\n" + "\tlibamdhip64.so.7 => /w/dist/lib/libamdhip64.so.7 (0x00007f1)\n" + f"\tlibhsa-runtime64.so.1 => {hsa} (0x00007f2)\n" + f"{extra}" + "\tlibc.so.6 => /lib64/libc.so.6 (0x00007f3)\n" + ) + + def test_both_from_dist(self): + self.assertEqual(cb.linkage_errors(self.ldd("/w/dist/lib/libhsa-runtime64.so.1"), self.LIB), []) + + def test_rocr_from_elsewhere(self): + errors = cb.linkage_errors(self.ldd("/opt/rocm/lib/libhsa-runtime64.so.1"), self.LIB) + self.assertEqual(len(errors), 1) + self.assertIn("libhsa-runtime64.so", errors[0]) + + def test_two_rocr_copies(self): + extra = "\tlibhsa-runtime64.so.2 => /opt/rocm/lib/libhsa-runtime64.so.2 (0x0)\n" + errors = cb.linkage_errors(self.ldd("/w/dist/lib/libhsa-runtime64.so.1", extra), self.LIB) + self.assertEqual(len(errors), 1) + + def test_missing_rocr(self): + text = "\tlibamdhip64.so.7 => /w/dist/lib/libamdhip64.so.7 (0x1)\n" + self.assertEqual(len(cb.linkage_errors(text, self.LIB)), 1) + + +class BuildTest(unittest.TestCase): + def test_rocr_is_installed_before_clr_is_configured(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + dist = root / "dist" + (dist / "include" / "hsa").mkdir(parents=True) + (dist / "lib").mkdir() + calls = [] + + def runner(cmd): + cmd = [str(c) for c in cmd] + calls.append(cmd) + # The base's ROCr headers are gone before ROCr installs. + if cmd[:2] == ["cmake", "--install"] and cmd[2].endswith("rocr"): + self.assertFalse((dist / "include" / "hsa").exists()) + if cmd[:2] == ["cmake", "--install"] and cmd[2].endswith("clr"): + for name in ("libamdhip64.so.7", "libhsa-runtime64.so.1"): + (dist / "lib" / name).write_text("") + + args = argparse.Namespace( + src=root / "src", build=root / "build", dist=dist, + runtime_lib=root / "build" / "runtime-lib", archs="gfx942", + ) + cb.cmd_build(args, runner=runner) + steps = [" ".join(c[c.index("cmake"):c.index("cmake") + 3]) for c in calls] + rocr_install = steps.index(f"cmake --install {root}/build/rocr") + clr_configure = next(i for i, c in enumerate(calls) if "projects/clr" in " ".join(c)) + self.assertLess(rocr_install, clr_configure) + self.assertEqual( + sorted(p.name for p in args.runtime_lib.iterdir()), + ["libamdhip64.so.7", "libhsa-runtime64.so.1"], + ) class ExecutableBitsTest(unittest.TestCase): From 9b9da2ea66865f684cc2ecc994234582493e1b20 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 10:14:40 +0200 Subject: [PATCH 11/89] ci(hrr): move the CI policy into projects/hrr/tests/config/ci.yaml The paths, the capture label, the GPU families and the unit job's ROCm SDK pin were constants in hrr_ci_configure.py and hrr-ci.yml, so a change to what CI runs needed a script change. ci.yaml now holds them and the configure job validates it strictly: an unknown key, trigger or family fails the run instead of being ignored. Every family with a runner now runs on every event by default, rather than following TheRock's presubmit/postsubmit/nightly sets. Per family, ci.yaml can disable it, narrow its events, override its runner or take the multi-GPU runner, so GPU load is tuned there. ci:gfx... labels and the workflow_dispatch families input still add or replace families. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 21 +- projects/hrr/tests/README.md | 25 +- projects/hrr/tests/config/ci.yaml | 84 +++++ .../hrr/tests/scripts/hrr_ci_configure.py | 303 +++++++++++------- .../tests/scripts/test_hrr_ci_configure.py | 197 +++++++++--- 5 files changed, 463 insertions(+), 167 deletions(-) create mode 100644 projects/hrr/tests/config/ci.yaml diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 70503463b576..97c1f0fe702b 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -1,8 +1,8 @@ # HRR CI: generated-source checks, the HRR helper-script tests and the host # unit suite, plus the GPU capture/replay suite (hrr-gpu.yml) for changes that # touch capture or replay. projects/hrr/tests/scripts/hrr_ci_configure.py -# decides what runs; "HRR CI Summary" reports every platform and GPU family in -# one table. +# decides what runs from projects/hrr/tests/config/ci.yaml; "HRR CI Summary" +# reports every platform and GPU family in one table. name: HRR CI @@ -17,7 +17,7 @@ on: workflow_dispatch: inputs: families: - description: "Comma-separated GPU families (TheRock keys, e.g. gfx94x,gfx110x). Empty: the nightly set." + description: "Comma-separated GPU families (TheRock keys, e.g. gfx94x,gfx110x). Empty: the families ci.yaml selects." type: string default: "" ignore_test_config: @@ -33,10 +33,9 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: - # The unit suite only needs HIP headers and the hip CMake package. Pinned so - # a run does not depend on the day's nightly. + # The unit suite only needs HIP headers and the hip CMake package; ci.yaml + # pins the version. ROCM_INDEX_URL: "https://rocm.nightlies.amd.com/whl-multi-arch/" - ROCM_SDK_VERSION: "10.1.0a20260822" jobs: configure: @@ -50,12 +49,15 @@ jobs: run_gpu: ${{ steps.decide.outputs.run_gpu }} families: ${{ steps.decide.outputs.families }} reason: ${{ steps.decide.outputs.reason }} + rocm_sdk_version: ${{ steps.decide.outputs.rocm_sdk_version }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - sparse-checkout: projects/hrr/tests/scripts/hrr_ci_configure.py + sparse-checkout: | + projects/hrr/tests/scripts/hrr_ci_configure.py + projects/hrr/tests/config/ci.yaml sparse-checkout-cone-mode: false - name: Checkout TheRock family matrix @@ -82,6 +84,9 @@ jobs: with: python-version: "3.12" + - name: Install PyYAML + run: python -m pip install --no-cache-dir PyYAML==6.0.2 + - name: Decide id: decide env: @@ -201,6 +206,8 @@ jobs: python-version: "3.12" - name: Install the ROCm SDK + env: + ROCM_SDK_VERSION: ${{ needs.configure.outputs.rocm_sdk_version }} run: | python -m pip install --no-cache-dir PyYAML==6.0.2 python -m pip install --no-cache-dir --index-url "${ROCM_INDEX_URL}" "rocm[devel]==${ROCM_SDK_VERSION}" diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index d5e05d46ded2..661b2a12b1dd 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -10,7 +10,8 @@ are the same ones it uses. tests/ unit/ C++ (Catch2). hrr-unit-tests: CPU only, no GPU, no capture runtime. integration/ C++ (Catch2). hrr-integration-tests: real capture, then replay, on a GPU. - config/ One YAML file per suite: every case, and where it runs. + config/ One YAML file per suite: every case, and where it runs. ci.yaml: + which CI jobs run and on which GPU families. include/ Headers both suites include (hrr_test_common.hh, hrr_test_process.hh). scripts/ CI helpers that run a suite and report on it, plus the unittest suites for those helpers and for ../tools. Not built. @@ -131,6 +132,28 @@ plus the tools in `../tools`: the HRR API ID compatibility check, the generated-source checks (`hrr_check_sources.py`), the playback-handler default in the generator, and the [api-matrix](../tools/api-matrix/README.md) scripts. +## Choosing what CI runs + +[`config/ci.yaml`](config/ci.yaml) is the policy the configure job of `hrr-ci.yml` +applies (`scripts/hrr_ci_configure.py`): + +- `paths.cpu`: a change under one of these runs the generated-source, script and unit + jobs. `paths.capture` also runs the capture-enabled build and the GPU suite. + `paths.excluded` counts for neither. +- `capture_label` (`ci:hrr-capture`) runs the GPU suite on a pull request that touches + no capture path. +- `families`: the GPU families the suite runs on, as TheRock family keys, on the runner + TheRock's family matrix names. By default each runs on every event (`pull_request`, + `push`, `schedule`, `workflow_dispatch`). Per family, `enabled: false` leaves it to + explicit requests, `triggers` narrows the events, `runner` overrides the runner label + and `multi_gpu: true` takes TheRock's multi-GPU runner. A `ci:gfx...` label adds a + family; the `workflow_dispatch` input `families` replaces the selection. Both can + name a disabled family. +- `rocm_sdk_version`: the ROCm SDK the unit job installs. + +An unknown key, trigger or family fails the configure job. To take load off the GPU +pools, disable families or narrow their triggers here. + ## Configuring where a case runs The build is one fat binary for every architecture in `HRR_ARCHS`. Nothing about a diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml new file mode 100644 index 000000000000..8d325d526b30 --- /dev/null +++ b/projects/hrr/tests/config/ci.yaml @@ -0,0 +1,84 @@ +# What HRR CI runs and where, read by +# projects/hrr/tests/scripts/hrr_ci_configure.py (hrr-ci.yml's configure job). +# Unknown keys, triggers or families are errors. +version: 1 + +# The ROCm SDK the host unit job installs. Pinned so a run does not depend on +# the day's nightly. +rocm_sdk_version: "10.1.0a20260822" + +# Path prefixes, relative to the repository root. A path ending in "/" matches +# everything under it; any other path matches only itself. +paths: + # A change under any of these runs the generated-source, script and unit jobs. + cpu: + - projects/hrr/ + - projects/clr/hipamd/src/hrr/ + - projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp + - projects/hip/include/hip/hip_runtime_api.h + - .github/workflows/hrr-ci.yml + - .github/workflows/hrr-gpu.yml + # A change under any of these also runs the capture-enabled build and the GPU + # suite. hip_runtime_api.h is deliberately absent: almost every HIP API change + # touches it, and the generated-source job already checks HRR against it. + capture: + - projects/clr/hipamd/src/hrr/ + - projects/clr/hipamd/src/hip_context.cpp + - projects/clr/hipamd/src/hip_code_object.cpp + - projects/clr/hipamd/src/hip_code_object.hpp + - projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp + - projects/hrr/CMakeLists.txt + - projects/hrr/include/ + - projects/hrr/playback/ + - projects/hrr/tests/CMakeLists.txt + - projects/hrr/tests/include/ + - projects/hrr/tests/integration/ + - projects/hrr/tests/config/ci.yaml + - projects/hrr/tests/config/integration.yaml + - projects/hrr/tests/scripts/run_catch2.py + - projects/hrr/tests/scripts/hrr_test_config.py + - projects/hrr/tests/scripts/summarize_junit.py + - projects/hrr/tests/scripts/hrr_capture_build.py + - projects/hrr/tests/scripts/hrr_ci_configure.py + - projects/hrr/tests/scripts/hrr_ci_report.py + - .github/workflows/hrr-ci.yml + - .github/workflows/hrr-gpu.yml + # Ignored by both lists; hrr-skills.yml covers these. + excluded: + - projects/hrr/skills/ + - projects/hrr/README.md + +# A pull request with this label runs the GPU suite even when it touches no +# capture path. +capture_label: ci:hrr-capture + +# The GPU families the suite runs on, as TheRock family keys +# (build_tools/github_actions/amdgpu_family_matrix.py). The integration binary +# is built for targets.arch in integration.yaml, which must cover them. +# +# Per family, all optional: +# enabled false: run only when requested by a ci:gfx... label or the +# workflow_dispatch families input. Default true. +# triggers Events that run this family: pull_request, push, schedule, +# workflow_dispatch. Default all four. +# runner Runner label to use instead of TheRock's. +# multi_gpu Use TheRock's multi-GPU runner; the T3 tier needs two devices. +# Default false. +# +# A family without a runner label is skipped. Every job queues on its own +# runner pool and the summary waits for all of them, so to reduce GPU load or +# take a pool out, set enabled: false or narrow triggers here. +families: + gfx90a: {} + gfx94x: {} + gfx950: + multi_gpu: true + gfx103x: {} + gfx110x: {} + gfx1150: {} + gfx1151: {} + gfx1153: {} + gfx120x: {} + # This pool is not reliably online; a job that finds no runner stays queued + # and holds up the summary until GitHub times it out. + gfx125x: {} diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index 4ac2f7dcf1b7..c39446cb881d 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -5,20 +5,22 @@ Writes these GitHub Actions outputs: - run_cpu "true" when the generated-source, script and unit jobs run. - run_gpu "true" when the capture-enabled build and the GPU suite run. - families JSON list of {"family", "runner", "gfx_targets"} for the GPU suite. - reason One line explaining the decision, shown in the summary. - -The GPU suite needs an amdhip64 with HRR capture compiled in, which only the -HRR GPU workflow builds. It runs automatically when a change touches capture -or replay (CAPTURE_PATHS), on any pull request labelled ci:hrr-capture, on -pushes to develop that touch capture, nightly, and on workflow_dispatch. - -GPU families follow TheRock's trigger policy (amdgpu_family_matrix.py): -pull requests test the presubmit families, pushes the postsubmit families and -the nightly schedule the nightly families, each limited to HRR_FAMILIES. A -ci:gfx... label adds that family, the same label TheRock CI uses. + run_cpu "true" when the generated-source, script and unit jobs run. + run_gpu "true" when the capture-enabled build and the GPU suite run. + families JSON list of {"family", "runner", "gfx_targets"} for the + GPU suite. + reason One line explaining the decision, shown in the summary. + rocm_sdk_version The ROCm SDK the unit job installs. + +The policy lives in projects/hrr/tests/config/ci.yaml. The GPU suite needs an +amdhip64 with HRR capture compiled in, which only the HRR GPU workflow builds. +It runs when a change touches a capture path, on any pull request with the +capture label, nightly and on workflow_dispatch. + +It runs on every enabled family whose triggers include the event. A ci:gfx... +label adds that family, the same label TheRock CI uses; the workflow_dispatch +families input replaces the selection. Both can name a disabled family. The +runner labels come from TheRock's family matrix unless ci.yaml overrides them. """ from __future__ import annotations @@ -32,81 +34,39 @@ from pathlib import Path from typing import Callable, Iterable -CAPTURE_LABEL = "ci:hrr-capture" - -# A change under any of these runs the generated-source, script and unit jobs. -CPU_PATHS = ( - "projects/hrr/", - "projects/clr/hipamd/src/hrr/", - "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp", - "projects/hip/include/hip/hip_runtime_api.h", - ".github/workflows/hrr-ci.yml", - ".github/workflows/hrr-gpu.yml", -) - -# hrr-skills.yml covers these. -EXCLUDED_PATHS = ("projects/hrr/skills/", "projects/hrr/README.md") - -# A change under any of these also runs the capture-enabled build and the GPU -# suite. hip_runtime_api.h is deliberately absent: almost every HIP API change -# touches it, and the generated-source job already checks HRR against it. -CAPTURE_PATHS = ( - "projects/clr/hipamd/src/hrr/", - "projects/clr/hipamd/src/hip_context.cpp", - "projects/clr/hipamd/src/hip_code_object.cpp", - "projects/clr/hipamd/src/hip_code_object.hpp", - "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp", - "projects/hrr/CMakeLists.txt", - "projects/hrr/include/", - "projects/hrr/playback/", - "projects/hrr/tests/CMakeLists.txt", - "projects/hrr/tests/include/", - "projects/hrr/tests/integration/", - "projects/hrr/tests/config/integration.yaml", - "projects/hrr/tests/scripts/run_catch2.py", - "projects/hrr/tests/scripts/hrr_test_config.py", - "projects/hrr/tests/scripts/summarize_junit.py", - "projects/hrr/tests/scripts/hrr_capture_build.py", - "projects/hrr/tests/scripts/hrr_ci_configure.py", - "projects/hrr/tests/scripts/hrr_ci_report.py", - ".github/workflows/hrr-ci.yml", - ".github/workflows/hrr-gpu.yml", -) - -# TheRock family keys the HRR fat binary covers (HRR_ARCHS in hrr-gpu.yml). -# gfx125x is built but only tested on request: its runner pool is not reliably -# online. -HRR_FAMILIES = ( - "gfx90a", - "gfx94x", - "gfx950", - "gfx103x", - "gfx110x", - "gfx1150", - "gfx1151", - "gfx1153", - "gfx120x", - "gfx125x", -) -ON_REQUEST_FAMILIES = ("gfx125x",) -# TheRock tests gfx950 only for submodule bumps; HRR also tests it nightly. -NIGHTLY_EXTRA_FAMILIES = ("gfx950",) -# The T3 tier needs two visible devices, so these families take TheRock's -# multi-GPU runner when it has one. -MULTI_GPU_FAMILIES = ("gfx950",) - -TRIGGER_FOR_EVENT = { - "pull_request": "presubmit", - "push": "postsubmit", - "schedule": "nightly", - "workflow_dispatch": "nightly", -} +import yaml + +DEFAULT_CONFIG = Path(__file__).resolve().parent.parent / "config" / "ci.yaml" + +EVENTS = ("pull_request", "push", "schedule", "workflow_dispatch") # The pull request files API returns at most this many files. A larger change # is treated as touching everything rather than guessed at. MAX_LISTED_FILES = 3000 +class ConfigError(ValueError): + pass + + +@dataclass(frozen=True) +class Family: + enabled: bool = True + triggers: tuple[str, ...] = EVENTS + runner: str = "" + multi_gpu: bool = False + + +@dataclass(frozen=True) +class Config: + rocm_sdk_version: str + cpu_paths: tuple[str, ...] + capture_paths: tuple[str, ...] + excluded_paths: tuple[str, ...] + capture_label: str + families: dict[str, Family] + + @dataclass class Decision: run_cpu: bool @@ -115,20 +75,105 @@ class Decision: families: list[dict] = field(default_factory=list) +def _mapping(where: str, value, required: Iterable[str], optional: Iterable[str] = ()) -> dict: + if not isinstance(value, dict): + raise ConfigError(f"{where}: expected a mapping") + unknown = sorted(set(value) - set(required) - set(optional)) + if unknown: + raise ConfigError(f"{where}: unknown key {', '.join(map(str, unknown))}") + missing = sorted(set(required) - set(value)) + if missing: + raise ConfigError(f"{where}: missing {', '.join(missing)}") + return value + + +def _string(where: str, value) -> str: + if not isinstance(value, str) or not value: + raise ConfigError(f"{where}: expected a non-empty string") + return value + + +def _strings(where: str, value) -> tuple[str, ...]: + if not isinstance(value, list) or not value: + raise ConfigError(f"{where}: expected a non-empty list") + return tuple(_string(f"{where}[{i}]", v) for i, v in enumerate(value)) + + +def _bool(where: str, value) -> bool: + if not isinstance(value, bool): + raise ConfigError(f"{where}: expected true or false") + return value + + +def _family(where: str, value) -> Family: + value = _mapping(where, value or {}, (), ("enabled", "triggers", "runner", "multi_gpu")) + triggers = _strings(f"{where}.triggers", value.get("triggers", list(EVENTS))) + for trigger in triggers: + if trigger not in EVENTS: + raise ConfigError( + f"{where}.triggers: unknown trigger {trigger!r} (expected {', '.join(EVENTS)})" + ) + return Family( + enabled=_bool(f"{where}.enabled", value.get("enabled", True)), + triggers=triggers, + runner=_string(f"{where}.runner", value["runner"]) if "runner" in value else "", + multi_gpu=_bool(f"{where}.multi_gpu", value.get("multi_gpu", False)), + ) + + +def parse_config(data) -> Config: + data = _mapping( + "top level", data, ("version", "rocm_sdk_version", "paths", "capture_label", "families") + ) + if type(data["version"]) is not int or data["version"] != 1: + raise ConfigError(f"version: expected 1, got {data['version']!r}") + paths = _mapping("paths", data["paths"], ("cpu", "capture", "excluded")) + families = data["families"] + if not isinstance(families, dict) or not families: + raise ConfigError("families: expected a mapping with at least one family") + return Config( + rocm_sdk_version=_string("rocm_sdk_version", data["rocm_sdk_version"]), + cpu_paths=_strings("paths.cpu", paths["cpu"]), + capture_paths=_strings("paths.capture", paths["capture"]), + excluded_paths=_strings("paths.excluded", paths["excluded"]), + capture_label=_string("capture_label", data["capture_label"]), + families={ + _string("families", key): _family(f"families.{key}", value) + for key, value in families.items() + }, + ) + + +def load_config(path: Path) -> Config: + try: + return parse_config(yaml.safe_load(path.read_text(encoding="utf-8"))) + except ConfigError as e: + raise ConfigError(f"{path}: {e}") from None + + +def check_families(config: Config, matrix: dict) -> None: + unknown = [key for key in config.families if key not in matrix] + if unknown: + raise ConfigError( + f"families: {', '.join(unknown)} not in TheRock's family matrix" + f" (known: {', '.join(sorted(matrix))})" + ) + + def _matches(path: str, prefixes: Iterable[str]) -> bool: return any(path == p or path.startswith(p) for p in prefixes) -def relevant_paths(paths: Iterable[str]) -> list[str]: - return [p for p in paths if not _matches(p, EXCLUDED_PATHS)] +def relevant_paths(paths: Iterable[str], config: Config) -> list[str]: + return [p for p in paths if not _matches(p, config.excluded_paths)] -def touches_cpu(paths: Iterable[str]) -> bool: - return any(_matches(p, CPU_PATHS) for p in relevant_paths(paths)) +def touches_cpu(paths: Iterable[str], config: Config) -> bool: + return any(_matches(p, config.cpu_paths) for p in relevant_paths(paths, config)) -def touches_capture(paths: Iterable[str]) -> bool: - return any(_matches(p, CAPTURE_PATHS) for p in relevant_paths(paths)) +def touches_capture(paths: Iterable[str], config: Config) -> bool: + return any(_matches(p, config.capture_paths) for p in relevant_paths(paths, config)) def labelled_families(labels: Iterable[str], matrix: dict) -> list[str]: @@ -162,33 +207,37 @@ def runner_for( def select_families( - trigger: str, + event: str, + config: Config, requested: list[str], labels: list[str], matrix: dict, pick_weighted: Callable[[list, str], str], -) -> list[dict]: - """GPU families to test, in HRR_FAMILIES order, skipping any without a runner.""" +) -> tuple[list[dict], list[str]]: + """GPU families to test, in ci.yaml order, and the chosen ones without a runner.""" if requested: + unknown = [key for key in requested if key not in config.families] + if unknown: + raise ConfigError( + f"requested families {', '.join(unknown)} are not in ci.yaml" + f" (known: {', '.join(config.families)})" + ) chosen = set(requested) else: chosen = { - key - for key in HRR_FAMILIES - if key not in ON_REQUEST_FAMILIES - and trigger in matrix.get(key, {}).get("linux", {}).get("tests_on_trigger", []) + key for key, family in config.families.items() + if family.enabled and event in family.triggers } - if trigger == "nightly": - chosen |= set(NIGHTLY_EXTRA_FAMILIES) - chosen |= set(labelled_families(labels, matrix)) + chosen |= set(labelled_families(labels, matrix)) & set(config.families) - families = [] - for key in HRR_FAMILIES: + families, no_runner = [], [] + for key, family in config.families.items(): if key not in chosen: continue linux = matrix.get(key, {}).get("linux", {}) - runner = runner_for(linux, pick_weighted, key in MULTI_GPU_FAMILIES) + runner = family.runner or runner_for(linux, pick_weighted, family.multi_gpu) if not runner: + no_runner.append(key) continue families.append( { @@ -197,7 +246,7 @@ def select_families( "gfx_targets": ",".join(linux.get("fetch-gfx-targets", [])), } ) - return families + return families, no_runner def decide( @@ -206,12 +255,12 @@ def decide( labels: list[str], is_fork: bool, requested: list[str], + config: Config, matrix: dict, pick_weighted: Callable[[list, str], str], ) -> Decision: """paths is None when the change set could not be listed in full.""" - trigger = TRIGGER_FOR_EVENT.get(event) - if trigger is None: + if event not in EVENTS: return Decision(False, False, f"event {event} does not run HRR CI") if event in ("schedule", "workflow_dispatch"): @@ -219,22 +268,24 @@ def decide( elif paths is None: run_cpu, capture, why = True, True, "change set too large to list" else: - run_cpu = touches_cpu(paths) - capture = touches_capture(paths) + run_cpu = touches_cpu(paths, config) + capture = touches_capture(paths, config) why = "the change touches capture or replay" if capture else ( "the change touches HRR" if run_cpu else "the change does not touch HRR" ) - labelled = event == "pull_request" and CAPTURE_LABEL in labels + labelled = event == "pull_request" and config.capture_label in labels if labelled and not capture: - run_cpu, capture, why = True, True, f"the pull request is labelled {CAPTURE_LABEL}" + run_cpu, capture, why = True, True, f"the pull request is labelled {config.capture_label}" if not capture: return Decision(run_cpu, False, why) if is_fork: return Decision(run_cpu, False, why + "; GPU suite skipped for a fork pull request") - families = select_families(trigger, requested, labels, matrix, pick_weighted) + families, no_runner = select_families(event, config, requested, labels, matrix, pick_weighted) + if no_runner: + why += f"; no runner for {', '.join(no_runner)}" if not families: return Decision(run_cpu, False, why + "; no GPU family with a runner was selected") return Decision(run_cpu, True, why, families) @@ -275,12 +326,13 @@ def load_matrix(therock_dir: Path): return families, amdgpu_family_matrix.select_weighted_label -def write_outputs(decision: Decision, output_path: str | None) -> None: +def write_outputs(decision: Decision, rocm_sdk_version: str, output_path: str | None) -> None: lines = [ f"run_cpu={'true' if decision.run_cpu else 'false'}", f"run_gpu={'true' if decision.run_gpu else 'false'}", f"families={json.dumps(decision.families, separators=(',', ':'))}", f"reason={decision.reason}", + f"rocm_sdk_version={rocm_sdk_version}", ] text = "\n".join(lines) + "\n" if output_path: @@ -302,6 +354,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--families", default="", help="Comma-separated family keys (workflow_dispatch)" ) parser.add_argument("--therock-dir", type=Path, required=True) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) return parser.parse_args(argv) @@ -310,17 +363,25 @@ def main(argv: list[str] | None = None) -> int: labels = json.loads(args.labels or "[]") requested = [f.strip().lower() for f in args.families.split(",") if f.strip()] - paths: list[str] | None = [] - if args.event == "pull_request" and args.pr_number: - paths = list_pr_files(args.repo, args.pr_number) - elif args.event == "push": - paths = list_push_files(args.repo, args.before, args.after) + try: + config = load_config(args.config) + matrix, pick_weighted = load_matrix(args.therock_dir) + check_families(config, matrix) - matrix, pick_weighted = load_matrix(args.therock_dir) - decision = decide( - args.event, paths, labels, args.is_fork == "true", requested, matrix, pick_weighted - ) - write_outputs(decision, os.environ.get("GITHUB_OUTPUT")) + paths: list[str] | None = [] + if args.event == "pull_request" and args.pr_number: + paths = list_pr_files(args.repo, args.pr_number) + elif args.event == "push": + paths = list_push_files(args.repo, args.before, args.after) + + decision = decide( + args.event, paths, labels, args.is_fork == "true", requested, + config, matrix, pick_weighted, + ) + except ConfigError as e: + print(f"::error::{e}") + return 1 + write_outputs(decision, config.rocm_sdk_version, os.environ.get("GITHUB_OUTPUT")) return 0 diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 7cc422b6bd40..5fb78a3fc902 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -3,10 +3,16 @@ # SPDX-License-Identifier: MIT """Tests for hrr_ci_configure.py.""" +import copy +import io import sys +import tempfile import unittest +from contextlib import redirect_stdout from pathlib import Path +import yaml + sys.path.insert(0, str(Path(__file__).resolve().parent)) import hrr_ci_configure as cfg # noqa: E402 @@ -18,7 +24,6 @@ "test-runs-on": "gpu-94x", "test-runs-on-labels": [{"label": "gpu-94x-a", "count": 1}], "fetch-gfx-targets": ["gfx942"], - "tests_on_trigger": ["presubmit", "postsubmit", "nightly"], } }, "gfx90a": { @@ -26,7 +31,13 @@ "family": "gfx90a", "test-runs-on": "gpu-90a", "fetch-gfx-targets": ["gfx90a"], - "tests_on_trigger": ["postsubmit", "nightly"], + } + }, + "gfx103x": { + "linux": { + "family": "gfx103X-all", + "test-runs-on": "gpu-103x", + "fetch-gfx-targets": ["gfx1030"], } }, "gfx110x": { @@ -34,7 +45,6 @@ "family": "gfx110X-all", "test-runs-on": "gpu-110x", "fetch-gfx-targets": ["gfx1100", "gfx1101"], - "tests_on_trigger": ["nightly"], } }, "gfx950": { @@ -43,14 +53,12 @@ "test-runs-on": "gpu-950-1", "test-runs-on-multi-gpu": "gpu-950-8", "fetch-gfx-targets": ["gfx950"], - "tests_on_trigger": ["submodule_bump"], } }, "gfx1150": { "linux": { "family": "gfx1150", "test-runs-on": "", - "tests_on_trigger": ["nightly"], } }, "gfx125x": { @@ -58,19 +66,42 @@ "family": "gfx125X-dcgpu", "test-runs-on": "gpu-125x", "fetch-gfx-targets": ["gfx1250"], - "tests_on_trigger": ["nightly"], } }, } +REPO_CONFIG = yaml.safe_load(cfg.DEFAULT_CONFIG.read_text(encoding="utf-8")) + +# The repository's paths with families that exercise every option. gfx103x is +# in MATRIX but not here. +FAMILIES = { + "gfx90a": {}, + "gfx94x": None, + "gfx950": {"multi_gpu": True}, + "gfx110x": {"triggers": ["schedule", "workflow_dispatch"]}, + "gfx1150": {}, + "gfx125x": {"enabled": False}, +} + + +def config_data(**changes): + data = copy.deepcopy(REPO_CONFIG) + data["families"] = copy.deepcopy(FAMILIES) + data.update(changes) + return data + + +CONFIG = cfg.parse_config(config_data()) +REPO = cfg.load_config(cfg.DEFAULT_CONFIG) + def pick_first(labels, _context): return labels[0]["label"] -def decide(event, paths, labels=(), is_fork=False, requested=()): +def decide(event, paths, labels=(), is_fork=False, requested=(), config=CONFIG): return cfg.decide( - event, paths, list(labels), is_fork, list(requested), MATRIX, pick_first + event, paths, list(labels), is_fork, list(requested), config, MATRIX, pick_first ) @@ -78,32 +109,112 @@ def family_keys(decision): return [f["family"] for f in decision.families] +class RepoConfigTest(unittest.TestCase): + def test_every_family_runs_on_every_event_by_default(self): + for key, family in REPO.families.items(): + self.assertTrue(family.enabled, key) + self.assertEqual(family.triggers, cfg.EVENTS, key) + + def test_gfx950_takes_the_multi_gpu_runner(self): + self.assertTrue(REPO.families["gfx950"].multi_gpu) + + class PathClassificationTest(unittest.TestCase): def test_playback_change_is_capture(self): - self.assertTrue(cfg.touches_capture(["projects/hrr/playback/hip_playback.cpp"])) + self.assertTrue(cfg.touches_capture(["projects/hrr/playback/hip_playback.cpp"], REPO)) def test_capture_source_is_capture(self): - self.assertTrue(cfg.touches_capture(["projects/clr/hipamd/src/hrr/hip_capture.cpp"])) + self.assertTrue( + cfg.touches_capture(["projects/clr/hipamd/src/hrr/hip_capture.cpp"], REPO) + ) + + def test_ci_config_is_capture(self): + self.assertTrue(cfg.touches_capture(["projects/hrr/tests/config/ci.yaml"], REPO)) def test_public_header_is_cpu_only(self): paths = ["projects/hip/include/hip/hip_runtime_api.h"] - self.assertTrue(cfg.touches_cpu(paths)) - self.assertFalse(cfg.touches_capture(paths)) + self.assertTrue(cfg.touches_cpu(paths, REPO)) + self.assertFalse(cfg.touches_capture(paths, REPO)) def test_unit_test_is_cpu_only(self): paths = ["projects/hrr/tests/unit/hrr_format_test.cc"] - self.assertTrue(cfg.touches_cpu(paths)) - self.assertFalse(cfg.touches_capture(paths)) + self.assertTrue(cfg.touches_cpu(paths, REPO)) + self.assertFalse(cfg.touches_capture(paths, REPO)) def test_skills_are_excluded(self): paths = ["projects/hrr/skills/hrr/SKILL.md", "projects/hrr/README.md"] - self.assertFalse(cfg.touches_cpu(paths)) + self.assertFalse(cfg.touches_cpu(paths, REPO)) def test_unrelated_change(self): - self.assertFalse(cfg.touches_cpu(["projects/rocminfo/rocminfo.cc"])) + self.assertFalse(cfg.touches_cpu(["projects/rocminfo/rocminfo.cc"], REPO)) def test_prefix_must_match_whole_component(self): - self.assertFalse(cfg.touches_cpu(["projects/hrrx/foo.cc"])) + self.assertFalse(cfg.touches_cpu(["projects/hrrx/foo.cc"], REPO)) + + +class ConfigValidationTest(unittest.TestCase): + def assertRejected(self, data, message): + with self.assertRaises(cfg.ConfigError) as caught: + cfg.parse_config(data) + self.assertIn(message, str(caught.exception)) + + def test_unknown_top_level_key(self): + self.assertRejected(config_data(gpu_families=[]), "unknown key gpu_families") + + def test_missing_key(self): + data = config_data() + del data["capture_label"] + self.assertRejected(data, "missing capture_label") + + def test_wrong_version(self): + self.assertRejected(config_data(version=2), "version: expected 1") + + def test_unknown_family_option(self): + families = {"gfx94x": {"runs_on": "x"}} + self.assertRejected(config_data(families=families), "families.gfx94x: unknown key runs_on") + + def test_unknown_trigger(self): + families = {"gfx94x": {"triggers": ["nightly"]}} + self.assertRejected(config_data(families=families), "unknown trigger 'nightly'") + + def test_enabled_must_be_a_bool(self): + families = {"gfx94x": {"enabled": "no"}} + self.assertRejected(config_data(families=families), "families.gfx94x.enabled") + + def test_empty_path_list(self): + data = config_data() + data["paths"]["capture"] = [] + self.assertRejected(data, "paths.capture: expected a non-empty list") + + def test_no_families(self): + self.assertRejected(config_data(families={}), "families: expected a mapping") + + def test_family_missing_from_matrix(self): + config = cfg.parse_config(config_data(families={"gfx94x": {}, "gfx999": {}})) + with self.assertRaises(cfg.ConfigError) as caught: + cfg.check_families(config, MATRIX) + self.assertIn("gfx999 not in TheRock's family matrix", str(caught.exception)) + + def test_load_error_names_the_file(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "ci.yaml" + path.write_text("version: 1\n") + with self.assertRaises(cfg.ConfigError) as caught: + cfg.load_config(path) + self.assertIn(str(path), str(caught.exception)) + + def test_main_reports_a_bad_config(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "ci.yaml" + path.write_text(yaml.safe_dump(config_data(version=2))) + out = io.StringIO() + with redirect_stdout(out): + status = cfg.main( + ["--event", "schedule", "--repo", "r", "--therock-dir", tmp, + "--config", str(path)] + ) + self.assertEqual(status, 1) + self.assertIn("::error::", out.getvalue()) class DecideTest(unittest.TestCase): @@ -117,12 +228,15 @@ def test_cpu_only_pull_request(self): self.assertTrue(d.run_cpu) self.assertFalse(d.run_gpu) - def test_capture_pull_request_uses_presubmit_families(self): + def test_capture_pull_request_runs_enabled_families(self): d = decide("pull_request", ["projects/clr/hipamd/src/hrr/hip_capture.cpp"]) self.assertTrue(d.run_gpu) - self.assertEqual(family_keys(d), ["gfx94x"]) - self.assertEqual(d.families[0]["runner"], "gpu-94x-a") - self.assertEqual(d.families[0]["gfx_targets"], "gfx942") + # gfx110x is not triggered by pull requests, gfx125x is disabled and + # gfx1150 has no runner. + self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950"]) + self.assertEqual(d.families[1]["runner"], "gpu-94x-a") + self.assertEqual(d.families[1]["gfx_targets"], "gfx942") + self.assertIn("no runner for gfx1150", d.reason) def test_rename_out_of_capture_counts(self): # The files API lists previous_filename alongside filename. @@ -130,7 +244,7 @@ def test_rename_out_of_capture_counts(self): self.assertTrue(d.run_gpu) def test_label_requests_gpu_on_unrelated_pull_request(self): - d = decide("pull_request", ["projects/rocminfo/x.cc"], labels=[cfg.CAPTURE_LABEL]) + d = decide("pull_request", ["projects/rocminfo/x.cc"], labels=["ci:hrr-capture"]) self.assertTrue(d.run_cpu) self.assertTrue(d.run_gpu) @@ -138,9 +252,13 @@ def test_family_label_adds_family(self): d = decide( "pull_request", ["projects/hrr/playback/x.cpp"], - labels=["ci:gfx110X-all", "ci:gfx90a"], + labels=["ci:gfx110X-all", "ci:gfx125x"], ) - self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx110x"]) + self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950", "gfx110x", "gfx125x"]) + + def test_label_for_family_not_in_config_is_ignored(self): + d = decide("pull_request", ["projects/hrr/playback/x.cpp"], labels=["ci:gfx103x"]) + self.assertNotIn("gfx103x", family_keys(d)) def test_fork_skips_gpu(self): d = decide("pull_request", ["projects/hrr/playback/x.cpp"], is_fork=True) @@ -148,10 +266,6 @@ def test_fork_skips_gpu(self): self.assertFalse(d.run_gpu) self.assertIn("fork", d.reason) - def test_push_uses_postsubmit_families(self): - d = decide("push", ["projects/hrr/playback/x.cpp"]) - self.assertEqual(family_keys(d), ["gfx90a", "gfx94x"]) - def test_push_without_capture_change_has_no_gpu(self): d = decide("push", ["projects/hrr/tests/unit/x.cc"]) self.assertTrue(d.run_cpu) @@ -162,23 +276,30 @@ def test_unlisted_change_set_runs_everything(self): self.assertTrue(d.run_cpu) self.assertTrue(d.run_gpu) - def test_nightly_skips_on_request_and_runnerless_families(self): + def test_schedule_follows_triggers(self): d = decide("schedule", []) self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950", "gfx110x"]) - def test_gfx950_uses_multi_gpu_runner(self): + def test_multi_gpu_runner(self): d = decide("schedule", []) runners = {f["family"]: f["runner"] for f in d.families} self.assertEqual(runners["gfx950"], "gpu-950-8") - def test_gfx950_not_on_pull_requests_by_default(self): - d = decide("pull_request", ["projects/hrr/playback/x.cpp"]) - self.assertNotIn("gfx950", family_keys(d)) + def test_runner_override(self): + config = cfg.parse_config(config_data(families={"gfx94x": {"runner": "my-gpu"}})) + d = decide("schedule", [], config=config) + self.assertEqual(d.families[0]["runner"], "my-gpu") + self.assertEqual(d.families[0]["gfx_targets"], "gfx942") - def test_dispatch_with_families(self): + def test_dispatch_families_replace_the_selection(self): d = decide("workflow_dispatch", [], requested=["gfx125x", "gfx1150"]) - # gfx1150 has no runner in this matrix. self.assertEqual(family_keys(d), ["gfx125x"]) + self.assertIn("no runner for gfx1150", d.reason) + + def test_dispatch_unknown_family(self): + with self.assertRaises(cfg.ConfigError) as caught: + decide("workflow_dispatch", [], requested=["gfx103x"]) + self.assertIn("gfx103x", str(caught.exception)) def test_unknown_event(self): d = decide("merge_group", []) @@ -188,16 +309,16 @@ def test_unknown_event(self): class OutputTest(unittest.TestCase): def test_write_outputs(self): - import tempfile - d = cfg.Decision(True, True, "why", [{"family": "gfx94x", "runner": "r", "gfx_targets": "gfx942"}]) with tempfile.TemporaryDirectory() as tmp: out = Path(tmp) / "out" - cfg.write_outputs(d, str(out)) + with redirect_stdout(io.StringIO()): + cfg.write_outputs(d, "1.2.3", str(out)) text = out.read_text() self.assertIn("run_cpu=true\n", text) self.assertIn('families=[{"family":"gfx94x","runner":"r","gfx_targets":"gfx942"}]\n', text) self.assertIn("reason=why\n", text) + self.assertIn("rocm_sdk_version=1.2.3\n", text) if __name__ == "__main__": From 59b1e0deb1de8faded1639fe29f2d90de18bb006 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 10:18:52 +0200 Subject: [PATCH 12/89] ci(hrr): build the GPU suite for targets.arch of integration.yaml The architecture list lived twice, as HRR_ARCHS in hrr-gpu.yml and as targets.arch in the suite configs, with a lint to keep them equal. hrr_capture_build.py now reads targets.arch from integration.yaml, so the config that decides where a case runs also decides what is built, and HRR_ARCHS and the lint's --workflow check go away. --archs still overrides it for a local build. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 2 -- .github/workflows/hrr-gpu.yml | 5 --- projects/hrr/tests/README.md | 8 +++-- .../hrr/tests/scripts/hrr_capture_build.py | 13 +++++-- projects/hrr/tests/scripts/hrr_test_config.py | 34 +++---------------- .../tests/scripts/test_hrr_capture_build.py | 13 +++++++ .../hrr/tests/scripts/test_hrr_test_config.py | 14 -------- 7 files changed, 33 insertions(+), 56 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 97c1f0fe702b..3a2b1ecc7703 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -151,7 +151,6 @@ jobs: # Cone mode also checks out projects/clr/hipamd/src/amdhip.def.in, # which gen_hrr_api_args.py reads on import. sparse-checkout: | - .github/workflows/hrr-gpu.yml projects/hrr projects/clr/hipamd/src/hrr @@ -167,7 +166,6 @@ jobs: python projects/hrr/tests/scripts/hrr_test_config.py lint projects/hrr/tests/config/unit.yaml projects/hrr/tests/config/integration.yaml - --workflow .github/workflows/hrr-gpu.yml - name: Run tests run: python -m pytest projects/hrr/tests/scripts diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 30e8e45d860e..bc1247fdcb82 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -25,11 +25,6 @@ on: permissions: contents: read -env: - # Every architecture the integration binary is built for. Must match - # targets.arch in projects/hrr/tests/config/*.yaml (hrr_test_config.py lint). - HRR_ARCHS: "gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1153;gfx1200;gfx1201;gfx1250" - jobs: build: name: Build capture-enabled HIP and HRR diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index 661b2a12b1dd..e5ad0c25a68a 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -156,7 +156,8 @@ pools, disable families or narrow their triggers here. ## Configuring where a case runs -The build is one fat binary for every architecture in `HRR_ARCHS`. Nothing about a +The build is one fat binary for every architecture in `targets.arch` of +`config/integration.yaml`. Nothing about a GPU is decided at build time. CI decides at run time, from [`config/integration.yaml`](config/integration.yaml) (and [`config/unit.yaml`](config/unit.yaml) for the CPU suite). @@ -190,8 +191,9 @@ skip: ``` Direct workloads (`[hrr-direct]`, `[direct]`) are spawned by a driver and must not -carry rules. `targets.arch` is the same list as `HRR_ARCHS` in -`.github/workflows/hrr-ci.yml`; `hrr_test_config.py lint --workflow` checks that. +carry rules. The GPU build compiles the integration binary for exactly the +`targets.arch` of `config/integration.yaml`, so adding an architecture there is +what builds it. CI compares each binary's registered cases against its YAML, in both directions, because a case behind an `#if` or a missing define disappears without any failure. diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index 4b9b1885e6dc..e87b05f19f03 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -36,6 +36,8 @@ from pathlib import Path from typing import Callable, Sequence +import hrr_test_config + Runner = Callable[[Sequence[str]], None] BASE_REPO = "ROCm/rocm-systems" @@ -77,6 +79,9 @@ "lib/libhsa-runtime64.so*", ) +# Its targets.arch is the architecture list the GPU suite is built for. +INTEGRATION_CONFIG = Path("projects/hrr/tests/config/integration.yaml") + # What the GPU suite runs. The host unit suite is built and run by hrr-ci.yml. HRR_TARGETS = ("hrr-playback", "hrr-integration-tests", "hrr-test-dispatch-tool") @@ -347,12 +352,16 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--build", type=Path, default=Path("build")) parser.add_argument("--runtime-lib", type=Path, default=Path("build/runtime-lib")) parser.add_argument("--base-json", type=Path, default=Path("build/base.json")) - parser.add_argument("--archs", default=os.environ.get("HRR_ARCHS", "")) + parser.add_argument( + "--archs", default="", + help=f"Semicolon-separated GPU targets; default targets.arch of {INTEGRATION_CONFIG}", + ) args = parser.parse_args(argv) for name in ("therock", "src", "dist", "build", "runtime_lib", "base_json"): setattr(args, name, getattr(args, name).resolve()) if args.command in ("build", "verify") and not args.archs: - parser.error("--archs (or HRR_ARCHS) is required") + config = hrr_test_config.load(args.src / INTEGRATION_CONFIG) + args.archs = ";".join(config["targets"]["arch"]) return args diff --git a/projects/hrr/tests/scripts/hrr_test_config.py b/projects/hrr/tests/scripts/hrr_test_config.py index 16cbfdb64bfd..af872d451259 100644 --- a/projects/hrr/tests/scripts/hrr_test_config.py +++ b/projects/hrr/tests/scripts/hrr_test_config.py @@ -15,7 +15,7 @@ ``kind: unsupported`` is permanent and needs only a reason. Usage: - hrr_test_config.py lint CONFIG [CONFIG ...] [--workflow hrr-ci.yml] + hrr_test_config.py lint CONFIG [CONFIG ...] hrr_test_config.py check --binary EXE --config CONFIG [--update] """ @@ -208,12 +208,8 @@ def _check_rule(errors: list[str], where: str, rule, targets: dict) -> None: _error(errors, where, "'issue' must be an https:// URL") -def lint(config: dict, cases: list[dict] | None = None, - workflow_archs: list[str] | None = None) -> list[str]: - """Schema and vocabulary errors. ``cases`` enables the workload-rule check. - - ``workflow_archs``, when given, must be the same set as ``targets.arch``. - """ +def lint(config: dict, cases: list[dict] | None = None) -> list[str]: + """Schema and vocabulary errors. ``cases`` enables the workload-rule check.""" errors: list[str] = [] unknown = set(config) - TOP_KEYS if unknown: @@ -234,14 +230,6 @@ def lint(config: dict, cases: list[dict] | None = None, isinstance(item, str) and item for item in values): _error(errors, "targets", f"'{key}' must be a non-empty list of strings") targets[key] = [] - if workflow_archs is not None and targets.get("arch") is not None: - got = set(targets["arch"]) - want = set(workflow_archs) - if got != want: - missing = ", ".join(sorted(want - got)) or "-" - extra = ", ".join(sorted(got - want)) or "-" - _error(errors, "targets.arch", - f"does not match HRR_ARCHS (missing: {missing}; extra: {extra})") suite_skip = config.get("skip", []) if not isinstance(suite_skip, list): _error(errors, "skip", "must be a list") @@ -382,18 +370,6 @@ def apply_listing(config: dict, names: list[str]) -> tuple[list[str], list[str]] return added, removed -def workflow_archs(path: str | Path) -> list[str]: - """The ``HRR_ARCHS`` list from the HRR CI workflow.""" - try: - text = Path(path).read_text(encoding="utf-8") - except OSError as exc: - raise ConfigError(f"cannot read {path}: {exc}") from exc - match = re.search(r'^ {2}HRR_ARCHS:\s*"([^"]+)"\s*$', text, re.MULTILINE) - if not match: - raise ConfigError(f"no HRR_ARCHS list in {path}") - return [arch for arch in match.group(1).split(";") if arch] - - def detect_os() -> str: return "windows" if os.name == "nt" else "linux" @@ -447,7 +423,6 @@ def _print_errors(path: str, errors: list[str]) -> None: def _cmd_lint(args: argparse.Namespace) -> int: - archs = workflow_archs(args.workflow) if args.workflow else None failed = False for path in args.configs: try: @@ -456,7 +431,7 @@ def _cmd_lint(args: argparse.Namespace) -> int: print(f"FAIL: {exc}", file=sys.stderr) failed = True continue - errors = lint(config, workflow_archs=archs) + errors = lint(config) if errors: _print_errors(path, errors) failed = True @@ -514,7 +489,6 @@ def main(argv: list[str] | None = None) -> int: lint_cmd = commands.add_parser("lint", help="validate config files") lint_cmd.add_argument("configs", nargs="+") - lint_cmd.add_argument("--workflow", help="HRR workflow whose HRR_ARCHS must match targets.arch") lint_cmd.set_defaults(func=_cmd_lint) check_cmd = commands.add_parser("check", help="compare a binary's cases to a config") diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index 7c7d1a6ae08e..3d939667252c 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -217,6 +217,19 @@ def runner(cmd): ) +class ArchsTest(unittest.TestCase): + SRC = Path(__file__).resolve().parents[4] + + def test_default_is_the_integration_config(self): + config = cb.hrr_test_config.load(self.SRC / cb.INTEGRATION_CONFIG) + args = cb.parse_args(["build", "--src", str(self.SRC)]) + self.assertEqual(args.archs, ";".join(config["targets"]["arch"])) + + def test_override(self): + args = cb.parse_args(["verify", "--src", str(self.SRC), "--archs", "gfx942;gfx1152"]) + self.assertEqual(args.archs, "gfx942;gfx1152") + + class ExecutableBitsTest(unittest.TestCase): def test_restores_bits(self): with tempfile.TemporaryDirectory() as tmp: diff --git a/projects/hrr/tests/scripts/test_hrr_test_config.py b/projects/hrr/tests/scripts/test_hrr_test_config.py index af5c45560a6b..9aad89369ffb 100644 --- a/projects/hrr/tests/scripts/test_hrr_test_config.py +++ b/projects/hrr/tests/scripts/test_hrr_test_config.py @@ -155,11 +155,6 @@ def test_a_direct_workload_cannot_carry_rules(self): self.assertTrue(any("workload" in error for error in errors)) self.assertEqual(cfg.lint(suite({"Unit_HRR_Foo_Direct": {}}), cases=listed), []) - def test_workflow_archs_must_match(self): - errors = cfg.lint(suite({"Runs": {}}), workflow_archs=["gfx90a"]) - self.assertTrue(any("HRR_ARCHS" in error for error in errors)) - self.assertEqual(cfg.lint(suite({"Runs": {}}), workflow_archs=list(ARCHS)), []) - class ListingTest(unittest.TestCase): def test_json_listing_keeps_tags_and_passes_the_selector(self): @@ -259,15 +254,6 @@ def test_nothing_visible_is_an_empty_list(self): self.assertEqual(cfg.detect_archs(lambda argv: None), []) -class WorkflowTest(unittest.TestCase): - def test_reads_hrr_archs(self): - tmp = tempfile.NamedTemporaryFile("w", delete=False, encoding="utf-8") - self.addCleanup(lambda: Path(tmp.name).unlink(missing_ok=True)) - tmp.write('env:\n HRR_ARCHS: "gfx90a;gfx942"\n') - tmp.close() - self.assertEqual(cfg.workflow_archs(tmp.name), ["gfx90a", "gfx942"]) - - class LintCommandTest(unittest.TestCase): def test_lint_command_fails_on_a_bad_file(self): tmp = tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False, encoding="utf-8") From d8b0f3a375ed4fdc219ee95c0584c4fd70d0be3c Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 10:47:14 +0200 Subject: [PATCH 13/89] ci(hrr): give the GPU suite the container's path to the install ROCM_SDK came from github.workspace, which in a container job is the runner's host path, not the workspace inside the container. On the gfx125x runner that path does not exist in the container, so --arch auto found no rocminfo and the job failed before running a case. Take the path from GITHUB_WORKSPACE, which is the workspace inside the container. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-gpu.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index bc1247fdcb82..89878301e014 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -165,12 +165,14 @@ jobs: env: FAMILY: ${{ matrix.family }} HRR_TEST_IGNORE_CONFIG: ${{ inputs.ignore_test_config && '1' || '' }} - # --arch auto finds this machine's GPU with the installed rocminfo. - ROCM_SDK: ${{ github.workspace }}/dist # A regions case faults the GPU on purpose; without this ROCr writes a # gpucore file of about 160 MB into the workspace. HSA_DISABLE_COREDUMP_ON_EXCEPTION: "1" + # --arch auto finds this machine's GPU with the installed rocminfo. + # github.workspace is the runner's host path, which a container need + # not have; GITHUB_WORKSPACE is the path inside the container. run: >- + ROCM_SDK="${GITHUB_WORKSPACE}/dist" python projects/hrr/tests/scripts/run_catch2.py --xml "integration-${FAMILY}.xml" --timeout 1200 From 16fd5e8dc522c3212ae963a6f4ff614dea657b58 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 16:16:18 +0200 Subject: [PATCH 14/89] ci(hrr): use the base's ROCr again in the interim GPU build This takes back the ROCr part of a824c56addc. HRR lives in projects/hrr and projects/clr only, so the interim build rebuilds just CLR with capture enabled, and ROCr comes prebuilt from the develop artifacts the build already installs, as before that commit. The ldd check stays: verify and install-base still fail unless the suite loads exactly one libamdhip64, the capture-enabled one built here, and one libhsa-runtime64, both from the installed tree. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-gpu.yml | 10 +- .../hrr/tests/scripts/hrr_capture_build.py | 130 ++++++------------ .../tests/scripts/test_hrr_capture_build.py | 91 ++++-------- 3 files changed, 72 insertions(+), 159 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 89878301e014..44b2f94223b1 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -1,6 +1,5 @@ -# HRR GPU suite: build ROCr and an amdhip64 with HRR capture compiled in from -# this checkout, then run the HRR capture/replay integration suite on each -# requested GPU family. +# HRR GPU suite: build an amdhip64 with HRR capture compiled in, then run the +# HRR capture/replay integration suite on each requested GPU family. # # Called by hrr-ci.yml, which decides when this runs and on which families. # @@ -41,7 +40,6 @@ jobs: projects/clr projects/hip projects/hrr - projects/rocr-runtime - name: Checkout TheRock artifact tools uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -85,7 +83,7 @@ jobs: name: hrr-capture-${{ github.sha }} path: | build/base.json - build/runtime-lib/ + build/capture-lib/ build/hrr/playback/hrr-playback build/hrr/tests/integration/hrr-integration-tests build/hrr/tests/integration/*.so @@ -156,7 +154,7 @@ jobs: name: hrr-capture-${{ github.sha }} path: build - - name: Install base ROCm artifacts and the runtime libraries built here + - name: Install base ROCm artifacts and the capture-enabled amdhip64 env: GITHUB_TOKEN: ${{ github.token }} run: python projects/hrr/tests/scripts/hrr_capture_build.py install-base diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index e87b05f19f03..163744bdafe0 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -9,19 +9,19 @@ Release builds compile capture out (HIP_HRR_CAPTURE_ENABLED undefined). This script installs the ROCm artifacts that rocm-systems' own TheRock CI built for -a recent develop commit, then builds ROCr and CLR (with capture enabled) from -this checkout over them, and builds projects/hrr on top. The compiler, comgr, -the device libraries, rocprofiler-register and the sysdeps stay prebuilt. The -GPU jobs install the same artifacts plus these two runtime libraries, so build -and test use the same libraries, and the binaries' RUNPATH needs no +a recent develop commit, rebuilds only CLR from this checkout with capture +enabled, installs it over those artifacts, and builds projects/hrr on top. HRR +lives in projects/hrr and projects/clr only; ROCr and everything else stay +prebuilt. The GPU jobs then install the same artifacts plus this amdhip64, so +build and test use the same libraries, and the binaries' RUNPATH needs no LD_LIBRARY_PATH. fetch-base Pick the base run and install the build-time artifacts. - build Build ROCr, then CLR with capture, into the base; build HRR. + build Build CLR with capture, install it into the base, build HRR. verify Check the suite's case list, the embedded GPU targets and - which runtime libraries the suite loads. + which amdhip64 and ROCr the suite loads. install-base On a GPU machine: install the run-time artifacts of the - recorded base run and the runtime libraries built here. + recorded base run and the capture-enabled amdhip64. """ from __future__ import annotations @@ -46,7 +46,7 @@ # The generic (target-neutral) artifacts are the same in every family group. BASE_ARTIFACT_GROUP = "gfx94X-dcgpu" -# Artifacts needed to build ROCr, CLR and projects/hrr. +# Artifacts needed to build CLR and projects/hrr. BUILD_ARTIFACTS = ( "amd-llvm_run", "amd-llvm_lib", "amd-llvm_dev", "base_run", "base_lib", "base_dev", @@ -64,20 +64,11 @@ "core-hip_lib", ) -# The libraries built from this checkout that replace the base's copies. -RUNTIME_LIB_GLOBS = ("libamdhip64.so*", "libhsa-runtime64.so*") - -# Everything ROCr installs into the base, relative to --dist, so its build -# replaces the base's ROCr files instead of mixing with them. -ROCR_INSTALLED = ( - "include/hsa", - "include/hsakmt", - "lib/cmake/hsa-runtime64", - "lib/cmake/hsakmt", - "lib/libhsakmt.a", - "lib/pkgconfig/libhsakmt.pc", - "lib/libhsa-runtime64.so*", -) +CAPTURE_LIB_GLOB = "libamdhip64.so*" + +# The suite must load exactly one copy of each, from the installed tree: the +# capture-enabled amdhip64 built here and the base's ROCr. +LOADED_LIBS = ("libamdhip64.so", "libhsa-runtime64.so") # Its targets.arch is the architecture list the GPU suite is built for. INTEGRATION_CONFIG = Path("projects/hrr/tests/config/integration.yaml") @@ -140,28 +131,6 @@ def fetch_command(therock: Path, base: dict, dist: Path, artifacts: Sequence[str ] -def rocr_configure_command(src: Path, build: Path, dist: Path) -> list: - sysdeps = dist / "lib" / "rocm_sysdeps" - cmake_dir = dist / "lib" / "llvm" / "lib" / "cmake" - # The sysdeps provide libelf, libdrm and libnuma, as in TheRock. - return [ - "env", f"PKG_CONFIG_PATH={sysdeps / 'lib' / 'pkgconfig'}", - "cmake", "-S", src / "projects" / "rocr-runtime", "-B", build / "rocr", - "-GNinja", "-Wno-dev", - "-DCMAKE_BUILD_TYPE=Release", - "-DBUILD_SHARED_LIBS=ON", - f"-DCMAKE_PREFIX_PATH={dist};{sysdeps}", - f"-DCMAKE_INSTALL_PREFIX={dist}", - "-DCMAKE_INSTALL_LIBDIR=lib", - # TheRock's libhsa-runtime64 RUNPATH. - "-DCMAKE_INSTALL_RPATH=$ORIGIN;$ORIGIN/rocm_sysdeps/lib", - # The blit and trap-handler kernels need the base's clang; ROCr's own - # hint (${CMAKE_PREFIX_PATH}/llvm) does not work with a list. - f"-DClang_DIR={cmake_dir / 'clang'}", - f"-DLLVM_DIR={cmake_dir / 'llvm'}", - ] - - def clr_configure_command(src: Path, build: Path, dist: Path) -> list: # HIP_HRR_CAPTURE_ENABLED is what compiles capture into amdhip64; CLR has # no CMake option for it. @@ -205,52 +174,40 @@ def hrr_configure_command(src: Path, build: Path, dist: Path, archs: str) -> lis ] -def remove_paths(root: Path, patterns: Sequence[str]) -> None: - for pattern in patterns: - for path in root.glob(pattern): - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path) - else: - path.unlink() - - -def remove_runtime_libs(lib_dir: Path) -> None: - """Remove the base's runtime libraries so only the ones built here remain.""" - remove_paths(lib_dir, RUNTIME_LIB_GLOBS) +def remove_amdhip64(lib_dir: Path) -> None: + """Remove the release amdhip64 so only the capture-enabled one remains.""" + for path in lib_dir.glob(CAPTURE_LIB_GLOB): + path.unlink() -def copy_runtime_libs(src_dir: Path, dst_dir: Path) -> list[str]: - """Copy libamdhip64.so* and libhsa-runtime64.so*, keeping symlinks.""" +def copy_amdhip64(src_dir: Path, dst_dir: Path) -> list[str]: + """Copy libamdhip64.so* from src_dir to dst_dir, keeping symlinks.""" dst_dir.mkdir(parents=True, exist_ok=True) copied = [] - for pattern in RUNTIME_LIB_GLOBS: - found = sorted(src_dir.glob(pattern)) - if not found: - raise SystemExit(f"no {pattern} in {src_dir}") - for path in found: - target = dst_dir / path.name - if target.is_symlink() or target.exists(): - target.unlink() - if path.is_symlink(): - os.symlink(os.readlink(path), target) - else: - shutil.copy2(path, target) - copied.append(path.name) + for path in sorted(src_dir.glob(CAPTURE_LIB_GLOB)): + target = dst_dir / path.name + if target.is_symlink() or target.exists(): + target.unlink() + if path.is_symlink(): + os.symlink(os.readlink(path), target) + else: + shutil.copy2(path, target) + copied.append(path.name) + if not copied: + raise SystemExit(f"no {CAPTURE_LIB_GLOB} in {src_dir}") return copied def linkage_errors(ldd_output: str, lib_dir: Path) -> list[str]: - """Each runtime library must resolve to exactly one file, in lib_dir.""" + """Each of LOADED_LIBS must resolve to exactly one file, in lib_dir.""" resolved: dict[str, set[str]] = {} for match in re.finditer(r"^\s*(\S+) => (\S+)", ldd_output, re.MULTILINE): name, path = match.groups() - for pattern in RUNTIME_LIB_GLOBS: - stem = pattern.rstrip("*") + for stem in LOADED_LIBS: if name.startswith(stem): resolved.setdefault(stem, set()).add(path) errors = [] - for pattern in RUNTIME_LIB_GLOBS: - stem = pattern.rstrip("*") + for stem in LOADED_LIBS: paths = resolved.get(stem, set()) if len(paths) != 1: errors.append(f"{stem} resolves to {sorted(paths) or 'nothing'}") @@ -265,7 +222,7 @@ def check_linkage(binary: Path, lib_dir: Path) -> int: for error in errors: print(f"::error::{binary.name}: {error}") if not errors: - print(f"{binary.name} loads the runtime libraries from {lib_dir}") + print(f"{binary.name} loads {' and '.join(LOADED_LIBS)} from {lib_dir}") return 1 if errors else 0 @@ -290,16 +247,11 @@ def cmd_fetch_base(args: argparse.Namespace, runner: Runner = run) -> int: def cmd_build(args: argparse.Namespace, runner: Runner = run) -> int: - runner(rocr_configure_command(args.src, args.build, args.dist)) - runner(["cmake", "--build", args.build / "rocr"]) - remove_paths(args.dist, ROCR_INSTALLED) - runner(["cmake", "--install", args.build / "rocr"]) - # CLR finds the ROCr just installed through its hsa-runtime64 package. runner(clr_configure_command(args.src, args.build, args.dist)) runner(["cmake", "--build", args.build / "clr"]) - remove_paths(args.dist / "lib", ("libamdhip64.so*",)) + remove_amdhip64(args.dist / "lib") runner(["cmake", "--install", args.build / "clr"]) - copy_runtime_libs(args.dist / "lib", args.runtime_lib) + copy_amdhip64(args.dist / "lib", args.capture_lib) runner(hrr_configure_command(args.src, args.build, args.dist, args.archs)) runner(["cmake", "--build", args.build / "hrr", "--target", *HRR_TARGETS]) return 0 @@ -335,8 +287,8 @@ def cmd_install_base(args: argparse.Namespace, runner: Runner = run) -> int: base = json.loads(args.base_json.read_text()) print(f"Base: {base['repo']} run {base['run_id']} at {base['commit']}") runner(fetch_command(args.therock, base, args.dist, RUNTIME_ARTIFACTS)) - remove_runtime_libs(args.dist / "lib") - copy_runtime_libs(args.runtime_lib, args.dist / "lib") + remove_amdhip64(args.dist / "lib") + copy_amdhip64(args.capture_lib, args.dist / "lib") restore_executable_bits(args.build) return check_linkage( args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests", args.dist / "lib" @@ -350,14 +302,14 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--src", type=Path, default=Path.cwd()) parser.add_argument("--dist", type=Path, default=Path("dist")) parser.add_argument("--build", type=Path, default=Path("build")) - parser.add_argument("--runtime-lib", type=Path, default=Path("build/runtime-lib")) + parser.add_argument("--capture-lib", type=Path, default=Path("build/capture-lib")) parser.add_argument("--base-json", type=Path, default=Path("build/base.json")) parser.add_argument( "--archs", default="", help=f"Semicolon-separated GPU targets; default targets.arch of {INTEGRATION_CONFIG}", ) args = parser.parse_args(argv) - for name in ("therock", "src", "dist", "build", "runtime_lib", "base_json"): + for name in ("therock", "src", "dist", "build", "capture_lib", "base_json"): setattr(args, name, getattr(args, name).resolve()) if args.command in ("build", "verify") and not args.archs: config = hrr_test_config.load(args.src / INTEGRATION_CONFIG) diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index 3d939667252c..f37a60da7177 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -56,16 +56,6 @@ def test_clr_compiles_capture_in(self): self.assertIn("-DCMAKE_INSTALL_PREFIX=/d", cmd) self.assertIn("-DHIP_COMMON_DIR=/s/projects/hip", cmd) - def test_rocr_installs_like_therock(self): - cmd = [str(c) for c in cb.rocr_configure_command(Path("/s"), Path("/b"), Path("/d"))] - self.assertEqual(cmd[:2], ["env", "PKG_CONFIG_PATH=/d/lib/rocm_sysdeps/lib/pkgconfig"]) - self.assertEqual(cmd[cmd.index("-S") + 1], "/s/projects/rocr-runtime") - self.assertIn("-DCMAKE_PREFIX_PATH=/d;/d/lib/rocm_sysdeps", cmd) - self.assertIn("-DCMAKE_INSTALL_PREFIX=/d", cmd) - self.assertIn("-DCMAKE_INSTALL_LIBDIR=lib", cmd) - self.assertIn("-DCMAKE_INSTALL_RPATH=$ORIGIN;$ORIGIN/rocm_sysdeps/lib", cmd) - self.assertIn("-DClang_DIR=/d/lib/llvm/lib/cmake/clang", cmd) - def test_hrr_builds_integration_suite_for_archs(self): cmd = [str(c) for c in cb.hrr_configure_command(Path("/s"), Path("/b"), Path("/d"), "gfx942;gfx950")] self.assertIn("-DGPU_TARGETS=gfx942;gfx950", cmd) @@ -104,53 +94,34 @@ def setUp(self): def tearDown(self): self._tmp.cleanup() - def make_lib(self, directory: Path, stem: str, soname: str, real: str, content: str) -> None: + def make_lib(self, directory: Path, real: str, content: str) -> None: directory.mkdir(parents=True, exist_ok=True) (directory / real).write_text(content) - os.symlink(real, directory / soname) - os.symlink(soname, directory / stem) - - def make_runtime(self, directory: Path, version: str, content: str) -> None: - self.make_lib(directory, "libamdhip64.so", "libamdhip64.so.7", - f"libamdhip64.so.7.{version}", content) - self.make_lib(directory, "libhsa-runtime64.so", "libhsa-runtime64.so.1", - f"libhsa-runtime64.so.1.{version}", content) + os.symlink(real, directory / "libamdhip64.so.7") + os.symlink("libamdhip64.so.7", directory / "libamdhip64.so") - def test_overlay_replaces_base_libraries(self): - built = self.tmp / "built" + def test_overlay_replaces_release_library_and_keeps_rocr(self): + capture = self.tmp / "capture" dist = self.tmp / "dist" / "lib" - self.make_runtime(built, "2.1", "pr") - self.make_runtime(dist, "2.0", "base") + self.make_lib(capture, "libamdhip64.so.7.2.1", "capture") + self.make_lib(dist, "libamdhip64.so.7.2.0", "release") (dist / "libhiprtc.so.7").write_text("keep") + (dist / "libhsa-runtime64.so.1").write_text("base") - cb.remove_runtime_libs(dist) - copied = cb.copy_runtime_libs(built, dist) + cb.remove_amdhip64(dist) + copied = cb.copy_amdhip64(capture, dist) - self.assertEqual(len(copied), 6) + self.assertEqual(sorted(copied), ["libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.7.2.1"]) self.assertFalse((dist / "libamdhip64.so.7.2.0").exists()) - self.assertFalse((dist / "libhsa-runtime64.so.1.2.0").exists()) - self.assertEqual((dist / "libamdhip64.so").read_text(), "pr") - self.assertEqual((dist / "libhsa-runtime64.so").read_text(), "pr") - self.assertTrue((dist / "libhsa-runtime64.so.1").is_symlink()) + self.assertEqual((dist / "libamdhip64.so").read_text(), "capture") + self.assertTrue((dist / "libamdhip64.so.7").is_symlink()) self.assertEqual((dist / "libhiprtc.so.7").read_text(), "keep") + self.assertEqual((dist / "libhsa-runtime64.so.1").read_text(), "base") - def test_copy_without_rocr_fails(self): - built = self.tmp / "built" - self.make_lib(built, "libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.7.2.1", "pr") + def test_copy_without_library_fails(self): + (self.tmp / "empty").mkdir() with self.assertRaises(SystemExit): - cb.copy_runtime_libs(built, self.tmp / "out") - - def test_remove_rocr_install(self): - dist = self.tmp / "dist" - (dist / "include" / "hsa").mkdir(parents=True) - (dist / "include" / "hsa" / "hsa.h").write_text("") - (dist / "include" / "hip").mkdir(parents=True) - self.make_runtime(dist / "lib", "2.0", "base") - cb.remove_paths(dist, cb.ROCR_INSTALLED) - self.assertFalse((dist / "include" / "hsa").exists()) - self.assertTrue((dist / "include" / "hip").exists()) - self.assertFalse(list((dist / "lib").glob("libhsa-runtime64.so*"))) - self.assertTrue((dist / "lib" / "libamdhip64.so.7").exists()) + cb.copy_amdhip64(self.tmp / "empty", self.tmp / "out") class LinkageTest(unittest.TestCase): @@ -184,37 +155,29 @@ def test_missing_rocr(self): class BuildTest(unittest.TestCase): - def test_rocr_is_installed_before_clr_is_configured(self): + def test_builds_clr_and_hrr_over_the_prebuilt_rocr(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) dist = root / "dist" - (dist / "include" / "hsa").mkdir(parents=True) - (dist / "lib").mkdir() + (dist / "lib").mkdir(parents=True) + (dist / "lib" / "libhsa-runtime64.so.1").write_text("base") calls = [] def runner(cmd): cmd = [str(c) for c in cmd] calls.append(cmd) - # The base's ROCr headers are gone before ROCr installs. - if cmd[:2] == ["cmake", "--install"] and cmd[2].endswith("rocr"): - self.assertFalse((dist / "include" / "hsa").exists()) - if cmd[:2] == ["cmake", "--install"] and cmd[2].endswith("clr"): - for name in ("libamdhip64.so.7", "libhsa-runtime64.so.1"): - (dist / "lib" / name).write_text("") + if cmd[:2] == ["cmake", "--install"]: + (dist / "lib" / "libamdhip64.so.7").write_text("capture") args = argparse.Namespace( src=root / "src", build=root / "build", dist=dist, - runtime_lib=root / "build" / "runtime-lib", archs="gfx942", + capture_lib=root / "build" / "capture-lib", archs="gfx942", ) cb.cmd_build(args, runner=runner) - steps = [" ".join(c[c.index("cmake"):c.index("cmake") + 3]) for c in calls] - rocr_install = steps.index(f"cmake --install {root}/build/rocr") - clr_configure = next(i for i, c in enumerate(calls) if "projects/clr" in " ".join(c)) - self.assertLess(rocr_install, clr_configure) - self.assertEqual( - sorted(p.name for p in args.runtime_lib.iterdir()), - ["libamdhip64.so.7", "libhsa-runtime64.so.1"], - ) + sources = [c[c.index("-S") + 1] for c in calls if "-S" in c] + self.assertEqual(sources, [f"{root}/src/projects/clr", f"{root}/src/projects/hrr"]) + self.assertEqual([p.name for p in args.capture_lib.iterdir()], ["libamdhip64.so.7"]) + self.assertEqual((dist / "lib" / "libhsa-runtime64.so.1").read_text(), "base") class ArchsTest(unittest.TestCase): From 5a4e0f94eb2a7a70ca25f7f45fc7aca3832c79c5 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 16:17:44 +0200 Subject: [PATCH 15/89] ci(hrr): take the 8-GPU gfx950 runner only nightly gfx950 used TheRock's multi-GPU runner on every event so the cases that need two devices ran. That runner is an 8-GPU node, and the gfx950 multi-GPU pool has two of them; neither TheRock's family matrix nor therock-ci-config has a smaller multi-GPU pool for any HRR family. Replace multi_gpu with multi_gpu_triggers, the events on which a family takes the multi-GPU runner, default none. gfx950 names only schedule: pull requests and pushes run on the single-GPU runner, where those cases skip, and the nightly run still covers them once a day. Signed-off-by: Daniel Olsson --- projects/hrr/tests/README.md | 5 +++- projects/hrr/tests/config/ci.yaml | 11 +++++-- .../hrr/tests/scripts/hrr_ci_configure.py | 29 +++++++++++++------ .../tests/scripts/test_hrr_ci_configure.py | 23 ++++++++++----- 4 files changed, 48 insertions(+), 20 deletions(-) diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index e5ad0c25a68a..e1444eda2c2e 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -146,7 +146,10 @@ applies (`scripts/hrr_ci_configure.py`): TheRock's family matrix names. By default each runs on every event (`pull_request`, `push`, `schedule`, `workflow_dispatch`). Per family, `enabled: false` leaves it to explicit requests, `triggers` narrows the events, `runner` overrides the runner label - and `multi_gpu: true` takes TheRock's multi-GPU runner. A `ci:gfx...` label adds a + and `multi_gpu_triggers` names the events on which it takes TheRock's multi-GPU + runner instead of the single-GPU one. TheRock's multi-GPU pools are 8-GPU nodes, so + only gfx950 uses one, nightly; elsewhere the cases that need two devices skip and + show in the summary's Skip column. A `ci:gfx...` label adds a family; the `workflow_dispatch` input `families` replaces the selection. Both can name a disabled family. - `rocm_sdk_version`: the ROCm SDK the unit job installs. diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 8d325d526b30..228f951ae095 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -62,8 +62,12 @@ capture_label: ci:hrr-capture # triggers Events that run this family: pull_request, push, schedule, # workflow_dispatch. Default all four. # runner Runner label to use instead of TheRock's. -# multi_gpu Use TheRock's multi-GPU runner; the T3 tier needs two devices. -# Default false. +# multi_gpu_triggers +# Events on which to take TheRock's multi-GPU runner instead of +# its single-GPU one. Default none. The cases that need two +# devices (the T3 tier) skip on a single GPU. TheRock's only +# multi-GPU pools are 8-GPU nodes, so name only events that must +# cover those cases. # # A family without a runner label is skipped. Every job queues on its own # runner pool and the summary waits for all of them, so to reduce GPU load or @@ -71,8 +75,9 @@ capture_label: ci:hrr-capture families: gfx90a: {} gfx94x: {} + # Nightly on an 8-GPU node, so the multi-device cases run once a day. gfx950: - multi_gpu: true + multi_gpu_triggers: [schedule] gfx103x: {} gfx110x: {} gfx1150: {} diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index c39446cb881d..4228eb862180 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -20,7 +20,8 @@ It runs on every enabled family whose triggers include the event. A ci:gfx... label adds that family, the same label TheRock CI uses; the workflow_dispatch families input replaces the selection. Both can name a disabled family. The -runner labels come from TheRock's family matrix unless ci.yaml overrides them. +runner labels come from TheRock's family matrix unless ci.yaml overrides them; +a family takes TheRock's multi-GPU runner only on its multi_gpu_triggers. """ from __future__ import annotations @@ -54,7 +55,7 @@ class Family: enabled: bool = True triggers: tuple[str, ...] = EVENTS runner: str = "" - multi_gpu: bool = False + multi_gpu_triggers: tuple[str, ...] = () @dataclass(frozen=True) @@ -105,19 +106,28 @@ def _bool(where: str, value) -> bool: return value -def _family(where: str, value) -> Family: - value = _mapping(where, value or {}, (), ("enabled", "triggers", "runner", "multi_gpu")) - triggers = _strings(f"{where}.triggers", value.get("triggers", list(EVENTS))) +def _triggers(where: str, value) -> tuple[str, ...]: + triggers = _strings(where, value) for trigger in triggers: if trigger not in EVENTS: raise ConfigError( - f"{where}.triggers: unknown trigger {trigger!r} (expected {', '.join(EVENTS)})" + f"{where}: unknown trigger {trigger!r} (expected {', '.join(EVENTS)})" ) + return triggers + + +def _family(where: str, value) -> Family: + value = _mapping( + where, value or {}, (), ("enabled", "triggers", "runner", "multi_gpu_triggers") + ) return Family( enabled=_bool(f"{where}.enabled", value.get("enabled", True)), - triggers=triggers, + triggers=_triggers(f"{where}.triggers", value.get("triggers", list(EVENTS))), runner=_string(f"{where}.runner", value["runner"]) if "runner" in value else "", - multi_gpu=_bool(f"{where}.multi_gpu", value.get("multi_gpu", False)), + multi_gpu_triggers=( + _triggers(f"{where}.multi_gpu_triggers", value["multi_gpu_triggers"]) + if "multi_gpu_triggers" in value else () + ), ) @@ -235,7 +245,8 @@ def select_families( if key not in chosen: continue linux = matrix.get(key, {}).get("linux", {}) - runner = family.runner or runner_for(linux, pick_weighted, family.multi_gpu) + multi_gpu = event in family.multi_gpu_triggers + runner = family.runner or runner_for(linux, pick_weighted, multi_gpu) if not runner: no_runner.append(key) continue diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 5fb78a3fc902..f69af93e2772 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -77,7 +77,7 @@ FAMILIES = { "gfx90a": {}, "gfx94x": None, - "gfx950": {"multi_gpu": True}, + "gfx950": {"multi_gpu_triggers": ["schedule"]}, "gfx110x": {"triggers": ["schedule", "workflow_dispatch"]}, "gfx1150": {}, "gfx125x": {"enabled": False}, @@ -115,8 +115,10 @@ def test_every_family_runs_on_every_event_by_default(self): self.assertTrue(family.enabled, key) self.assertEqual(family.triggers, cfg.EVENTS, key) - def test_gfx950_takes_the_multi_gpu_runner(self): - self.assertTrue(REPO.families["gfx950"].multi_gpu) + def test_multi_gpu_runner_only_nightly(self): + for key, family in REPO.families.items(): + self.assertIn(family.multi_gpu_triggers, ((), ("schedule",)), key) + self.assertEqual(REPO.families["gfx950"].multi_gpu_triggers, ("schedule",)) class PathClassificationTest(unittest.TestCase): @@ -177,6 +179,12 @@ def test_unknown_trigger(self): families = {"gfx94x": {"triggers": ["nightly"]}} self.assertRejected(config_data(families=families), "unknown trigger 'nightly'") + def test_unknown_multi_gpu_trigger(self): + families = {"gfx950": {"multi_gpu_triggers": ["nightly"]}} + self.assertRejected( + config_data(families=families), "families.gfx950.multi_gpu_triggers: unknown trigger" + ) + def test_enabled_must_be_a_bool(self): families = {"gfx94x": {"enabled": "no"}} self.assertRejected(config_data(families=families), "families.gfx94x.enabled") @@ -280,10 +288,11 @@ def test_schedule_follows_triggers(self): d = decide("schedule", []) self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950", "gfx110x"]) - def test_multi_gpu_runner(self): - d = decide("schedule", []) - runners = {f["family"]: f["runner"] for f in d.families} - self.assertEqual(runners["gfx950"], "gpu-950-8") + def test_multi_gpu_runner_on_its_triggers_only(self): + nightly = {f["family"]: f["runner"] for f in decide("schedule", []).families} + self.assertEqual(nightly["gfx950"], "gpu-950-8") + pr = decide("pull_request", ["projects/hrr/playback/x.cpp"]) + self.assertEqual({f["family"]: f["runner"] for f in pr.families}["gfx950"], "gpu-950-1") def test_runner_override(self): config = cfg.parse_config(config_data(families={"gfx94x": {"runner": "my-gpu"}})) From 5d4198d982af37eab370341e2ba90600b81b4d82 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 16:18:18 +0200 Subject: [PATCH 16/89] ci(hrr): leave gfx125x to explicit requests In both runs of this workflow the MI455 runner picked the gfx125x job up, but run_catch2.py's --arch auto found no GPU in the container, so no case ran and the job failed. Until that is understood, disable the family in ci.yaml. TheRock's runner config also reserves that small pool for designated workflows. A ci:gfx125x label or the dispatch families input still runs it. Signed-off-by: Daniel Olsson --- projects/hrr/tests/config/ci.yaml | 9 ++++++--- projects/hrr/tests/scripts/test_hrr_ci_configure.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 228f951ae095..d0dd7c34ccf3 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -84,6 +84,9 @@ families: gfx1151: {} gfx1153: {} gfx120x: {} - # This pool is not reliably online; a job that finds no runner stays queued - # and holds up the summary until GitHub times it out. - gfx125x: {} + # Off until understood: in two runs the MI455 runner picked the job up, but + # --arch auto found no GPU, so no case ran. TheRock also reserves this small + # pool for designated workflows. A ci:gfx125x label or the workflow_dispatch + # families input still runs it. + gfx125x: + enabled: false diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index f69af93e2772..a58061afcf50 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -112,7 +112,7 @@ def family_keys(decision): class RepoConfigTest(unittest.TestCase): def test_every_family_runs_on_every_event_by_default(self): for key, family in REPO.families.items(): - self.assertTrue(family.enabled, key) + self.assertEqual(family.enabled, key != "gfx125x", key) self.assertEqual(family.triggers, cfg.EVENTS, key) def test_multi_gpu_runner_only_nightly(self): From f5d98cf22c3c2dba85058dafb8b8607f6799a2aa Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 16:19:27 +0200 Subject: [PATCH 17/89] ci(hrr): treat a pull request from a deleted head repository as a fork The GPU suite is skipped for fork pull requests, detected from head.repo.fork. When the head repository has been deleted, head.repo is null, the expression gave false, and the pull request was treated as coming from this repository. The configure job now gets head.repo.full_name and counts a pull request as a fork unless that is this repository, so a missing head repository is a fork too. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 5 +-- .../hrr/tests/scripts/hrr_ci_configure.py | 11 ++++-- .../tests/scripts/test_hrr_ci_configure.py | 36 +++++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 3a2b1ecc7703..cc495871576b 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -98,13 +98,14 @@ jobs: BEFORE: ${{ github.event.before }} AFTER: ${{ github.sha }} LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }} - IS_FORK: ${{ github.event.pull_request.head.repo.fork || false }} + # Empty when the head repository was deleted, which counts as a fork. + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} FAMILIES: ${{ inputs.families }} run: >- python projects/hrr/tests/scripts/hrr_ci_configure.py --event "${EVENT}" --repo "${REPO}" --pr-number "${PR_NUMBER}" --before "${BEFORE}" --after "${AFTER}" --labels "${LABELS}" - --is-fork "${IS_FORK}" --families "${FAMILIES}" + --head-repo "${HEAD_REPO}" --families "${FAMILIES}" --therock-dir therock codegen: diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index 4228eb862180..0f803a245498 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -302,6 +302,11 @@ def decide( return Decision(run_cpu, True, why, families) +def is_fork(event: str, head_repo: str, repo: str) -> bool: + """A pull request from another repository, or from one since deleted.""" + return event == "pull_request" and head_repo != repo + + def list_pr_files(repo: str, number: str) -> list[str] | None: out = subprocess.run( [ @@ -360,7 +365,9 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--before", default="") parser.add_argument("--after", default="") parser.add_argument("--labels", default="[]", help="JSON list of label names") - parser.add_argument("--is-fork", default="false") + parser.add_argument( + "--head-repo", default="", help="owner/name of the pull request's head repository" + ) parser.add_argument( "--families", default="", help="Comma-separated family keys (workflow_dispatch)" ) @@ -386,7 +393,7 @@ def main(argv: list[str] | None = None) -> int: paths = list_push_files(args.repo, args.before, args.after) decision = decide( - args.event, paths, labels, args.is_fork == "true", requested, + args.event, paths, labels, is_fork(args.event, args.head_repo, args.repo), requested, config, matrix, pick_weighted, ) except ConfigError as e: diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index a58061afcf50..519467098efb 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -5,11 +5,13 @@ import copy import io +import os import sys import tempfile import unittest from contextlib import redirect_stdout from pathlib import Path +from unittest import mock import yaml @@ -316,6 +318,40 @@ def test_unknown_event(self): self.assertFalse(d.run_gpu) +class ForkTest(unittest.TestCase): + REPO = "ROCm/rocm-systems" + + def test_same_repository_is_not_a_fork(self): + self.assertFalse(cfg.is_fork("pull_request", self.REPO, self.REPO)) + + def test_other_repository_is_a_fork(self): + self.assertTrue(cfg.is_fork("pull_request", "someone/rocm-systems", self.REPO)) + + def test_deleted_head_repository_is_a_fork(self): + self.assertTrue(cfg.is_fork("pull_request", "", self.REPO)) + + def test_only_pull_requests_are_forks(self): + self.assertFalse(cfg.is_fork("push", "", self.REPO)) + + def test_main_skips_gpu_for_a_deleted_head_repository(self): + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(cfg, "list_pr_files", return_value=["projects/hrr/playback/x.cpp"]), \ + mock.patch.object(cfg, "load_matrix", return_value=(MATRIX, pick_first)), \ + mock.patch.dict(os.environ, {"GITHUB_OUTPUT": ""}): + path = Path(tmp) / "ci.yaml" + path.write_text(yaml.safe_dump(config_data())) + out = io.StringIO() + with redirect_stdout(out): + status = cfg.main( + ["--event", "pull_request", "--repo", self.REPO, "--pr-number", "1", + "--head-repo", "", "--therock-dir", tmp, "--config", str(path)] + ) + self.assertEqual(status, 0) + self.assertIn("run_cpu=true\n", out.getvalue()) + self.assertIn("run_gpu=false\n", out.getvalue()) + self.assertIn("fork", out.getvalue()) + + class OutputTest(unittest.TestCase): def test_write_outputs(self): d = cfg.Decision(True, True, "why", [{"family": "gfx94x", "runner": "r", "gfx_targets": "gfx942"}]) From 651f3f56c2a4621725cae49e103dcb84bb021087 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 16:21:39 +0200 Subject: [PATCH 18/89] ci(hrr): do not restart HRR CI for labels that change nothing The workflow runs on labeled so that ci:hrr-capture and ci:gfx... take effect at once. Every labeled event shared the pull request's concurrency group, so adding any label, say documentation, cancelled the run in progress and started all GPU families again. A labeled event for any other label now gets a concurrency group of its own, keyed by the label, so it cancels nothing, and the configure job decides that it runs nothing. The workflow's expression and the script's label_matters() make the same test, and a script test checks that the workflow names the label ci.yaml configures. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 17 ++++++++-- projects/hrr/tests/README.md | 3 +- .../hrr/tests/scripts/hrr_ci_configure.py | 19 ++++++++++- .../tests/scripts/test_hrr_ci_configure.py | 34 +++++++++++++++++++ 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index cc495871576b..965e40dbceb8 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -28,8 +28,16 @@ on: permissions: contents: read +# Adding a label that cannot change what runs (label_matters() in +# hrr_ci_configure.py makes the same test) gets a group of its own, so it does +# not cancel the run in progress; configure then runs nothing for it. concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + group: >- + ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}${{ + github.event.action == 'labeled' + && github.event.label.name != 'ci:hrr-capture' + && !startsWith(github.event.label.name, 'ci:gfx') + && format('-label-{0}', github.event.label.name) || '' }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: @@ -98,6 +106,8 @@ jobs: BEFORE: ${{ github.event.before }} AFTER: ${{ github.sha }} LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }} + ACTION: ${{ github.event.action }} + LABEL: ${{ github.event.label.name }} # Empty when the head repository was deleted, which counts as a fork. HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} FAMILIES: ${{ inputs.families }} @@ -105,7 +115,7 @@ jobs: python projects/hrr/tests/scripts/hrr_ci_configure.py --event "${EVENT}" --repo "${REPO}" --pr-number "${PR_NUMBER}" --before "${BEFORE}" --after "${AFTER}" --labels "${LABELS}" - --head-repo "${HEAD_REPO}" --families "${FAMILIES}" + --action "${ACTION}" --label "${LABEL}" --head-repo "${HEAD_REPO}" --families "${FAMILIES}" --therock-dir therock codegen: @@ -150,10 +160,11 @@ jobs: with: persist-credentials: false # Cone mode also checks out projects/clr/hipamd/src/amdhip.def.in, - # which gen_hrr_api_args.py reads on import. + # which gen_hrr_api_args.py reads on import. A test reads hrr-ci.yml. sparse-checkout: | projects/hrr projects/clr/hipamd/src/hrr + .github/workflows - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index e1444eda2c2e..60ed28bf06d6 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -141,7 +141,8 @@ applies (`scripts/hrr_ci_configure.py`): jobs. `paths.capture` also runs the capture-enabled build and the GPU suite. `paths.excluded` counts for neither. - `capture_label` (`ci:hrr-capture`) runs the GPU suite on a pull request that touches - no capture path. + no capture path. Adding it or a `ci:gfx...` label restarts the run; adding any + other label leaves the run in progress alone. - `families`: the GPU families the suite runs on, as TheRock family keys, on the runner TheRock's family matrix names. By default each runs on every event (`pull_request`, `push`, `schedule`, `workflow_dispatch`). Per family, `enabled: false` leaves it to diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index 0f803a245498..a416d3d8b260 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -201,6 +201,13 @@ def labelled_families(labels: Iterable[str], matrix: dict) -> list[str]: return wanted +def label_matters(label: str, config: Config) -> bool: + """Whether adding this label can change what runs. hrr-ci.yml's concurrency + group makes the same test, so that other labels do not cancel a run.""" + label = label.strip().lower() + return label == config.capture_label.lower() or label.startswith("ci:gfx") + + def runner_for( linux: dict, pick_weighted: Callable[[list, str], str], multi_gpu: bool = False ) -> str: @@ -269,10 +276,17 @@ def decide( config: Config, matrix: dict, pick_weighted: Callable[[list, str], str], + added_label: str | None = None, ) -> Decision: - """paths is None when the change set could not be listed in full.""" + """paths is None when the change set could not be listed in full. + + added_label is the label a labeled event added; the run for any other + pull request event already covers a label that cannot change what runs. + """ if event not in EVENTS: return Decision(False, False, f"event {event} does not run HRR CI") + if added_label is not None and not label_matters(added_label, config): + return Decision(False, False, f"the label {added_label} does not change what HRR CI runs") if event in ("schedule", "workflow_dispatch"): run_cpu, capture, why = True, True, f"{event} run" @@ -365,6 +379,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--before", default="") parser.add_argument("--after", default="") parser.add_argument("--labels", default="[]", help="JSON list of label names") + parser.add_argument("--action", default="", help="The pull_request event's action") + parser.add_argument("--label", default="", help="The label a labeled event added") parser.add_argument( "--head-repo", default="", help="owner/name of the pull request's head repository" ) @@ -395,6 +411,7 @@ def main(argv: list[str] | None = None) -> int: decision = decide( args.event, paths, labels, is_fork(args.event, args.head_repo, args.repo), requested, config, matrix, pick_weighted, + added_label=args.label if args.action == "labeled" else None, ) except ConfigError as e: print(f"::error::{e}") diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 519467098efb..3676e9b0571c 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -318,6 +318,40 @@ def test_unknown_event(self): self.assertFalse(d.run_gpu) +class LabelEventTest(unittest.TestCase): + CAPTURE = ["projects/hrr/playback/x.cpp"] + WORKFLOW = Path(__file__).resolve().parents[4] / ".github" / "workflows" / "hrr-ci.yml" + + def test_which_labels_matter(self): + for label in ("ci:hrr-capture", "CI:HRR-Capture", "ci:gfx110x", "ci:gfx110X-all"): + self.assertTrue(cfg.label_matters(label, REPO), label) + for label in ("documentation", "ci:skip", "hrr"): + self.assertFalse(cfg.label_matters(label, REPO), label) + + def test_other_label_runs_nothing(self): + d = cfg.decide( + "pull_request", self.CAPTURE, ["ci:hrr-capture", "documentation"], False, [], + CONFIG, MATRIX, pick_first, added_label="documentation", + ) + self.assertFalse(d.run_cpu) + self.assertFalse(d.run_gpu) + self.assertIn("documentation", d.reason) + + def test_capture_or_family_label_reruns(self): + for label in ("ci:hrr-capture", "ci:gfx110x"): + d = cfg.decide( + "pull_request", self.CAPTURE, [label], False, [], + CONFIG, MATRIX, pick_first, added_label=label, + ) + self.assertTrue(d.run_gpu, label) + + def test_concurrency_group_makes_the_same_test(self): + text = self.WORKFLOW.read_text(encoding="utf-8") + group = text[text.index("concurrency:"):text.index("cancel-in-progress:")] + self.assertIn(f"github.event.label.name != '{REPO.capture_label}'", group) + self.assertIn("!startsWith(github.event.label.name, 'ci:gfx')", group) + + class ForkTest(unittest.TestCase): REPO = "ROCm/rocm-systems" From d4b7e2536aaa31d55a841c23b8b047900265c369 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 16:24:35 +0200 Subject: [PATCH 19/89] ci(hrr): check that the suite is built for every family's GPU targets The configure job passed each family's fetch-gfx-targets to the GPU workflow, which never read them. Meanwhile nothing checked that the integration binary, built for targets.arch of integration.yaml, has code for the GPUs a family's runners have: gfx110x's include gfx1103, which targets.arch lacked. The configure job now fails unless targets.arch covers every target TheRock lists for every family in ci.yaml, disabled ones included since a label can request them; a family that names a single target, such as gfx1150, counts as that target. gfx1103 is added to targets.arch of both suite configs, and the unused gfx_targets field is dropped. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 1 + .github/workflows/hrr-gpu.yml | 2 +- projects/hrr/tests/README.md | 4 +- projects/hrr/tests/config/ci.yaml | 3 +- projects/hrr/tests/config/integration.yaml | 2 +- projects/hrr/tests/config/unit.yaml | 2 +- .../hrr/tests/scripts/hrr_ci_configure.py | 56 ++++++++++++++++--- .../tests/scripts/test_hrr_ci_configure.py | 35 ++++++++++-- 8 files changed, 86 insertions(+), 19 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 965e40dbceb8..70a1813fa0d7 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -66,6 +66,7 @@ jobs: sparse-checkout: | projects/hrr/tests/scripts/hrr_ci_configure.py projects/hrr/tests/config/ci.yaml + projects/hrr/tests/config/integration.yaml sparse-checkout-cone-mode: false - name: Checkout TheRock family matrix diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 44b2f94223b1..cd0ad4196af0 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -13,7 +13,7 @@ on: workflow_call: inputs: families: - description: JSON list of {family, runner, gfx_targets} from hrr_ci_configure.py + description: JSON list of {family, runner} from hrr_ci_configure.py type: string required: true ignore_test_config: diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index 60ed28bf06d6..8b3b86526e2a 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -155,7 +155,9 @@ applies (`scripts/hrr_ci_configure.py`): name a disabled family. - `rocm_sdk_version`: the ROCm SDK the unit job installs. -An unknown key, trigger or family fails the configure job. To take load off the GPU +An unknown key, trigger or family fails the configure job, and so does a family whose +GPU targets in TheRock's family matrix are not all in `targets.arch` of +`config/integration.yaml`. To take load off the GPU pools, disable families or narrow their triggers here. ## Configuring where a case runs diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index d0dd7c34ccf3..9ed00440a4cf 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -54,7 +54,8 @@ capture_label: ci:hrr-capture # The GPU families the suite runs on, as TheRock family keys # (build_tools/github_actions/amdgpu_family_matrix.py). The integration binary -# is built for targets.arch in integration.yaml, which must cover them. +# is built for targets.arch in integration.yaml; the configure job fails unless +# that covers every GPU target TheRock lists for each family here. # # Per family, all optional: # enabled false: run only when requested by a ci:gfx... label or the diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 4c367900ac74..62235d7262f6 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -15,7 +15,7 @@ binary: hrr-integration-tests select: "[hrr]~[direct]" targets: os: [linux, windows] - arch: [gfx90a, gfx942, gfx950, gfx1030, gfx1100, gfx1101, gfx1102, gfx1150, gfx1151, gfx1153, gfx1200, gfx1201, gfx1250] + arch: [gfx90a, gfx942, gfx950, gfx1030, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1153, gfx1200, gfx1201, gfx1250] cases: Unit_HRR_AllApisRoundtrip: {} Unit_HRR_AllApis_Direct: {} diff --git a/projects/hrr/tests/config/unit.yaml b/projects/hrr/tests/config/unit.yaml index 58621b8259ac..7ee09c4e287c 100644 --- a/projects/hrr/tests/config/unit.yaml +++ b/projects/hrr/tests/config/unit.yaml @@ -15,7 +15,7 @@ binary: hrr-unit-tests select: "*" targets: os: [linux, windows] - arch: [gfx90a, gfx942, gfx950, gfx1030, gfx1100, gfx1101, gfx1102, gfx1150, gfx1151, gfx1153, gfx1200, gfx1201, gfx1250] + arch: [gfx90a, gfx942, gfx950, gfx1030, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1153, gfx1200, gfx1201, gfx1250] cases: Unit_HRR_AllocKind_FreeDispatch: {} Unit_HRR_Format_BadMagic: {} diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index a416d3d8b260..894f1b4a7590 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -7,8 +7,7 @@ run_cpu "true" when the generated-source, script and unit jobs run. run_gpu "true" when the capture-enabled build and the GPU suite run. - families JSON list of {"family", "runner", "gfx_targets"} for the - GPU suite. + families JSON list of {"family", "runner"} for the GPU suite. reason One line explaining the decision, shown in the summary. rocm_sdk_version The ROCm SDK the unit job installs. @@ -29,6 +28,7 @@ import argparse import json import os +import re import subprocess import sys from dataclasses import dataclass, field @@ -38,6 +38,8 @@ import yaml DEFAULT_CONFIG = Path(__file__).resolve().parent.parent / "config" / "ci.yaml" +# Its targets.arch is what the GPU suite is built for. +INTEGRATION_CONFIG = DEFAULT_CONFIG.with_name("integration.yaml") EVENTS = ("pull_request", "push", "schedule", "workflow_dispatch") @@ -154,6 +156,14 @@ def parse_config(data) -> Config: ) +def load_archs(path: Path) -> list[str]: + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + return list(_strings("targets.arch", data["targets"]["arch"])) + except (ConfigError, KeyError, TypeError) as e: + raise ConfigError(f"{path}: no targets.arch list ({e})") from None + + def load_config(path: Path) -> Config: try: return parse_config(yaml.safe_load(path.read_text(encoding="utf-8"))) @@ -170,6 +180,38 @@ def check_families(config: Config, matrix: dict) -> None: ) +def family_targets(linux: dict) -> list[str]: + """The GPU targets of a family's test machines, without feature suffixes. + + TheRock lists them as fetch-gfx-targets; a family that names a single + target (gfx1150) may leave that list empty. + """ + targets = [t.split(":")[0] for t in linux.get("fetch-gfx-targets", [])] + family = str(linux.get("family", "")) + if not targets and re.fullmatch(r"gfx[0-9a-f]+", family): + targets = [family] + return list(dict.fromkeys(targets)) + + +def check_targets(config: Config, matrix: dict, archs: Iterable[str]) -> None: + """The suite must be built for every target of every family ci.yaml lists, + enabled or not, since a label can request any of them.""" + archs = set(archs) + problems = [] + for key in config.families: + targets = family_targets(matrix[key].get("linux", {})) + missing = [t for t in targets if t not in archs] + if not targets: + problems.append(f"TheRock lists no GPU targets for {key}") + elif missing: + problems.append(f"{key} needs {', '.join(missing)}") + if problems: + raise ConfigError( + f"targets.arch of {INTEGRATION_CONFIG.name} does not cover the families: " + + "; ".join(problems) + ) + + def _matches(path: str, prefixes: Iterable[str]) -> bool: return any(path == p or path.startswith(p) for p in prefixes) @@ -257,13 +299,7 @@ def select_families( if not runner: no_runner.append(key) continue - families.append( - { - "family": key, - "runner": runner, - "gfx_targets": ",".join(linux.get("fetch-gfx-targets", [])), - } - ) + families.append({"family": key, "runner": runner}) return families, no_runner @@ -389,6 +425,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) parser.add_argument("--therock-dir", type=Path, required=True) parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--integration-config", type=Path, default=INTEGRATION_CONFIG) return parser.parse_args(argv) @@ -401,6 +438,7 @@ def main(argv: list[str] | None = None) -> int: config = load_config(args.config) matrix, pick_weighted = load_matrix(args.therock_dir) check_families(config, matrix) + check_targets(config, matrix, load_archs(args.integration_config)) paths: list[str] | None = [] if args.event == "pull_request" and args.pr_number: diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 3676e9b0571c..8a6b578975df 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -205,6 +205,33 @@ def test_family_missing_from_matrix(self): cfg.check_families(config, MATRIX) self.assertIn("gfx999 not in TheRock's family matrix", str(caught.exception)) + def test_family_targets(self): + self.assertEqual(cfg.family_targets(MATRIX["gfx110x"]["linux"]), ["gfx1100", "gfx1101"]) + # gfx1150 lists no fetch targets; its family name is the target. + self.assertEqual(cfg.family_targets(MATRIX["gfx1150"]["linux"]), ["gfx1150"]) + self.assertEqual( + cfg.family_targets({"fetch-gfx-targets": ["gfx942", "gfx942:xnack+"]}), ["gfx942"] + ) + self.assertEqual(cfg.family_targets({"family": "gfx101X-dgpu"}), []) + + def test_targets_cover_every_family(self): + archs = ["gfx90a", "gfx942", "gfx950", "gfx1100", "gfx1101", "gfx1150", "gfx1250"] + cfg.check_targets(CONFIG, MATRIX, archs) + with self.assertRaises(cfg.ConfigError) as caught: + cfg.check_targets(CONFIG, MATRIX, [a for a in archs if a not in ("gfx1101", "gfx1250")]) + # gfx125x is disabled, but a label can still request it. + self.assertIn("gfx110x needs gfx1101; gfx125x needs gfx1250", str(caught.exception)) + + def test_family_without_targets_is_an_error(self): + config = cfg.parse_config(config_data(families={"gfx94x": {}})) + matrix = {"gfx94x": {"linux": {"family": "gfx94X-dcgpu"}}} + with self.assertRaises(cfg.ConfigError) as caught: + cfg.check_targets(config, matrix, ["gfx942"]) + self.assertIn("no GPU targets for gfx94x", str(caught.exception)) + + def test_repo_integration_config_lists_archs(self): + self.assertIn("gfx1103", cfg.load_archs(cfg.INTEGRATION_CONFIG)) + def test_load_error_names_the_file(self): with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "ci.yaml" @@ -244,8 +271,7 @@ def test_capture_pull_request_runs_enabled_families(self): # gfx110x is not triggered by pull requests, gfx125x is disabled and # gfx1150 has no runner. self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950"]) - self.assertEqual(d.families[1]["runner"], "gpu-94x-a") - self.assertEqual(d.families[1]["gfx_targets"], "gfx942") + self.assertEqual(d.families[1], {"family": "gfx94x", "runner": "gpu-94x-a"}) self.assertIn("no runner for gfx1150", d.reason) def test_rename_out_of_capture_counts(self): @@ -300,7 +326,6 @@ def test_runner_override(self): config = cfg.parse_config(config_data(families={"gfx94x": {"runner": "my-gpu"}})) d = decide("schedule", [], config=config) self.assertEqual(d.families[0]["runner"], "my-gpu") - self.assertEqual(d.families[0]["gfx_targets"], "gfx942") def test_dispatch_families_replace_the_selection(self): d = decide("workflow_dispatch", [], requested=["gfx125x", "gfx1150"]) @@ -388,14 +413,14 @@ def test_main_skips_gpu_for_a_deleted_head_repository(self): class OutputTest(unittest.TestCase): def test_write_outputs(self): - d = cfg.Decision(True, True, "why", [{"family": "gfx94x", "runner": "r", "gfx_targets": "gfx942"}]) + d = cfg.Decision(True, True, "why", [{"family": "gfx94x", "runner": "r"}]) with tempfile.TemporaryDirectory() as tmp: out = Path(tmp) / "out" with redirect_stdout(io.StringIO()): cfg.write_outputs(d, "1.2.3", str(out)) text = out.read_text() self.assertIn("run_cpu=true\n", text) - self.assertIn('families=[{"family":"gfx94x","runner":"r","gfx_targets":"gfx942"}]\n', text) + self.assertIn('families=[{"family":"gfx94x","runner":"r"}]\n', text) self.assertIn("reason=why\n", text) self.assertIn("rocm_sdk_version=1.2.3\n", text) From db321b4d42eb883c4de10af1d4d39bee66e40148 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 16:25:40 +0200 Subject: [PATCH 20/89] ci(hrr): make the suite configs agree on targets.arch again Before 59b1e0deb1d, lint compared each suite config's targets.arch with HRR_ARCHS in the workflow, which also kept unit.yaml and integration.yaml equal. With HRR_ARCHS gone, nothing did. Both lists are the architecture vocabulary of the skip rules, and integration.yaml also decides what the GPU suite is built for, so a rule written for one suite could name an architecture the other does not know. lint now fails when the configs it is given name different targets.arch sets; the script job already lints both together. Signed-off-by: Daniel Olsson --- projects/hrr/tests/scripts/hrr_test_config.py | 30 ++++++++++++++++++- .../hrr/tests/scripts/test_hrr_test_config.py | 23 ++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/scripts/hrr_test_config.py b/projects/hrr/tests/scripts/hrr_test_config.py index af872d451259..4244e4871096 100644 --- a/projects/hrr/tests/scripts/hrr_test_config.py +++ b/projects/hrr/tests/scripts/hrr_test_config.py @@ -15,7 +15,7 @@ ``kind: unsupported`` is permanent and needs only a reason. Usage: - hrr_test_config.py lint CONFIG [CONFIG ...] + hrr_test_config.py lint CONFIG [CONFIG ...] (their targets.arch must agree) hrr_test_config.py check --binary EXE --config CONFIG [--update] """ @@ -264,6 +264,29 @@ def lint(config: dict, cases: list[dict] | None = None) -> list[str]: return errors +def arch_mismatches(configs: dict[str, dict]) -> list[str]: + """The suites share one architecture vocabulary, so every ``targets.arch`` + must name the same set as the first config's.""" + archs = { + path: set(config["targets"]["arch"]) + for path, config in configs.items() + if isinstance(config.get("targets"), dict) + and isinstance(config["targets"].get("arch"), list) + } + if len(archs) < 2: + return [] + first, *rest = archs + errors = [] + for path in rest: + if archs[path] != archs[first]: + missing = ", ".join(sorted(archs[first] - archs[path])) or "-" + extra = ", ".join(sorted(archs[path] - archs[first])) or "-" + errors.append( + f"targets.arch of {path} differs from {first} (missing: {missing}; extra: {extra})" + ) + return errors + + def list_cases(binary: str, select: str = "*") -> list[dict]: """Every case ``select`` matches, hidden ones included when ``select`` is ``*``. @@ -424,6 +447,7 @@ def _print_errors(path: str, errors: list[str]) -> None: def _cmd_lint(args: argparse.Namespace) -> int: failed = False + loaded = {} for path in args.configs: try: config = load(path) @@ -431,12 +455,16 @@ def _cmd_lint(args: argparse.Namespace) -> int: print(f"FAIL: {exc}", file=sys.stderr) failed = True continue + loaded[path] = config errors = lint(config) if errors: _print_errors(path, errors) failed = True else: print(f"OK: {path}") + for error in arch_mismatches(loaded): + print(f"FAIL: {error}", file=sys.stderr) + failed = True return 1 if failed else 0 diff --git a/projects/hrr/tests/scripts/test_hrr_test_config.py b/projects/hrr/tests/scripts/test_hrr_test_config.py index 9aad89369ffb..96b3d634a5d2 100644 --- a/projects/hrr/tests/scripts/test_hrr_test_config.py +++ b/projects/hrr/tests/scripts/test_hrr_test_config.py @@ -266,6 +266,29 @@ def test_lint_command_fails_on_a_bad_file(self): self.assertEqual(rc, 1) self.assertIn("version", err.getvalue()) + def test_lint_command_fails_when_targets_arch_differ(self): + other = suite({"Runs": {}}) + other["targets"]["arch"] = ARCHS[:-1] + ["gfx1200"] + paths = [] + for config in (suite({"Runs": {}}), other): + tmp = tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False, encoding="utf-8") + self.addCleanup(lambda name=tmp.name: Path(name).unlink(missing_ok=True)) + tmp.write(cfg.render(config)) + tmp.close() + paths.append(tmp.name) + err = io.StringIO() + with contextlib.redirect_stderr(err), contextlib.redirect_stdout(io.StringIO()): + rc = cfg.main(["lint", *paths]) + self.assertEqual(rc, 1) + self.assertIn("missing: gfx1151; extra: gfx1200", err.getvalue()) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(cfg.main(["lint", paths[0], paths[0]]), 0) + + def test_repo_suites_share_targets_arch(self): + config_dir = Path(__file__).resolve().parent.parent / "config" + configs = {name: cfg.load(config_dir / name) for name in ("unit.yaml", "integration.yaml")} + self.assertEqual(cfg.arch_mismatches(configs), []) + if __name__ == "__main__": unittest.main() From 7a164f96c8a569f729de08ac37fc10b25c695776 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 16:26:55 +0200 Subject: [PATCH 21/89] ci(hrr): report a failed configure job in the HRR CI summary The summary ran only when configure said the CPU or GPU jobs should run. When configure itself failed, for a bad ci.yaml say, its outputs were empty, so the summary was skipped as well and the run had no HRR CI Summary result at all. The summary now also runs when configure did not succeed, says so, and fails. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 8 ++++++- projects/hrr/tests/scripts/hrr_ci_report.py | 21 ++++++++++++++----- .../hrr/tests/scripts/test_hrr_ci_report.py | 11 ++++++++++ 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 70a1813fa0d7..0e3b915ae690 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -291,7 +291,11 @@ jobs: summary: name: HRR CI Summary needs: [configure, codegen, scripts, unit, gpu] - if: ${{ always() && (needs.configure.outputs.run_cpu == 'true' || needs.configure.outputs.run_gpu == 'true') }} + # Also when configure failed: then nothing else ran, and this job fails. + if: >- + ${{ always() && (needs.configure.result != 'success' + || needs.configure.outputs.run_cpu == 'true' + || needs.configure.outputs.run_gpu == 'true') }} runs-on: ubuntu-24.04 permissions: actions: read @@ -324,10 +328,12 @@ jobs: - name: Summarize env: GH_TOKEN: ${{ github.token }} + CONFIGURE_RESULT: ${{ needs.configure.result }} REASON: ${{ needs.configure.outputs.reason }} RUN_GPU: ${{ needs.configure.outputs.run_gpu }} FAMILIES: ${{ needs.configure.outputs.families }} run: >- python3 projects/hrr/tests/scripts/hrr_ci_report.py + --configure-result "${CONFIGURE_RESULT}" --reason "${REASON}" --run-gpu "${RUN_GPU}" --families "${FAMILIES}" --results-dir results --base-json base/base.json diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index d8a9308dd0d1..f6fc95063d2b 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -10,8 +10,8 @@ produced results (cancelled, timed out, still queued) still has a row; * the per-suite results table from summarize_junit.py. -Exits 1 when any HRR job failed or was cancelled, or a results file could not -be read. +Exits 1 when the configure job did not succeed, any HRR job failed or was +cancelled, or a results file could not be read. """ from __future__ import annotations @@ -60,7 +60,15 @@ def missing_families(families: list[dict], rows: list[dict]) -> list[str]: return [f["family"] for f in families if f["family"] not in seen] -def render_header(reason: str, run_gpu: bool, families: list[dict], base: dict | None) -> str: +def render_header( + reason: str, run_gpu: bool, families: list[dict], base: dict | None, + configure_result: str = "success", +) -> str: + if configure_result != "success": + return ( + f"## HRR CI\n\n- Configure: {configure_result}, so nothing else ran;" + " see the Configure job's log.\n\n" + ) lines = ["## HRR CI", "", f"- Scope: {reason}."] if run_gpu: names = ", ".join(f["family"] for f in families) or "none" @@ -114,6 +122,7 @@ def list_jobs(repo: str, run_id: str, attempt: str) -> list[dict]: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--configure-result", default="success") parser.add_argument("--reason", default="") parser.add_argument("--run-gpu", default="false") parser.add_argument("--families", default="[]") @@ -133,7 +142,9 @@ def main(argv: list[str] | None = None) -> int: rows = classify_jobs(list_jobs(args.repo, args.run_id, args.run_attempt)) missing = missing_families(families, rows) - markdown = render_header(args.reason, run_gpu, families, base) + render_jobs(rows, missing) + configured = args.configure_result == "success" + markdown = render_header(args.reason, run_gpu, families, base, args.configure_result) + markdown += render_jobs(rows, missing) errors: list[str] = [] args.results_dir.mkdir(parents=True, exist_ok=True) markdown += summarize_junit.render(args.results_dir, errors) @@ -145,7 +156,7 @@ def main(argv: list[str] | None = None) -> int: out.write(markdown) for error in errors: print(f"::error::unreadable JUnit XML {error}") - return 0 if overall_ok(rows, missing) and not errors else 1 + return 0 if configured and overall_ok(rows, missing) and not errors else 1 if __name__ == "__main__": diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index 35136dd18e40..fa75508a852b 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -91,6 +91,17 @@ def test_main_appends_results_table(self): self.assertEqual(rc, 0) self.assertIn("## HRR results by platform", out.getvalue()) + def test_main_fails_when_configure_did_not_succeed(self): + for result in ("failure", "cancelled"): + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(rpt, "list_jobs", return_value=JOBS[:1]), \ + mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ + contextlib.redirect_stdout(io.StringIO()) as out: + rc = rpt.main(["--configure-result", result, "--run-gpu", "", "--families", "", + "--results-dir", tmp]) + self.assertEqual(rc, 1, result) + self.assertIn(f"Configure: {result}, so nothing else ran", out.getvalue()) + if __name__ == "__main__": unittest.main() From 225ebc65880c0cb79aa128704910b381e638a4b9 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 16:30:20 +0200 Subject: [PATCH 22/89] ci(hrr): give every ci.yaml family a row in the HRR CI summary summarize_junit.py had its own list of families, which had drifted from ci.yaml: gfx125x had no row, so its results never showed, and families that were never scheduled showed as "no results" as if their job had failed to upload. The configure job now also outputs the ci.yaml families it did not schedule, each with the reason (disabled, not on this event, no runner label, not requested). The summary's results table has a row for every scheduled family and one for each of those, saying why it did not run. The header names each family's runner, and the table notes that Skip includes the cases that need two GPUs on a single-GPU runner. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 3 ++ .../hrr/tests/scripts/hrr_ci_configure.py | 29 +++++++++--- projects/hrr/tests/scripts/hrr_ci_report.py | 16 +++++-- projects/hrr/tests/scripts/summarize_junit.py | 45 +++++++++++-------- .../tests/scripts/test_hrr_ci_configure.py | 15 ++++++- .../hrr/tests/scripts/test_hrr_ci_report.py | 20 ++++++++- .../hrr/tests/scripts/test_summarize_junit.py | 21 +++++++-- 7 files changed, 113 insertions(+), 36 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 0e3b915ae690..8a7d32b703cc 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -56,6 +56,7 @@ jobs: run_cpu: ${{ steps.decide.outputs.run_cpu }} run_gpu: ${{ steps.decide.outputs.run_gpu }} families: ${{ steps.decide.outputs.families }} + skipped_families: ${{ steps.decide.outputs.skipped_families }} reason: ${{ steps.decide.outputs.reason }} rocm_sdk_version: ${{ steps.decide.outputs.rocm_sdk_version }} steps: @@ -332,8 +333,10 @@ jobs: REASON: ${{ needs.configure.outputs.reason }} RUN_GPU: ${{ needs.configure.outputs.run_gpu }} FAMILIES: ${{ needs.configure.outputs.families }} + SKIPPED: ${{ needs.configure.outputs.skipped_families }} run: >- python3 projects/hrr/tests/scripts/hrr_ci_report.py --configure-result "${CONFIGURE_RESULT}" --reason "${REASON}" --run-gpu "${RUN_GPU}" --families "${FAMILIES}" + --skipped-families "${SKIPPED}" --results-dir results --base-json base/base.json diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index 894f1b4a7590..8afc11fbe97b 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -8,6 +8,8 @@ run_cpu "true" when the generated-source, script and unit jobs run. run_gpu "true" when the capture-enabled build and the GPU suite run. families JSON list of {"family", "runner"} for the GPU suite. + skipped_families JSON list of {"family", "reason"}: the other ci.yaml + families, when the GPU suite runs. reason One line explaining the decision, shown in the summary. rocm_sdk_version The ROCm SDK the unit job installs. @@ -47,6 +49,8 @@ # is treated as touching everything rather than guessed at. MAX_LISTED_FILES = 3000 +NO_RUNNER = "no runner label in TheRock's runner config" + class ConfigError(ValueError): pass @@ -76,6 +80,7 @@ class Decision: run_gpu: bool reason: str families: list[dict] = field(default_factory=list) + skipped: list[dict] = field(default_factory=list) def _mapping(where: str, value, required: Iterable[str], optional: Iterable[str] = ()) -> dict: @@ -272,8 +277,9 @@ def select_families( labels: list[str], matrix: dict, pick_weighted: Callable[[list, str], str], -) -> tuple[list[dict], list[str]]: - """GPU families to test, in ci.yaml order, and the chosen ones without a runner.""" +) -> tuple[list[dict], list[dict]]: + """GPU families to test and the other ci.yaml families with the reason, in + ci.yaml order.""" if requested: unknown = [key for key in requested if key not in config.families] if unknown: @@ -289,18 +295,25 @@ def select_families( } chosen |= set(labelled_families(labels, matrix)) & set(config.families) - families, no_runner = [], [] + families, skipped = [], [] for key, family in config.families.items(): if key not in chosen: + if requested: + reason = "not requested" + elif not family.enabled: + reason = "disabled in ci.yaml" + else: + reason = f"not run on {event} (ci.yaml triggers)" + skipped.append({"family": key, "reason": reason}) continue linux = matrix.get(key, {}).get("linux", {}) multi_gpu = event in family.multi_gpu_triggers runner = family.runner or runner_for(linux, pick_weighted, multi_gpu) if not runner: - no_runner.append(key) + skipped.append({"family": key, "reason": NO_RUNNER}) continue families.append({"family": key, "runner": runner}) - return families, no_runner + return families, skipped def decide( @@ -344,12 +357,13 @@ def decide( if is_fork: return Decision(run_cpu, False, why + "; GPU suite skipped for a fork pull request") - families, no_runner = select_families(event, config, requested, labels, matrix, pick_weighted) + families, skipped = select_families(event, config, requested, labels, matrix, pick_weighted) + no_runner = [s["family"] for s in skipped if s["reason"] == NO_RUNNER] if no_runner: why += f"; no runner for {', '.join(no_runner)}" if not families: return Decision(run_cpu, False, why + "; no GPU family with a runner was selected") - return Decision(run_cpu, True, why, families) + return Decision(run_cpu, True, why, families, skipped) def is_fork(event: str, head_repo: str, repo: str) -> bool: @@ -397,6 +411,7 @@ def write_outputs(decision: Decision, rocm_sdk_version: str, output_path: str | f"run_cpu={'true' if decision.run_cpu else 'false'}", f"run_gpu={'true' if decision.run_gpu else 'false'}", f"families={json.dumps(decision.families, separators=(',', ':'))}", + f"skipped_families={json.dumps(decision.skipped, separators=(',', ':'))}", f"reason={decision.reason}", f"rocm_sdk_version={rocm_sdk_version}", ] diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index f6fc95063d2b..f49bec344f61 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -8,7 +8,8 @@ * why this run tested what it did, and which base build the GPU suite used; * the conclusion of every unit, build and GPU job, so a job that never produced results (cancelled, timed out, still queued) still has a row; - * the per-suite results table from summarize_junit.py. + * the per-suite results table from summarize_junit.py, with a row for every + ci.yaml family, saying why when one did not run. Exits 1 when the configure job did not succeed, any HRR job failed or was cancelled, or a results file could not be read. @@ -71,8 +72,11 @@ def render_header( ) lines = ["## HRR CI", "", f"- Scope: {reason}."] if run_gpu: - names = ", ".join(f["family"] for f in families) or "none" - lines.append(f"- GPU families: {names}.") + names = ", ".join( + f"{f['family']} (`{f['runner']}`)" if f.get("runner") else f["family"] + for f in families + ) or "none" + lines.append(f"- GPU families and runners: {names}.") if base: lines.append( f"- Base ROCm artifacts: [{base['repo']} run {base['run_id']}]({base['run_url']})" @@ -126,6 +130,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--reason", default="") parser.add_argument("--run-gpu", default="false") parser.add_argument("--families", default="[]") + parser.add_argument("--skipped-families", default="[]") parser.add_argument("--results-dir", type=Path, default=Path("results")) parser.add_argument("--base-json", type=Path) parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "")) @@ -135,6 +140,7 @@ def main(argv: list[str] | None = None) -> int: run_gpu = args.run_gpu == "true" families = json.loads(args.families or "[]") if run_gpu else [] + skipped = json.loads(args.skipped_families or "[]") if run_gpu else [] base = None if args.base_json and args.base_json.is_file(): base = json.loads(args.base_json.read_text()) @@ -147,7 +153,9 @@ def main(argv: list[str] | None = None) -> int: markdown += render_jobs(rows, missing) errors: list[str] = [] args.results_dir.mkdir(parents=True, exist_ok=True) - markdown += summarize_junit.render(args.results_dir, errors) + table_families = [(f["family"], "") for f in families] + table_families += [(s["family"], s["reason"]) for s in skipped] + markdown += summarize_junit.render(args.results_dir, errors, table_families) sys.stdout.write(markdown) summary = os.environ.get("GITHUB_STEP_SUMMARY") diff --git a/projects/hrr/tests/scripts/summarize_junit.py b/projects/hrr/tests/scripts/summarize_junit.py index a183e7430c02..4fe32e391c2f 100644 --- a/projects/hrr/tests/scripts/summarize_junit.py +++ b/projects/hrr/tests/scripts/summarize_junit.py @@ -19,19 +19,10 @@ import xml.etree.ElementTree as ET from pathlib import Path -# Display order matches .github/workflows/hrr-ci.yml. -COLUMNS = ( +# The unit job's matrix in .github/workflows/hrr-ci.yml. +UNIT_COLUMNS = ( ("unit-ubuntu-24.04.xml", "Unit", "Linux"), ("unit-windows-2022.xml", "Unit", "Windows"), - ("integration-gfx90a.xml", "Integration", "gfx90a"), - ("integration-gfx94x.xml", "Integration", "gfx94x"), - ("integration-gfx950.xml", "Integration", "gfx950"), - ("integration-gfx103x.xml", "Integration", "gfx103x"), - ("integration-gfx110x.xml", "Integration", "gfx110x"), - ("integration-gfx120x.xml", "Integration", "gfx120x"), - ("integration-gfx1151.xml", "Integration", "gfx1151"), - ("integration-gfx1150.xml", "Integration", "gfx1150"), - ("integration-gfx1153.xml", "Integration", "gfx1153"), ) MAX_FAILED_NAMES = 8 @@ -114,19 +105,36 @@ def failed_cell(names: list[str]) -> str: return text -def render(directory: Path, errors: list[str]) -> str: +def render( + directory: Path, errors: list[str], families: list[tuple[str, str]] | None = None +) -> str: + """families: (family, why it did not run, or "") per integration row. By + default, one row per integration-.xml present.""" + if families is None: + families = [ + (path.stem[len("integration-"):], "") + for path in sorted(directory.glob("integration-*.xml")) + ] + columns = [*UNIT_COLUMNS] + not_run = {} + for family, why in families: + columns.append((f"integration-{family}.xml", "Integration", family)) + not_run[family] = why + rows: list[tuple[str, str, dict[str, object] | None]] = [] - for filename, suite, platform in COLUMNS: + for filename, suite, platform in columns: path = directory / filename if path.is_file(): rows.append((suite, platform, try_summarize(path, errors))) + elif not_run.get(platform): + rows.append((suite, platform, {"not_run": not_run[platform]})) else: rows.append((suite, platform, None)) extra = sorted( path.name for path in directory.glob("*.xml") - if path.name not in {filename for filename, _, _ in COLUMNS} + if path.name not in {filename for filename, _, _ in columns} ) for filename in extra: rows.append(("Other", filename, try_summarize(directory / filename, errors))) @@ -141,6 +149,9 @@ def render(directory: Path, errors: list[str]) -> str: if data is None: lines.append(f"| {suite} | {platform} | — | — | — | — | _no results_ |") continue + if "not_run" in data: + lines.append(f"| {suite} | {platform} | — | — | — | — | _not run: {data['not_run']}_ |") + continue if "error" in data: lines.append( f"| {suite} | {platform} | — | — | — | — | _unreadable XML: {data['error']}_ |" @@ -162,7 +173,7 @@ def render(directory: Path, errors: list[str]) -> str: lines.extend( [ "", - "Counts are top-level Catch2 cases. Missing XML means that job did not upload results (cancelled or not started). Unreadable XML is usually a job killed mid-run; its results are incomplete.", + "Counts are top-level Catch2 cases. Skip is a case that skipped itself, such as the cases that need two GPUs on a single-GPU runner; Config skip is a case a rule in the suite config skipped. No results means that job did not upload results (cancelled or not started). Unreadable XML is usually a job killed mid-run; its results are incomplete.", "", "### Counts grid", "", @@ -176,9 +187,7 @@ def render(directory: Path, errors: list[str]) -> str: ("Config skip", "CONFIG")): values = [] for _, _, data in rows: - values.append( - cell(None if data is None or "error" in data else data["counts"][key]) - ) + values.append(cell(data["counts"][key] if data and "counts" in data else None)) lines.append("| " + " | ".join([label, *values]) + " |") return "\n".join(lines) + "\n" diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 8a6b578975df..067ec184faf7 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -273,6 +273,14 @@ def test_capture_pull_request_runs_enabled_families(self): self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950"]) self.assertEqual(d.families[1], {"family": "gfx94x", "runner": "gpu-94x-a"}) self.assertIn("no runner for gfx1150", d.reason) + self.assertEqual( + d.skipped, + [ + {"family": "gfx110x", "reason": "not run on pull_request (ci.yaml triggers)"}, + {"family": "gfx1150", "reason": cfg.NO_RUNNER}, + {"family": "gfx125x", "reason": "disabled in ci.yaml"}, + ], + ) def test_rename_out_of_capture_counts(self): # The files API lists previous_filename alongside filename. @@ -331,6 +339,7 @@ def test_dispatch_families_replace_the_selection(self): d = decide("workflow_dispatch", [], requested=["gfx125x", "gfx1150"]) self.assertEqual(family_keys(d), ["gfx125x"]) self.assertIn("no runner for gfx1150", d.reason) + self.assertIn({"family": "gfx94x", "reason": "not requested"}, d.skipped) def test_dispatch_unknown_family(self): with self.assertRaises(cfg.ConfigError) as caught: @@ -413,7 +422,10 @@ def test_main_skips_gpu_for_a_deleted_head_repository(self): class OutputTest(unittest.TestCase): def test_write_outputs(self): - d = cfg.Decision(True, True, "why", [{"family": "gfx94x", "runner": "r"}]) + d = cfg.Decision( + True, True, "why", [{"family": "gfx94x", "runner": "r"}], + [{"family": "gfx125x", "reason": "disabled in ci.yaml"}], + ) with tempfile.TemporaryDirectory() as tmp: out = Path(tmp) / "out" with redirect_stdout(io.StringIO()): @@ -421,6 +433,7 @@ def test_write_outputs(self): text = out.read_text() self.assertIn("run_cpu=true\n", text) self.assertIn('families=[{"family":"gfx94x","runner":"r"}]\n', text) + self.assertIn('skipped_families=[{"family":"gfx125x","reason":"disabled in ci.yaml"}]\n', text) self.assertIn("reason=why\n", text) self.assertIn("rocm_sdk_version=1.2.3\n", text) diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index fa75508a852b..52f0a6cc277a 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -69,8 +69,9 @@ def test_missing_family_fails(self): class RenderTest(unittest.TestCase): def test_header_names_base(self): base = {"repo": "ROCm/rocm-systems", "run_id": "7", "run_url": "https://r/7", "commit": "0123456789abcdef"} - text = rpt.render_header("the change touches capture or replay", True, FAMILIES, base) - self.assertIn("GPU families: gfx94x, gfx950, gfx110x.", text) + families = [{"family": "gfx94x", "runner": "linux-gfx942-1gpu"}, {"family": "gfx950"}] + text = rpt.render_header("the change touches capture or replay", True, families, base) + self.assertIn("GPU families and runners: gfx94x (`linux-gfx942-1gpu`), gfx950.", text) self.assertIn("[ROCm/rocm-systems run 7](https://r/7) at `0123456789ab`", text) def test_header_without_gpu(self): @@ -91,6 +92,21 @@ def test_main_appends_results_table(self): self.assertEqual(rc, 0) self.assertIn("## HRR results by platform", out.getvalue()) + def test_main_gives_skipped_families_a_row(self): + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(rpt, "list_jobs", return_value=JOBS[5:6]), \ + mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ + contextlib.redirect_stdout(io.StringIO()) as out: + rc = rpt.main([ + "--run-gpu", "true", "--results-dir", tmp, + "--families", '[{"family":"gfx94x","runner":"r"}]', + "--skipped-families", '[{"family":"gfx125x","reason":"disabled in ci.yaml"}]', + ]) + self.assertEqual(rc, 0) + self.assertIn("| Integration | gfx94x | — | — | — | — | _no results_ |", out.getvalue()) + self.assertIn("| Integration | gfx125x | — | — | — | — | _not run: disabled in ci.yaml_ |", + out.getvalue()) + def test_main_fails_when_configure_did_not_succeed(self): for result in ("failure", "cancelled"): with tempfile.TemporaryDirectory() as tmp, \ diff --git a/projects/hrr/tests/scripts/test_summarize_junit.py b/projects/hrr/tests/scripts/test_summarize_junit.py index 07bab8206249..e2a2d49a8089 100644 --- a/projects/hrr/tests/scripts/test_summarize_junit.py +++ b/projects/hrr/tests/scripts/test_summarize_junit.py @@ -130,18 +130,31 @@ def test_render_marks_missing_unreadable_and_extra(self): self.write("integration-gfx90a.xml", " Date: Wed, 14 Oct 2026 20:48:03 +0200 Subject: [PATCH 23/89] [hrr] CI: match the capture label case-insensitively; capture paths also run the host suite label_matters() and labelled_families() lowercase labels, the gate did not, so CI:HRR-Capture passed the gate and ran nothing. Paths that are only in paths.capture (hip_context.cpp, hip_code_object.*) gave run_cpu=false, run_gpu=true. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/scripts/hrr_ci_configure.py | 5 ++++- projects/hrr/tests/scripts/test_hrr_ci_configure.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index 8afc11fbe97b..9c963dd7a076 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -348,10 +348,13 @@ def decide( "the change touches HRR" if run_cpu else "the change does not touch HRR" ) - labelled = event == "pull_request" and config.capture_label in labels + labelled = event == "pull_request" and config.capture_label.lower() in {l.strip().lower() for l in labels} if labelled and not capture: run_cpu, capture, why = True, True, f"the pull request is labelled {config.capture_label}" + # Capture paths that the CPU list omits (hip_context.cpp, hip_code_object.*) + # still need the host unit suite. + run_cpu = run_cpu or capture if not capture: return Decision(run_cpu, False, why) if is_fork: diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 067ec184faf7..742e32d80f4e 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -292,6 +292,15 @@ def test_label_requests_gpu_on_unrelated_pull_request(self): self.assertTrue(d.run_cpu) self.assertTrue(d.run_gpu) + def test_label_match_ignores_case(self): + d = decide("pull_request", ["projects/rocminfo/x.cc"], labels=["CI:HRR-Capture"]) + self.assertTrue(d.run_gpu) + + def test_capture_only_path_also_runs_cpu(self): + d = decide("pull_request", ["projects/clr/hipamd/src/hip_context.cpp"]) + self.assertTrue(d.run_cpu) + self.assertTrue(d.run_gpu) + def test_family_label_adds_family(self): d = decide( "pull_request", From c1150cd53cef55443896ffd7b37728670cb2e790 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 20:48:15 +0200 Subject: [PATCH 24/89] [hrr] CI: report a failed job listing as an error Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/scripts/hrr_ci_report.py | 8 +++++++- projects/hrr/tests/scripts/test_hrr_ci_report.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index f49bec344f61..8d939149d9ec 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -145,7 +145,13 @@ def main(argv: list[str] | None = None) -> int: if args.base_json and args.base_json.is_file(): base = json.loads(args.base_json.read_text()) - rows = classify_jobs(list_jobs(args.repo, args.run_id, args.run_attempt)) + try: + jobs = list_jobs(args.repo, args.run_id, args.run_attempt) + except subprocess.CalledProcessError as e: + detail = (e.stderr or "").strip() or f"gh api exited {e.returncode}" + print(f"::error::could not list the jobs of run {args.run_id}: {detail}") + return 1 + rows = classify_jobs(jobs) missing = missing_families(families, rows) configured = args.configure_result == "success" diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index 52f0a6cc277a..f1823316dba1 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -7,6 +7,7 @@ import io import os import sys +import subprocess import tempfile import unittest from pathlib import Path @@ -107,6 +108,16 @@ def test_main_gives_skipped_families_a_row(self): self.assertIn("| Integration | gfx125x | — | — | — | — | _not run: disabled in ci.yaml_ |", out.getvalue()) + def test_main_reports_job_listing_failure(self): + err = subprocess.CalledProcessError(1, "gh", stderr="HTTP 403: rate limited\n") + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(rpt, "list_jobs", side_effect=err), \ + mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ + contextlib.redirect_stdout(io.StringIO()) as out: + rc = rpt.main(["--run-id", "7", "--results-dir", tmp]) + self.assertEqual(rc, 1) + self.assertIn("::error::could not list the jobs of run 7: HTTP 403: rate limited", out.getvalue()) + def test_main_fails_when_configure_did_not_succeed(self): for result in ("failure", "cancelled"): with tempfile.TemporaryDirectory() as tmp, \ From 49e1660d371d34f233e04db77f1d0813292477cd Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 20:48:15 +0200 Subject: [PATCH 25/89] [hrr] CI: carry the CLR-built hiprtc to the test job with amdhip64; fix test name Only libamdhip64 was overlaid; the base's release libhiprtc was loaded next to a CLR from this checkout. Co-Authored-By: Claude Sonnet 5.5 --- .../hrr/tests/scripts/hrr_capture_build.py | 36 ++++++++++--------- .../tests/scripts/test_hrr_capture_build.py | 12 ++++--- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index 163744bdafe0..8a8db4c6e70c 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -64,7 +64,9 @@ "core-hip_lib", ) -CAPTURE_LIB_GLOB = "libamdhip64.so*" +# What CLR's install writes into lib/: amdhip64 plus hiprtc and its builtins. +# All of it comes from this checkout, so all of it is carried to the test job. +CAPTURE_LIB_GLOBS = ("libamdhip64.so*", "libhiprtc*.so*") # The suite must load exactly one copy of each, from the installed tree: the # capture-enabled amdhip64 built here and the base's ROCr. @@ -175,26 +177,28 @@ def hrr_configure_command(src: Path, build: Path, dist: Path, archs: str) -> lis def remove_amdhip64(lib_dir: Path) -> None: - """Remove the release amdhip64 so only the capture-enabled one remains.""" - for path in lib_dir.glob(CAPTURE_LIB_GLOB): - path.unlink() + """Remove the release amdhip64 and hiprtc so only the CLR build's remain.""" + for glob in CAPTURE_LIB_GLOBS: + for path in lib_dir.glob(glob): + path.unlink() def copy_amdhip64(src_dir: Path, dst_dir: Path) -> list[str]: - """Copy libamdhip64.so* from src_dir to dst_dir, keeping symlinks.""" + """Copy the CAPTURE_LIB_GLOBS libraries from src_dir to dst_dir, keeping symlinks.""" dst_dir.mkdir(parents=True, exist_ok=True) copied = [] - for path in sorted(src_dir.glob(CAPTURE_LIB_GLOB)): - target = dst_dir / path.name - if target.is_symlink() or target.exists(): - target.unlink() - if path.is_symlink(): - os.symlink(os.readlink(path), target) - else: - shutil.copy2(path, target) - copied.append(path.name) - if not copied: - raise SystemExit(f"no {CAPTURE_LIB_GLOB} in {src_dir}") + for glob in CAPTURE_LIB_GLOBS: + for path in sorted(src_dir.glob(glob)): + target = dst_dir / path.name + if target.is_symlink() or target.exists(): + target.unlink() + if path.is_symlink(): + os.symlink(os.readlink(path), target) + else: + shutil.copy2(path, target) + copied.append(path.name) + if not any(name.startswith("libamdhip64.so") for name in copied): + raise SystemExit(f"no libamdhip64.so* in {src_dir}") return copied diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index f37a60da7177..a6752183b808 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -66,7 +66,7 @@ def test_build_targets_cover_uploaded_executables(self): names = {Path(rel).name for rel in cb.EXECUTABLES} self.assertTrue(names <= set(cb.HRR_TARGETS)) - def test_build_and_runtime_artifacts_are_generic_names(self): + def test_runtime_artifacts_are_a_subset_of_build_artifacts(self): self.assertTrue(set(cb.RUNTIME_ARTIFACTS) <= set(cb.BUILD_ARTIFACTS)) @@ -105,17 +105,21 @@ def test_overlay_replaces_release_library_and_keeps_rocr(self): dist = self.tmp / "dist" / "lib" self.make_lib(capture, "libamdhip64.so.7.2.1", "capture") self.make_lib(dist, "libamdhip64.so.7.2.0", "release") - (dist / "libhiprtc.so.7").write_text("keep") + (dist / "libhiprtc.so.7").write_text("release") + (capture / "libhiprtc.so.7").write_text("capture") (dist / "libhsa-runtime64.so.1").write_text("base") cb.remove_amdhip64(dist) copied = cb.copy_amdhip64(capture, dist) - self.assertEqual(sorted(copied), ["libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.7.2.1"]) + self.assertEqual( + sorted(copied), + ["libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.7.2.1", "libhiprtc.so.7"], + ) self.assertFalse((dist / "libamdhip64.so.7.2.0").exists()) self.assertEqual((dist / "libamdhip64.so").read_text(), "capture") self.assertTrue((dist / "libamdhip64.so.7").is_symlink()) - self.assertEqual((dist / "libhiprtc.so.7").read_text(), "keep") + self.assertEqual((dist / "libhiprtc.so.7").read_text(), "capture") self.assertEqual((dist / "libhsa-runtime64.so.1").read_text(), "base") def test_copy_without_library_fails(self): From 9256582ceed6f11bb4a0f24e32be670f508d7ecd Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 20:48:15 +0200 Subject: [PATCH 26/89] [hrr] CI: sync TheRock pin, always upload base record, per-commit push group, warn on missing runner config - TheRock pin c01e3a3f (2026-10-12) as in therock-multi-arch-ci.yml - hrr-base upload runs when cancelled() is false - push runs get a group per sha: cancel-in-progress is off for push but a newer pending run replaced a queued one, so that commit was never classified - warn when ROCm/therock-ci-config cannot be checked out Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-ci.yml | 9 +++++++-- .github/workflows/hrr-gpu.yml | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 8a7d32b703cc..44567172168b 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -33,7 +33,7 @@ permissions: # not cancel the run in progress; configure then runs nothing for it. concurrency: group: >- - ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}${{ + ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.sha) || github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'ci:hrr-capture' && !startsWith(github.event.label.name, 'ci:gfx') @@ -76,12 +76,13 @@ jobs: persist-credentials: false repository: ROCm/TheRock # Keep in sync with the TheRock ref in therock-multi-arch-ci.yml. - ref: 05add1921fd03b0649f054ec2a7c54d162c0cd99 # 2026-10-08 + ref: c01e3a3f3f67966dd35612cf363b49964f33cd47 # 2026-10-12 path: therock sparse-checkout: build_tools/github_actions # Current runner labels, as TheRock's setup_multi_arch.yml loads them. - name: Checkout TheRock CI runner config + id: ci_config uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 continue-on-error: true with: @@ -90,6 +91,10 @@ jobs: ref: main path: ci-config + - name: Warn when runner config is unavailable + if: ${{ steps.ci_config.outcome == 'failure' }} + run: echo "::warning::could not check out ROCm/therock-ci-config; runner labels may be out of date" + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index cd0ad4196af0..9484a114a90e 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -47,7 +47,7 @@ jobs: persist-credentials: false repository: ROCm/TheRock # Keep in sync with the TheRock ref in therock-multi-arch-ci.yml. - ref: 05add1921fd03b0649f054ec2a7c54d162c0cd99 # 2026-10-08 + ref: c01e3a3f3f67966dd35612cf363b49964f33cd47 # 2026-10-12 path: therock sparse-checkout: | build_tools @@ -70,6 +70,7 @@ jobs: run: python3 projects/hrr/tests/scripts/hrr_capture_build.py verify - name: Upload base build record + if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: hrr-base-${{ github.sha }} @@ -133,7 +134,7 @@ jobs: with: persist-credentials: false repository: ROCm/TheRock - ref: 05add1921fd03b0649f054ec2a7c54d162c0cd99 # 2026-10-08 + ref: c01e3a3f3f67966dd35612cf363b49964f33cd47 # 2026-10-12 path: therock sparse-checkout: | build_tools From 1627a716a1403f1ab68d5c00d56676c6159d08b4 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 14 Oct 2026 22:45:33 +0200 Subject: [PATCH 27/89] [hrr] CI: install the GPU job's Python requirements into a venv The test job runs on shared self-hosted GPU runners. python -m pip install wrote into setup-python's interpreter under _work/_tool on the host, which the next job on that machine reuses. Install into a virtualenv under RUNNER_TEMP instead, which is removed when the job ends. --- .github/workflows/hrr-gpu.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 9484a114a90e..0c762868bad4 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -146,8 +146,15 @@ jobs: with: python-version: "3.12" + # Install into a virtualenv under RUNNER_TEMP, which is removed when the + # job ends. A bare "python -m pip install" writes into setup-python's + # interpreter under _work/_tool on the runner's host, which the next job + # on the same machine reuses. Later steps pick the venv up from PATH. - name: Install Python requirements - run: python -m pip install --no-cache-dir -r therock/requirements.txt + run: | + python -m venv "${RUNNER_TEMP}/hrr-venv" + "${RUNNER_TEMP}/hrr-venv/bin/python" -m pip install --no-cache-dir -r therock/requirements.txt + echo "${RUNNER_TEMP}/hrr-venv/bin" >> "${GITHUB_PATH}" - name: Download build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 From 5f411f6645435e57f293b722580249722b04ec0f Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 15 Oct 2026 08:35:12 +0200 Subject: [PATCH 28/89] [hrr] CI: carry the new develop unit tests into the unit job Develop added hrr_limits_test, hrr_repair_test and hrr_slots_test to hrr-unit-tests, and a new hrr-handler-tests binary (six handler test files). unit.yaml listed none of them, so the compiled-case-list check would fail, and nothing ran hrr-handler-tests at all. - unit.yaml: add the 42 new cases. The 17 in hrr_repair_test.cc are compiled out under _WIN32, so they carry an unsupported-on-windows rule. - hrr_test_config.py check: a case an OS-wide unsupported rule covers is not MISSING on that OS; test added. - handler.yaml: the 28 hrr-handler-tests cases (Linux only). - hrr-ci.yml: lint handler.yaml; check and run hrr-handler-tests in the Linux unit job; upload its results; run tools/tests (generator rules) with the script tests, with the HIP headers in the sparse checkout. Case lists were taken from the TEST_CASE sources, not a built binary: no ROCm SDK locally, so the listing was not run. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-ci.yml | 33 +++++- projects/hrr/tests/config/handler.yaml | 50 ++++++++ projects/hrr/tests/config/unit.yaml | 110 ++++++++++++++++++ projects/hrr/tests/scripts/hrr_test_config.py | 17 ++- .../hrr/tests/scripts/test_hrr_test_config.py | 20 ++++ 5 files changed, 227 insertions(+), 3 deletions(-) create mode 100644 projects/hrr/tests/config/handler.yaml diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 44567172168b..3aa571640e9a 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -168,8 +168,12 @@ jobs: persist-credentials: false # Cone mode also checks out projects/clr/hipamd/src/amdhip.def.in, # which gen_hrr_api_args.py reads on import. A test reads hrr-ci.yml. + # The generator-rule tests in tools/tests regenerate the handlers from + # the HIP headers. sparse-checkout: | projects/hrr + projects/hip/include + projects/clr/hipamd/include projects/clr/hipamd/src/hrr .github/workflows @@ -184,10 +188,11 @@ jobs: run: >- python projects/hrr/tests/scripts/hrr_test_config.py lint projects/hrr/tests/config/unit.yaml + projects/hrr/tests/config/handler.yaml projects/hrr/tests/config/integration.yaml - name: Run tests - run: python -m pytest projects/hrr/tests/scripts + run: python -m pytest projects/hrr/tests/scripts projects/hrr/tools/tests unit: name: Unit (${{ matrix.os }}) @@ -275,12 +280,36 @@ jobs: --arch none -- "build/tests/unit/${EXE}" + # hrr-handler-tests is POSIX only (tests/unit/CMakeLists.txt), and an + # AddressSanitizer build that reaches no GPU. Its results land beside the + # unit results as an "Other" row in the summary. + - name: Check the compiled handler case list + if: ${{ runner.os == 'Linux' }} + run: >- + python projects/hrr/tests/scripts/hrr_test_config.py check + --binary build/tests/unit/hrr-handler-tests + --config projects/hrr/tests/config/handler.yaml + + - name: Run handler suite + if: ${{ runner.os == 'Linux' }} + env: + OS: ${{ matrix.os }} + HRR_TEST_IGNORE_CONFIG: ${{ inputs.ignore_test_config && '1' || '' }} + run: >- + python projects/hrr/tests/scripts/run_catch2.py + --xml "handler-${OS}.xml" + --config projects/hrr/tests/config/handler.yaml + --arch none + -- build/tests/unit/hrr-handler-tests + - name: Upload results if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: results-unit-${{ matrix.os }} - path: unit-${{ matrix.os }}.xml + path: | + unit-${{ matrix.os }}.xml + handler-${{ matrix.os }}.xml if-no-files-found: ignore retention-days: 7 diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml new file mode 100644 index 000000000000..267370f807d6 --- /dev/null +++ b/projects/hrr/tests/config/handler.yaml @@ -0,0 +1,50 @@ +# HRR test configuration. Every case the binary registers has an entry here. +# An empty entry runs on every OS and GPU architecture. +# +# A skip rule matches when every key under "when" matches (a list is OR, a +# missing key matches anything, and a value may be a glob such as gfx11*). +# kind: disabled is a temporary regression and requires a public issue URL. +# kind: unsupported is permanent and needs only a reason. +# +# hrr-handler-tests is built only on POSIX (tests/unit/CMakeLists.txt), so this +# suite targets Linux only. +# +# Regenerate the case names from a built binary with: +# hrr_test_config.py check --binary --config --update +# Listing needs no GPU. Say so in the commit message when the set changes. +version: 1 +suite: unit +binary: hrr-handler-tests +select: "*" +targets: + os: [linux] + arch: [gfx90a, gfx942, gfx950, gfx1030, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1153, gfx1200, gfx1201, gfx1250] +cases: + Unit_HRR_AccessPolicyWindow_KernelNodeAttributeTranslatesBasePtr: {} + Unit_HRR_AccessPolicyWindow_StreamAttributeTranslatesBasePtr: {} + Unit_HRR_Blob_AbsentBlobDoesNotEndTheReplay: {} + Unit_HRR_Blob_CapRefusalEndsTheReplay: {} + Unit_HRR_GraphDeps_InlineCapacity: {} + Unit_HRR_GraphDeps_MemAllocNodeRefusesOversizeCount: {} + Unit_HRR_GraphDeps_SymbolCopyNodesRefuseOversizeCount: {} + Unit_HRR_GraphDeps_SymbolCopyNodesRefuseTruncatedList: {} + Unit_HRR_GraphDeps_TruncatedDependencyListIsRefused: {} + Unit_HRR_HostBudget_BlobCacheChargesOnce: {} + Unit_HRR_HostBudget_BlobCacheRefusesOverBudgetAndMissing: {} + Unit_HRR_HostBudget_CodeObjectCacheIsChargedSeparately: {} + Unit_HRR_HostBudget_ConcurrentLoadsOfOneBlobChargeOnce: {} + Unit_HRR_HostBudget_DispatchFreesReplacedHostAllocation: {} + Unit_HRR_HostBudget_DispatchTableUsesTheChargingHandlers: {} + Unit_HRR_HostBudget_HostAllocationHandlersRefuseOverBudget: {} + Unit_HRR_HostBudget_LandingBufferChargesGrowthOnly: {} + Unit_HRR_Query_CompletingQueryReturnsSuccess: {} + Unit_HRR_Query_DefaultHasNoAttemptBound: {} + Unit_HRR_Query_NeverCompletingQueryEndsTheReplay: {} + Unit_HRR_Query_OtherErrorIsReturnedAtOnce: {} + Unit_HRR_Query_StopsWhenReplayIsFatal: {} + Unit_HRR_Query_ZeroBoundMakesNoCall: {} + Unit_HRR_Refusals_GeneratedHandlersRefuseUntranslatableSlots: {} + Unit_HRR_Refusals_LinkAddFileIsRefused: {} + Unit_HRR_Refusals_LinkCreateRefusesPointerCarryingOptions: {} + Unit_HRR_Refusals_Memcpy2DNoOpCopiesAreNotRefused: {} + Unit_HRR_Refusals_Memcpy2DWithUnmappedEndIsRefused: {} diff --git a/projects/hrr/tests/config/unit.yaml b/projects/hrr/tests/config/unit.yaml index 143a62e70b68..e53e576ab044 100644 --- a/projects/hrr/tests/config/unit.yaml +++ b/projects/hrr/tests/config/unit.yaml @@ -25,6 +25,28 @@ cases: Unit_HRR_Format_SmallPayloadForType: {} Unit_HRR_Format_TruncatedEvent: {} Unit_HRR_Format_UnknownEventType: {} + Unit_HRR_Limits_ByteBudget: {} + Unit_HRR_Limits_ByteBudgetConcurrent: {} + Unit_HRR_Limits_ContiguousSequenceReplays: {} + Unit_HRR_Limits_DuplicateIsReportedImmediately: {} + Unit_HRR_Limits_DuplicateSequenceEndsReplay: {} + Unit_HRR_Limits_EventCap: {} + Unit_HRR_Limits_FatalFlagEndsWait: {} + Unit_HRR_Limits_FindDuplicateSeq: {} + Unit_HRR_Limits_FindSeqGap: {} + Unit_HRR_Limits_NoStallWhileACallIsInFlight: {} + Unit_HRR_Limits_ParseU64: {} + Unit_HRR_Limits_PlaybackContextHostBudget: {} + Unit_HRR_Limits_PlaybackRefusesTooManyEvents: {} + Unit_HRR_Limits_PlaybackRefusesTooManyThreads: {} + Unit_HRR_Limits_PlaybackRefusesWallSecondsAboveMaximum: {} + Unit_HRR_Limits_QueryRetryIsBounded: {} + Unit_HRR_Limits_ReadFileMissing: {} + Unit_HRR_Limits_ReadFileRefusesOversize: {} + Unit_HRR_Limits_ScopedCharge: {} + Unit_HRR_Limits_SequenceGapEndsReplay: {} + Unit_HRR_Limits_ThreadCap: {} + Unit_HRR_Limits_WallClockWatchdog: {} Unit_HRR_Playback_PlacementHeldRanges: {} Unit_HRR_Playback_PlacementPlan: {} Unit_HRR_Playback_ReplayedH2DDrainGraphGuard: {} @@ -44,6 +66,94 @@ cases: Unit_HRR_Recovery_TornPayload: {} Unit_HRR_Recovery_TrailerCountMismatch: {} Unit_HRR_Recovery_TrailerOvercounts: {} + Unit_HRR_Repair_DropsSetuidAndSetgid: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_Repair_EventsLinkIsRefused: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_Repair_KeepsFilePermissions: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_Repair_LinkAtOldTempNameIsLeftAlone: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_Repair_LinkedPidDirectoryIsNotIndexed: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_Repair_ManifestLinkIsRefused: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_Repair_PidDirectoryLinkIsRefused: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_Repair_RecoversTornEventsWithoutLeftovers: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_Repair_RootManifestLinkIsRefusedBeforeFanOut: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_Repair_UserNamedLinkIsResolved: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_SafeFs_DotDotAfterLinkIsResolvedByTheKernel: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_SafeFs_FdOwnsItsDescriptor: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_SafeFs_OpenDirRefusesLinks: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_SafeFs_RandomHexAndErrnoText: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_SafeFs_TempFileModeAndName: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_SafeFs_TempIsExclusiveAndReplaceDoesNotFollow: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_SafeFs_UnlinkAtRemovesTheNameNotTheTarget: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_repair_test.cc is compiled out under _WIN32 (POSIX file-descriptor APIs) + Unit_HRR_Slots_BoundedCstr: {} + Unit_HRR_Slots_JitOptionArray: {} + Unit_HRR_Slots_JitOptionClassification: {} Unit_HRR_TranslatePtr_ExactMatch: {} Unit_HRR_TranslatePtr_OutOfRange_ReturnsNull: {} Unit_HRR_TranslatePtr_SubAlloc_ReturnsOffset: {} diff --git a/projects/hrr/tests/scripts/hrr_test_config.py b/projects/hrr/tests/scripts/hrr_test_config.py index 4244e4871096..89a1c8d57aa6 100644 --- a/projects/hrr/tests/scripts/hrr_test_config.py +++ b/projects/hrr/tests/scripts/hrr_test_config.py @@ -397,6 +397,19 @@ def detect_os() -> str: return "windows" if os.name == "nt" else "linux" +def compiled_out_here(config: dict, name: str, os_name: str) -> bool: + """True when an OS-wide ``unsupported`` rule covers the case on ``os_name``. + + Such a case may be compiled out of the binary there (an ``#ifndef _WIN32`` + around the whole file), so ``check`` does not call its absence MISSING. + A rule that names an architecture is not OS-wide and does not count. + """ + return any( + rule.get("kind") == "unsupported" and rule_matches(rule, os_name, []) is not None + for rule in _rules_for(config, name) + ) + + def _archs_in(text: str) -> list[str]: found = [] for line in text.splitlines(): @@ -487,7 +500,9 @@ def _cmd_check(args: argparse.Namespace) -> int: f"({len(added)} added, {len(removed)} removed)") for name in removed: print(f" removed {name}") - expected = set(config["cases"]) + os_name = detect_os() + expected = {name for name in config["cases"] + if not compiled_out_here(config, name, os_name)} actual = set(names) missing = sorted(expected - actual) unexpected = sorted(actual - expected) diff --git a/projects/hrr/tests/scripts/test_hrr_test_config.py b/projects/hrr/tests/scripts/test_hrr_test_config.py index 96b3d634a5d2..849eb0dd711d 100644 --- a/projects/hrr/tests/scripts/test_hrr_test_config.py +++ b/projects/hrr/tests/scripts/test_hrr_test_config.py @@ -199,6 +199,26 @@ def test_check_reports_both_directions(self): self.assertIn("UNEXPECTED", err.getvalue()) self.assertIn("New", err.getvalue()) + def test_check_allows_a_case_compiled_out_where_it_is_unsupported(self): + self.write(suite({ + "Kept": {}, + "PosixOnly": {"skip": [rule({"os": "windows"})]}, + "OneArch": {"skip": [rule({"os": "windows", "arch": "gfx90a"})]}, + })) + payload = listing(("Kept", [])) + for os_name, rc, text in (("windows", 1, "OneArch"), ("linux", 1, "PosixOnly")): + with self.subTest(os=os_name): + with mock.patch.object(cfg, "detect_os", return_value=os_name), \ + mock.patch.object(cfg.subprocess, "run", return_value=completed(payload)): + err = io.StringIO() + with contextlib.redirect_stderr(err): + got = cfg.main(["check", "--binary", "./bin", "--config", str(self.path)]) + self.assertEqual(got, rc) + self.assertIn("MISSING", err.getvalue()) + self.assertIn(text, err.getvalue()) + if os_name == "windows": + self.assertNotIn("PosixOnly", err.getvalue()) + def test_update_keeps_rules_and_adds_empty_entries(self): kept = rule({"os": "linux", "arch": "gfx1151"}, "disabled", "npu", "https://github.com/ROCm/rocm-systems/issues/1") From d7c3f6bba91c4c8c2791b71c70a89dbede9811f0 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 15 Oct 2026 13:05:51 +0200 Subject: [PATCH 29/89] [hrr] CI: close the label-masking hole and two silent-pass paths Review findings on PR #13256: 1. An unrelated label re-triggered the workflow with run_cpu/run_gpu false, so every HRR job and the summary job went skipped. GitHub treats a skipped check as satisfying a required status, so the label could turn a failing HRR CI green on the same commit. The summary now also runs on labeled events and passes through the verdict of the earlier run that actually tested this commit (attempt 1 first); failing closed when that verdict cannot be read. labeled stays in the trigger types for ci:hrr-capture / ci:gfx*. 2. hrr_test_config.py computed unexpected = actual - expected, but expected already dropped cases covered by OS-wide unsupported rules, so a correctly compiled-in case was reported as UNEXPECTED. Compare against the full config case list instead; missing keeps the relaxed set. 3. hrr_check_sources.py base_header() treated any non-zero git show as "file absent at base" and passed the api-compat check silently. It now verifies that the base commit resolves first (fail closed) and only maps genuine path-absence messages to "nothing to compare". --- .github/workflows/hrr-ci.yml | 43 +++++++++++- projects/hrr/tests/scripts/hrr_ci_report.py | 40 ++++++++++- projects/hrr/tests/scripts/hrr_test_config.py | 11 ++- .../tests/scripts/test_hrr_check_sources.py | 67 +++++++++++++++++-- .../tests/scripts/test_hrr_ci_configure.py | 37 ++++++++++ .../hrr/tests/scripts/test_hrr_ci_report.py | 44 ++++++++++++ .../hrr/tests/scripts/test_hrr_test_config.py | 30 +++++++++ projects/hrr/tools/hrr_check_sources.py | 50 ++++++++++++-- 8 files changed, 307 insertions(+), 15 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 3aa571640e9a..e61bb62d073e 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -6,6 +6,11 @@ name: HRR CI +# labeled stays in the types so that adding ci:hrr-capture or ci:gfx* can +# still start a run; hrr_ci_configure.py decides from the label itself +# whether anything runs, and the summary job reports the verdict of the run +# that actually tested the commit (see `pass_through_ok`). A label that +# changes nothing therefore cannot mask a failing run on the same commit. on: pull_request: types: [opened, synchronize, reopened, labeled] @@ -330,7 +335,8 @@ jobs: if: >- ${{ always() && (needs.configure.result != 'success' || needs.configure.outputs.run_cpu == 'true' - || needs.configure.outputs.run_gpu == 'true') }} + || needs.configure.outputs.run_gpu == 'true' + || (github.event_name == 'pull_request' && github.event.action == 'labeled')) }} runs-on: ubuntu-24.04 permissions: actions: read @@ -351,6 +357,7 @@ jobs: pattern: results-* merge-multiple: true path: results + if-no-files-found: ignore - name: Download base build record if: ${{ needs.configure.outputs.run_gpu == 'true' }} @@ -360,6 +367,36 @@ jobs: name: hrr-base-${{ github.sha }} path: base + # A labeled pull request run whose configure decided nothing runs no + # HRR job of its own to summarize. GitHub keeps the most recent check + # run of a name on a commit, so skipping this summary would let an + # unrelated label turn a failing HRR CI into a passing one: fetch the + # verdict of the run that actually tested this commit instead, and + # fail (not pass) when it cannot be read. + - name: Find the run this commit was actually tested by + if: >- + ${{ github.event_name == 'pull_request' && github.event.action == 'labeled' + && needs.configure.outputs.run_cpu == 'false' + && needs.configure.outputs.run_gpu == 'false' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + # The most recent earlier run of this workflow for this exact + # commit; attempt 1 of a re-run first, so that a passing re-run + # cannot hide the run that really failed. + line="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs?head_branch=${GITHUB_HEAD_REF}&per_page=100" \ + --jq '[.workflow_runs[] + | select(.id != env.GITHUB_RUN_ID + and .head_sha == env.GITHUB_SHA + and ((.name // "") | endswith(env.GITHUB_JOB)) + and .status != "queued")] + | (map(select(.run_attempt == 1)) + .) | first | @tsv')" || line="" + if [ -n "${line}" ]; then + echo "run_id=$(printf '%s' "${line}" | cut -f1)" >> "${GITHUB_OUTPUT}" + echo "run_attempt=$(printf '%s' "${line}" | cut -f2)" >> "${GITHUB_OUTPUT}" + fi + - name: Summarize env: GH_TOKEN: ${{ github.token }} @@ -368,9 +405,13 @@ jobs: RUN_GPU: ${{ needs.configure.outputs.run_gpu }} FAMILIES: ${{ needs.configure.outputs.families }} SKIPPED: ${{ needs.configure.outputs.skipped_families }} + UPSTREAM_RUN: ${{ steps.upstream.outputs.run_id }} + UPSTREAM_ATTEMPT: ${{ steps.upstream.outputs.run_attempt }} run: >- python3 projects/hrr/tests/scripts/hrr_ci_report.py --configure-result "${CONFIGURE_RESULT}" --reason "${REASON}" --run-gpu "${RUN_GPU}" --families "${FAMILIES}" --skipped-families "${SKIPPED}" --results-dir results --base-json base/base.json + --pass-through --upstream-run "${UPSTREAM_RUN}" + --upstream-attempt "${UPSTREAM_ATTEMPT}" diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index 8d939149d9ec..940b1dcabef1 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -103,6 +103,18 @@ def overall_ok(rows: list[dict], missing: list[str]) -> bool: return not missing and not any(r["conclusion"] in BAD_CONCLUSIONS for r in rows) +def pass_through_ok(rows: list[dict]) -> tuple[bool, str]: + """Verdict on an earlier run's jobs. A job that has not finished or is + something other than a plain success is not allowed to count as pass: a + label run must never turn a failing HRR CI into a passing one.""" + if not rows: + return False, "the run that tested this commit lists no HRR jobs" + for row in rows: + if row["conclusion"] not in ("success", "skipped", "neutral"): + return False, f"{row['kind']} {row['name'] or row['conclusion']} is {row['conclusion']}" + return True, "" + + def list_jobs(repo: str, run_id: str, attempt: str) -> list[dict]: out = subprocess.run( [ @@ -136,6 +148,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "")) parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID", "")) parser.add_argument("--run-attempt", default=os.environ.get("GITHUB_RUN_ATTEMPT", "1")) + parser.add_argument("--pass-through", action="store_true", + help="deliver the verdict of the run that tested this commit") + parser.add_argument("--upstream-run", default="", + help="that run's id; empty when no earlier run was found") + parser.add_argument("--upstream-attempt", default="") args = parser.parse_args(argv) run_gpu = args.run_gpu == "true" @@ -145,15 +162,34 @@ def main(argv: list[str] | None = None) -> int: if args.base_json and args.base_json.is_file(): base = json.loads(args.base_json.read_text()) + job_run, job_attempt, exiting = args.run_id, args.run_attempt, False + if args.pass_through: + job_run = args.upstream_run or args.run_id + job_attempt = args.upstream_attempt + if not job_attempt: + # Nothing before this run, or the lookup failed: leave with a red + # summary, which a later re-run resolves, rather than green. + print(f"::error::no earlier run of HRR CI found for this commit; " + f"a label event cannot verify it, so this summary fails for run {args.run_id}") + return 1 + exiting = True + try: - jobs = list_jobs(args.repo, args.run_id, args.run_attempt) + jobs = list_jobs(args.repo, job_run, job_attempt) except subprocess.CalledProcessError as e: detail = (e.stderr or "").strip() or f"gh api exited {e.returncode}" - print(f"::error::could not list the jobs of run {args.run_id}: {detail}") + print(f"::error::could not list the jobs of run {job_run}: {detail}") return 1 rows = classify_jobs(jobs) missing = missing_families(families, rows) + if exiting: + ok, why = pass_through_ok(rows) + if not ok: + print(f"::error::HRR CI on this commit reported {why}; " + f"a label event must not turn it into a pass") + return 1 + configured = args.configure_result == "success" markdown = render_header(args.reason, run_gpu, families, base, args.configure_result) markdown += render_jobs(rows, missing) diff --git a/projects/hrr/tests/scripts/hrr_test_config.py b/projects/hrr/tests/scripts/hrr_test_config.py index 89a1c8d57aa6..bbd4a66a62c3 100644 --- a/projects/hrr/tests/scripts/hrr_test_config.py +++ b/projects/hrr/tests/scripts/hrr_test_config.py @@ -501,11 +501,16 @@ def _cmd_check(args: argparse.Namespace) -> int: for name in removed: print(f" removed {name}") os_name = detect_os() - expected = {name for name in config["cases"] - if not compiled_out_here(config, name, os_name)} + # MISSING is measured against the cases an OS-wide unsupported rule may + # have compiled out here; UNEXPECTED against every case the config + # knows. Subtracting the relaxed set from the binary's list would blame + # a case that is legitimately still compiled into the binary. + unsupported = {name for name in config["cases"] + if compiled_out_here(config, name, os_name)} + expected = {name for name in config["cases"] if name not in unsupported} actual = set(names) missing = sorted(expected - actual) - unexpected = sorted(actual - expected) + unexpected = sorted(actual - set(config["cases"])) errors = lint(config, cases=found) if not missing and not unexpected and not errors: print(f"OK: {len(actual)} test cases present, matching {args.config}") diff --git a/projects/hrr/tests/scripts/test_hrr_check_sources.py b/projects/hrr/tests/scripts/test_hrr_check_sources.py index 934bcc019917..11c9608d536a 100644 --- a/projects/hrr/tests/scripts/test_hrr_check_sources.py +++ b/projects/hrr/tests/scripts/test_hrr_check_sources.py @@ -5,6 +5,7 @@ import contextlib import io +import subprocess import sys import tempfile import unittest @@ -94,7 +95,7 @@ def test_compares_against_base(self): self.regenerate() ok, _ = self.quiet( chk.check_api_compat, self.root, self.out, "abc", - self.runner, lambda root, sha: "base header\n", + self.runner, lambda root, sha: ("base header\n", None), ) self.assertTrue(ok) cmd = self.calls[0] @@ -107,7 +108,7 @@ def test_incompatible_ids_fail(self): self.rc["check_hrr_api_compat.py"] = 1 ok, output = self.quiet( chk.check_api_compat, self.root, self.out, "abc", - self.runner, lambda root, sha: "base\n", + self.runner, lambda root, sha: ("base\n", None), ) self.assertFalse(ok) self.assertIn("not compatible with abc", output) @@ -116,21 +117,79 @@ def test_missing_base_header_passes(self): self.regenerate() ok, output = self.quiet( chk.check_api_compat, self.root, self.out, "abc", - self.runner, lambda root, sha: None, + self.runner, lambda root, sha: (None, None), ) self.assertTrue(ok) self.assertEqual(self.calls, []) self.assertIn("does not exist at abc", output) + def test_git_failure_fails_instead_of_passing(self): + # A base blob the blob:none clone cannot fetch, or a base commit it + # does not have: not "the file is absent", so no silent pass. + self.regenerate() + failure = (None, "cannot read projects/hrr/include/hrr/hrr_api_args.h " + "at abc: fatal: could not read from remote repository") + ok, output = self.quiet( + chk.check_api_compat, self.root, self.out, "abc", + self.runner, lambda root, sha: failure, + ) + self.assertFalse(ok) + self.assertEqual(self.calls, []) + self.assertIn("::error::cannot read", output) + self.assertIn("do not pass it silently", output) + def test_needs_regenerated_header(self): ok, output = self.quiet( chk.check_api_compat, self.root, self.out, "abc", - self.runner, lambda root, sha: "x", + self.runner, lambda root, sha: ("x", None), ) self.assertFalse(ok) self.assertIn("run the generated check first", output) +class BaseHeaderTest(unittest.TestCase): + """base_header's telling "the path is absent" from "git broke".""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + for argv in (["init", "-q"], ["config", "user.email", "t@example.com"], + ["config", "user.name", "t"]): + subprocess.run(["git", "-C", str(self.root), *argv], check=True, + capture_output=True, text=True) + + def commit(self, relpath: str, text: str) -> str: + path = self.root / relpath + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + subprocess.run(["git", "-C", str(self.root), "add", relpath], check=True) + subprocess.run(["git", "-C", str(self.root), "commit", "-qm", relpath], check=True) + return subprocess.run( + ["git", "-C", str(self.root), "rev-parse", "HEAD"], + check=True, capture_output=True, text=True, + ).stdout.strip() + + def test_content_at_base(self): + sha = self.commit(chk.API_ARGS_HEADER, "// base\n") + text, failure = chk.base_header(self.root, sha) + self.assertEqual(text, "// base\n") + self.assertIsNone(failure) + + def test_path_genuinely_absent_passes_as_none(self): + # Committed, but never this path: the only "nothing to compare" case. + sha = self.commit("README.md", "x\n") + text, failure = chk.base_header(self.root, sha) + self.assertIsNone(text) + self.assertIsNone(failure) + + def test_base_commit_the_clone_does_not_have_fails(self): + text, failure = chk.base_header(self.root, "0" * 40) + self.assertIsNone(text) + self.assertIn("cannot read", failure) + self.assertIn("at " + "0" * 40, failure) + + class ApiMatrixTest(Tree): def test_up_to_date(self): ok, _ = self.quiet(chk.check_api_matrix, self.root, self.out, self.runner) diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 742e32d80f4e..8c40aa9c9941 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -7,6 +7,7 @@ import io import os import sys +import re import tempfile import unittest from contextlib import redirect_stdout @@ -388,6 +389,42 @@ def test_capture_or_family_label_reruns(self): ) self.assertTrue(d.run_gpu, label) + def test_labeled_trigger_and_summary_report_a_verdict(self): + """Regression for the label that can turn a failing HRR CI green. + + github keeps the most recent check run of a name on a commit, and a + skipped check satisfies a required-status rule, so a labeled run + that configures nothing must not let its summary skip on top of the + run that failed: the summary must run for every labeled event and + answer with the verdict of the run that tested the commit (the + label run knows nothing itself), never a blanket pass. + """ + text = self.WORKFLOW.read_text(encoding="utf-8") + types = re.search(r"types:\s*\[([^\]]*)\]", text).group(1) + self.assertIn("labeled", types) + + summary = text[text.index(" summary:"):] + # The summary also runs when the label run configured nothing. + self.assertIn("github.event.action == 'labeled'", summary) + # And it answers with an actual verdict, never a skip of its own. + self.assertNotIn("needs.configure.result != 'success' || needs.configure.outputs.run_cpu == 'true' ||", + summary) + self.assertIn("--pass-through", summary) + self.assertIn("--upstream-run", summary) + # The lookup step feeds --upstream-run and only runs for a label + # run that configured soly nothing; a run that did fetch its verdict + # failing is what closes the hole, so the fetch may not be fatal. + lookup = summary[summary.index("Find the run this commit was actually tested by"):] + lookup = lookup[:lookup.index("- name: Summarize")] + self.assertIn("steps.upstream.outputs.run_id", summary) + self.assertIn("run_cpu == 'false'", lookup) + self.assertIn("run_gpu == 'false'", lookup) + self.assertIn("continue-on-error: true", lookup) + self.assertIn("run_attempt == 1", lookup) + self.assertIn("head_sha == env.GITHUB_SHA", lookup) + # No earlier run found -> empty outputs -> the report exits 1. + self.assertIn("if [ -n", lookup) + def test_concurrency_group_makes_the_same_test(self): text = self.WORKFLOW.read_text(encoding="utf-8") group = text[text.index("concurrency:"):text.index("cancel-in-progress:")] diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index f1823316dba1..0646c8d23052 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -130,5 +130,49 @@ def test_main_fails_when_configure_did_not_succeed(self): self.assertIn(f"Configure: {result}, so nothing else ran", out.getvalue()) +class PassThroughTest(unittest.TestCase): + """A label run with nothing to run reports the run that tested the + commit; it must never report greener than that run.""" + LABEL_ARGS = ["--pass-through", "--upstream-run", "99", "--upstream-attempt", "1", + "--run-id", "100", "--run-attempt", "1"] + GREEN = [JOBS[1], JOBS[2], JOBS[5]] # generated, unit ubuntu, integration gfx94x + + def run_main(self, jobs, extra=()): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + with mock.patch.object(rpt, "list_jobs", return_value=jobs) as listed, \ + mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ + contextlib.redirect_stdout(io.StringIO()) as out: + rc = rpt.main(["--results-dir", tmp.name, *self.LABEL_ARGS, *extra]) + return rc, out.getvalue(), listed + + def test_delivers_the_earlier_run_jobs(self): + rc, text, listed = self.run_main(self.GREEN) + self.assertEqual(rc, 0, text) + listed.assert_called_once_with(mock.ANY, "99", "1") + self.assertIn("| Generated sources | — | [success](c0) |", text) + + def test_fails_when_the_earlier_run_failed(self): + rc, text, _ = self.run_main(JOBS) # windows unit failed, gfx950 cancelled + self.assertEqual(rc, 1) + self.assertIn("a label event must not turn it into a pass", text) + + def test_fails_on_a_job_that_has_not_finished(self): + jobs = [{"name": "HRR script tests", "conclusion": "in_progress", "html_url": "u9"}] + rc, text, _ = self.run_main(jobs) + self.assertEqual(rc, 1) + self.assertIn("is in_progress", text) + + def test_fails_when_no_earlier_run_was_found(self): + rc, text, _ = self.run_main(self.GREEN, extra=["--upstream-attempt", ""]) + self.assertEqual(rc, 1) + self.assertIn("no earlier run of HRR CI found", text) + + def test_fails_when_the_earlier_run_has_no_hrr_jobs(self): + rc, text, _ = self.run_main([JOBS[0]]) # only Configure matches nothing + self.assertEqual(rc, 1) + self.assertIn("lists no HRR jobs", text) + + if __name__ == "__main__": unittest.main() diff --git a/projects/hrr/tests/scripts/test_hrr_test_config.py b/projects/hrr/tests/scripts/test_hrr_test_config.py index 849eb0dd711d..cd264df59059 100644 --- a/projects/hrr/tests/scripts/test_hrr_test_config.py +++ b/projects/hrr/tests/scripts/test_hrr_test_config.py @@ -219,6 +219,36 @@ def test_check_allows_a_case_compiled_out_where_it_is_unsupported(self): if os_name == "windows": self.assertNotIn("PosixOnly", err.getvalue()) + def test_supported_case_compiled_in_is_not_unexpected(self): + # An OS-wide unsupported rule (even an empty one) only relaxes the + # MISSING side: the case stays in the config, so a binary that still + # compiles it is correct and must not be reported UNEXPECTED. + self.write(suite({ + "Kept": {}, + "NoWhen": {"skip": [rule({})]}, + "LinuxOnly": {"skip": [rule({"os": "linux"})]}, + })) + payload = listing(("Kept", []), ("NoWhen", []), ("LinuxOnly", [])) + with mock.patch.object(cfg, "detect_os", return_value="linux"), \ + mock.patch.object(cfg.subprocess, "run", return_value=completed(payload)): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + rc = cfg.main(["check", "--binary", "./bin", "--config", str(self.path)]) + self.assertEqual(rc, 0, err.getvalue()) + self.assertNotIn("UNEXPECTED", err.getvalue()) + self.assertIn("OK: 3 test cases present", out.getvalue()) + + def test_case_not_in_the_config_is_still_unexpected(self): + self.write(suite({"Kept": {}, "LinuxOnly": {"skip": [rule({"os": "linux"})]}})) + payload = listing(("Kept", []), ("LinuxOnly", []), ("Bogus", [])) + with mock.patch.object(cfg, "detect_os", return_value="linux"), \ + mock.patch.object(cfg.subprocess, "run", return_value=completed(payload)): + err = io.StringIO() + with contextlib.redirect_stderr(err): + rc = cfg.main(["check", "--binary", "./bin", "--config", str(self.path)]) + self.assertEqual(rc, 1) + self.assertIn("UNEXPECTED (added without updating the config): Bogus", err.getvalue()) + def test_update_keeps_rules_and_adds_empty_entries(self): kept = rule({"os": "linux", "arch": "gfx1151"}, "disabled", "npu", "https://github.com/ROCm/rocm-systems/issues/1") diff --git a/projects/hrr/tools/hrr_check_sources.py b/projects/hrr/tools/hrr_check_sources.py index 8d6c0fc15960..54d9a3dc9980 100644 --- a/projects/hrr/tools/hrr_check_sources.py +++ b/projects/hrr/tools/hrr_check_sources.py @@ -18,6 +18,7 @@ import argparse import difflib +import re import subprocess import sys from pathlib import Path @@ -95,14 +96,49 @@ def check_generated(root: Path, out: Path, runner: Runner = run) -> bool: return True -def base_header(root: Path, base_sha: str) -> str | None: - """hrr_api_args.h at base_sha, or None when the base has no copy.""" +def base_header(root: Path, base_sha: str) -> tuple[str | None, str | None]: + """hrr_api_args.h at base_sha. + + Returns (content, None) on success, (None, None) when the path genuinely + does not exist at base_sha -- the only case with nothing to compare -- + and (None, reason) for any other git failure. The base commit must be + verified to resolve first: in a plain checkout an unresolvable rev and + a missing path print the same 'does not exist in' text, and the + codegen job's ``blob:none`` clone can also report 'not our ref' for a + commit it does not have -- an unreadable base must fail the check + instead of passing an incompatible API-ID renumbering. + """ + verify = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--verify", "--quiet", + f"{base_sha}^{{commit}}"], + capture_output=True, + text=True, + ) + if verify.returncode != 0: + detail = (verify.stderr or verify.stdout).strip() + return None, ( + f"cannot read {API_ARGS_HEADER} at {base_sha}: the base commit does not " + f"resolve ({detail or f'git rev-parse exited {verify.returncode}'})" + ) proc = subprocess.run( ["git", "-C", str(root), "show", f"{base_sha}:{API_ARGS_HEADER}"], capture_output=True, text=True, ) - return proc.stdout if proc.returncode == 0 else None + if proc.returncode == 0: + return proc.stdout, None + detail = (proc.stderr or proc.stdout).strip() + if re.search(r"does not exist in|exists on disk, but not in", detail): + return None, None + if re.search(r"not our ref|error:Server does not allow request", detail): + return None, ( + f"cannot read {API_ARGS_HEADER} at {base_sha}: " + f"the blob could not be fetched ({detail.splitlines()[-1]})" + ) + return None, ( + f"cannot read {API_ARGS_HEADER} at {base_sha}: " + f"{detail or f'git show exited {proc.returncode}'}" + ) def resolve_base(root: Path, base_sha: str) -> str: @@ -120,13 +156,17 @@ def check_api_compat( out: Path, base_sha: str, runner: Runner = run, - read_base: Callable[[Path, str], str | None] = base_header, + read_base: Callable[[Path, str], tuple[str | None, str | None]] = base_header, ) -> bool: current = out / GENERATED_FILES[API_ARGS_HEADER] if not current.is_file(): error("api-compat needs the regenerated header; run the generated check first") return False - text = read_base(root, base_sha) + text, failure = read_base(root, base_sha) + if failure: + error(failure) + error("the API-ID compatibility with the base could not be verified; do not pass it silently") + return False if text is None: print(f"{API_ARGS_HEADER} does not exist at {base_sha}; nothing to compare") return True From a0ea4dc13166531b951b65abff5b6198fe2a37f5 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 15 Oct 2026 17:14:31 +0000 Subject: [PATCH 30/89] [hrr] CI: fix the label run's pass-through summary d7c3f6b let a label run that runs nothing report the run that tested the commit, but the lookup could never find that run: the step had no id, jq cannot @tsv an object, the name filter compared the run name with the job id, and GITHUB_SHA is the merge commit rather than the PR head the runs API lists. --pass-through was also passed on every run, so every HRR CI Summary failed with "no earlier run found". The lookup is now in hrr_ci_report.py, with tests, and only a labeled run whose configure selected nothing uses it. It takes the newest other pull_request run on the PR head that has a verdict of its own (an HRR job that ran, or a configure that failed or has not finished), ordered by its latest attempt, and fails unless that run's HRR jobs and its own summary succeeded. With no such run there is nothing to hide and it passes, so a label on a PR that touches no HRR path stays green. --- .github/workflows/hrr-ci.yml | 45 ++---- projects/hrr/tests/scripts/hrr_ci_report.py | 147 +++++++++++++----- .../tests/scripts/test_hrr_ci_configure.py | 25 ++- .../hrr/tests/scripts/test_hrr_ci_report.py | 147 ++++++++++++++++-- 4 files changed, 257 insertions(+), 107 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index e61bb62d073e..a992c05557de 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -367,36 +367,6 @@ jobs: name: hrr-base-${{ github.sha }} path: base - # A labeled pull request run whose configure decided nothing runs no - # HRR job of its own to summarize. GitHub keeps the most recent check - # run of a name on a commit, so skipping this summary would let an - # unrelated label turn a failing HRR CI into a passing one: fetch the - # verdict of the run that actually tested this commit instead, and - # fail (not pass) when it cannot be read. - - name: Find the run this commit was actually tested by - if: >- - ${{ github.event_name == 'pull_request' && github.event.action == 'labeled' - && needs.configure.outputs.run_cpu == 'false' - && needs.configure.outputs.run_gpu == 'false' }} - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - run: | - # The most recent earlier run of this workflow for this exact - # commit; attempt 1 of a re-run first, so that a passing re-run - # cannot hide the run that really failed. - line="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs?head_branch=${GITHUB_HEAD_REF}&per_page=100" \ - --jq '[.workflow_runs[] - | select(.id != env.GITHUB_RUN_ID - and .head_sha == env.GITHUB_SHA - and ((.name // "") | endswith(env.GITHUB_JOB)) - and .status != "queued")] - | (map(select(.run_attempt == 1)) + .) | first | @tsv')" || line="" - if [ -n "${line}" ]; then - echo "run_id=$(printf '%s' "${line}" | cut -f1)" >> "${GITHUB_OUTPUT}" - echo "run_attempt=$(printf '%s' "${line}" | cut -f2)" >> "${GITHUB_OUTPUT}" - fi - - name: Summarize env: GH_TOKEN: ${{ github.token }} @@ -405,13 +375,20 @@ jobs: RUN_GPU: ${{ needs.configure.outputs.run_gpu }} FAMILIES: ${{ needs.configure.outputs.families }} SKIPPED: ${{ needs.configure.outputs.skipped_families }} - UPSTREAM_RUN: ${{ steps.upstream.outputs.run_id }} - UPSTREAM_ATTEMPT: ${{ steps.upstream.outputs.run_attempt }} + # A labeled run that configured nothing has no HRR job of its own. + # GitHub keeps the most recent check run of a name on a commit, so + # its summary reports the run that tested this commit instead of + # skipping, which would let the label turn a failing HRR CI green. + PASS_THROUGH: >- + ${{ github.event_name == 'pull_request' && github.event.action == 'labeled' + && needs.configure.result == 'success' + && needs.configure.outputs.run_cpu == 'false' + && needs.configure.outputs.run_gpu == 'false' }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} run: >- python3 projects/hrr/tests/scripts/hrr_ci_report.py --configure-result "${CONFIGURE_RESULT}" --reason "${REASON}" --run-gpu "${RUN_GPU}" --families "${FAMILIES}" --skipped-families "${SKIPPED}" --results-dir results --base-json base/base.json - --pass-through --upstream-run "${UPSTREAM_RUN}" - --upstream-attempt "${UPSTREAM_ATTEMPT}" + --pass-through "${PASS_THROUGH}" --head-sha "${HEAD_SHA}" diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index 940b1dcabef1..690375523e62 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -13,6 +13,10 @@ Exits 1 when the configure job did not succeed, any HRR job failed or was cancelled, or a results file could not be read. + +With --pass-through true (a label run that ran no HRR job) it reports the +newest other run on --head-sha that has a verdict instead, and exits 1 unless +that run's HRR jobs and its own summary succeeded, or when it cannot be read. """ from __future__ import annotations @@ -103,37 +107,112 @@ def overall_ok(rows: list[dict], missing: list[str]) -> bool: return not missing and not any(r["conclusion"] in BAD_CONCLUSIONS for r in rows) -def pass_through_ok(rows: list[dict]) -> tuple[bool, str]: - """Verdict on an earlier run's jobs. A job that has not finished or is - something other than a plain success is not allowed to count as pass: a - label run must never turn a failing HRR CI into a passing one.""" - if not rows: - return False, "the run that tested this commit lists no HRR jobs" +SUMMARY_JOB = "HRR CI Summary" + + +def tested_commit(jobs: list[dict], rows: list[dict]) -> bool: + """A run whose configure selected nothing still lists its HRR jobs, as + skipped, and tested nothing. One whose configure failed, or has not + finished, skipped them too but has a verdict of its own to report. A run + that lists no jobs yet is passed over; its own summary comes later and is + the newer check.""" + configure = next((j for j in jobs if j.get("name") == "Configure"), {}) + if (configure.get("conclusion") or configure.get("status")) not in (None, "success", "skipped"): + return True + return any(r["conclusion"] != "skipped" for r in rows) + + +def pass_through_ok(rows: list[dict], summary_conclusion: str) -> tuple[bool, str]: + """Verdict on an earlier run. Its own summary decides, since that is the + check a label run would otherwise hide, and every HRR job must also have + finished without failing: a label run must never turn a failing HRR CI + into a passing one.""" for row in rows: if row["conclusion"] not in ("success", "skipped", "neutral"): return False, f"{row['kind']} {row['name'] or row['conclusion']} is {row['conclusion']}" + if summary_conclusion != "success": + return False, f"{SUMMARY_JOB} is {summary_conclusion or 'missing'}" return True, "" -def list_jobs(repo: str, run_id: str, attempt: str) -> list[dict]: +def gh_api(path: str, jq: str) -> list: + """The values `gh api --paginate --jq` prints for path, one per jq result.""" out = subprocess.run( - [ - "gh", "api", "--paginate", - f"repos/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", - "--jq", ".jobs[]", - ], + ["gh", "api", "--paginate", path, "--jq", jq], check=True, capture_output=True, text=True, ).stdout decoder = json.JSONDecoder() - jobs, pos = [], 0 + values, pos = [], 0 while pos < len(out): while pos < len(out) and out[pos].isspace(): pos += 1 if pos >= len(out): break - job, pos = decoder.raw_decode(out, pos) - jobs.append(job) - return jobs + value, pos = decoder.raw_decode(out, pos) + values.append(value) + return values + + +def list_jobs(repo: str, run_id: str, attempt: str) -> list[dict]: + return gh_api(f"repos/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", ".jobs[]") + + +def find_tested_run(repo: str, run_id: str, head_sha: str) -> tuple[dict, list[dict], str] | None: + """The newest other pull_request run of this workflow on head_sha that + tested it, with its HRR jobs and its summary's conclusion. Its latest + attempt counts, as it does for the checks GitHub shows. Label runs that + ran nothing themselves are passed over.""" + [workflow_id] = gh_api(f"repos/{repo}/actions/runs/{run_id}", ".workflow_id") + runs = gh_api( + f"repos/{repo}/actions/workflows/{workflow_id}/runs" + f"?event=pull_request&head_sha={head_sha}&per_page=100", + ".workflow_runs[]", + ) + runs = [r for r in runs if str(r["id"]) != str(run_id)] + # A re-run keeps created_at; run_started_at moves with the latest attempt, + # whose check runs are the newest on the commit. + runs.sort(key=lambda r: r.get("run_started_at") or r["created_at"], reverse=True) + for run in runs: + jobs = list_jobs(repo, str(run["id"]), str(run["run_attempt"])) + rows = classify_jobs(jobs) + if tested_commit(jobs, rows): + summary = next((j for j in jobs if j.get("name") == SUMMARY_JOB), {}) + return run, rows, summary.get("conclusion") or summary.get("status") or "" + return None + + +def pass_through(repo: str, run_id: str, head_sha: str) -> int: + """Summary for a label run that ran no HRR job: the verdict of the run + that tested the commit, and never greener than it. Passes when no run + tested the commit, since there is then no failure to hide. Fails while + that run is still in progress; its own summary, written later, is then + the newer check.""" + try: + found = find_tested_run(repo, run_id, head_sha) + except (subprocess.CalledProcessError, ValueError) as e: + detail = (getattr(e, "stderr", "") or "").strip() or str(e) + print(f"::error::could not find the HRR CI run that tested {head_sha}: {detail}") + return 1 + if found is None: + print(f"No earlier HRR CI run ran HRR jobs on {head_sha}; nothing to report.") + return 0 + run, rows, summary_conclusion = found + markdown = ( + f"## HRR CI\n\n- This label changes nothing that runs; the verdict is that of " + f"[run {run['id']} attempt {run['run_attempt']}]({run.get('html_url', '')}), " + f"which tested this commit.\n\n" + ) + render_jobs(rows, []) + sys.stdout.write(markdown) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as out: + out.write(markdown) + ok, why = pass_through_ok(rows, summary_conclusion) + if not ok: + print(f"::error::HRR CI on this commit reported {why}; " + f"a label event must not turn it into a pass") + return 1 + return 0 def main(argv: list[str] | None = None) -> int: @@ -148,11 +227,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "")) parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID", "")) parser.add_argument("--run-attempt", default=os.environ.get("GITHUB_RUN_ATTEMPT", "1")) - parser.add_argument("--pass-through", action="store_true", - help="deliver the verdict of the run that tested this commit") - parser.add_argument("--upstream-run", default="", - help="that run's id; empty when no earlier run was found") - parser.add_argument("--upstream-attempt", default="") + parser.add_argument("--pass-through", choices=("true", "false"), default="false", + help="true: report the earlier run that tested --head-sha instead") + # Not GITHUB_SHA: on a pull request that is the merge commit, which no + # earlier run is listed under. + parser.add_argument("--head-sha", default="") args = parser.parse_args(argv) run_gpu = args.run_gpu == "true" @@ -162,34 +241,20 @@ def main(argv: list[str] | None = None) -> int: if args.base_json and args.base_json.is_file(): base = json.loads(args.base_json.read_text()) - job_run, job_attempt, exiting = args.run_id, args.run_attempt, False - if args.pass_through: - job_run = args.upstream_run or args.run_id - job_attempt = args.upstream_attempt - if not job_attempt: - # Nothing before this run, or the lookup failed: leave with a red - # summary, which a later re-run resolves, rather than green. - print(f"::error::no earlier run of HRR CI found for this commit; " - f"a label event cannot verify it, so this summary fails for run {args.run_id}") - return 1 - exiting = True + if args.pass_through == "true": + if not args.head_sha: + parser.error("--pass-through true needs --head-sha, the pull request head") + return pass_through(args.repo, args.run_id, args.head_sha) try: - jobs = list_jobs(args.repo, job_run, job_attempt) + jobs = list_jobs(args.repo, args.run_id, args.run_attempt) except subprocess.CalledProcessError as e: detail = (e.stderr or "").strip() or f"gh api exited {e.returncode}" - print(f"::error::could not list the jobs of run {job_run}: {detail}") + print(f"::error::could not list the jobs of run {args.run_id}: {detail}") return 1 rows = classify_jobs(jobs) missing = missing_families(families, rows) - if exiting: - ok, why = pass_through_ok(rows) - if not ok: - print(f"::error::HRR CI on this commit reported {why}; " - f"a label event must not turn it into a pass") - return 1 - configured = args.configure_result == "success" markdown = render_header(args.reason, run_gpu, families, base, args.configure_result) markdown += render_jobs(rows, missing) diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 8c40aa9c9941..45067424e1ed 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -409,21 +409,16 @@ def test_labeled_trigger_and_summary_report_a_verdict(self): # And it answers with an actual verdict, never a skip of its own. self.assertNotIn("needs.configure.result != 'success' || needs.configure.outputs.run_cpu == 'true' ||", summary) - self.assertIn("--pass-through", summary) - self.assertIn("--upstream-run", summary) - # The lookup step feeds --upstream-run and only runs for a label - # run that configured soly nothing; a run that did fetch its verdict - # failing is what closes the hole, so the fetch may not be fatal. - lookup = summary[summary.index("Find the run this commit was actually tested by"):] - lookup = lookup[:lookup.index("- name: Summarize")] - self.assertIn("steps.upstream.outputs.run_id", summary) - self.assertIn("run_cpu == 'false'", lookup) - self.assertIn("run_gpu == 'false'", lookup) - self.assertIn("continue-on-error: true", lookup) - self.assertIn("run_attempt == 1", lookup) - self.assertIn("head_sha == env.GITHUB_SHA", lookup) - # No earlier run found -> empty outputs -> the report exits 1. - self.assertIn("if [ -n", lookup) + # Only a label run that configured nothing passes the verdict through; + # every other run reports its own jobs. + step = summary[summary.index("PASS_THROUGH:"):summary.index("run: >-")] + self.assertIn("github.event.action == 'labeled'", step) + self.assertIn("needs.configure.result == 'success'", step) + self.assertIn("run_cpu == 'false'", step) + self.assertIn("run_gpu == 'false'", step) + self.assertIn('--pass-through "${PASS_THROUGH}"', summary) + # The runs API keys a run by the pull request head, not the merge commit. + self.assertIn("github.event.pull_request.head.sha", summary) def test_concurrency_group_makes_the_same_test(self): text = self.WORKFLOW.read_text(encoding="utf-8") diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index 0646c8d23052..ec132b3f5910 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -130,48 +130,161 @@ def test_main_fails_when_configure_did_not_succeed(self): self.assertIn(f"Configure: {result}, so nothing else ran", out.getvalue()) +class GhApiTest(unittest.TestCase): + def test_reads_every_value_across_pages(self): + out = subprocess.CompletedProcess([], 0, stdout='{"a": 1}\n{"a": 2}{"a": 3}\n7\n') + with mock.patch.object(rpt.subprocess, "run", return_value=out) as run: + self.assertEqual(rpt.gh_api("p", ".x[]"), [{"a": 1}, {"a": 2}, {"a": 3}, 7]) + self.assertEqual(run.call_args.args[0], ["gh", "api", "--paginate", "p", "--jq", ".x[]"]) + + def test_empty_output_is_no_values(self): + out = subprocess.CompletedProcess([], 0, stdout="\n") + with mock.patch.object(rpt.subprocess, "run", return_value=out): + self.assertEqual(rpt.gh_api("p", ".x[]"), []) + + def test_unreadable_workflow_id_fails_the_summary(self): + with mock.patch.object(rpt, "gh_api", return_value=[]), \ + contextlib.redirect_stdout(io.StringIO()) as out: + rc = rpt.main(["--repo", "R", "--pass-through", "true", "--head-sha", "abc", + "--run-id", "100"]) + self.assertEqual(rc, 1) + self.assertIn("could not find the HRR CI run that tested abc", out.getvalue()) + + class PassThroughTest(unittest.TestCase): """A label run with nothing to run reports the run that tested the commit; it must never report greener than that run.""" - LABEL_ARGS = ["--pass-through", "--upstream-run", "99", "--upstream-attempt", "1", + LABEL_ARGS = ["--pass-through", "true", "--head-sha", "abc", "--run-id", "100", "--run-attempt", "1"] GREEN = [JOBS[1], JOBS[2], JOBS[5]] # generated, unit ubuntu, integration gfx94x + SUMMARY_OK = {"name": "HRR CI Summary", "conclusion": "success", "html_url": "s0"} + SUMMARY_RED = {"name": "HRR CI Summary", "conclusion": "failure", "html_url": "s1"} + GREEN = GREEN + [SUMMARY_OK] + # What a run whose configure selected nothing lists: its jobs, skipped. + LABEL_ONLY = [ + {"name": "Configure", "conclusion": "success", "html_url": "u0"}, + {"name": "Generated sources", "conclusion": "skipped", "html_url": "k1"}, + {"name": "HRR script tests", "conclusion": "skipped", "html_url": "k2"}, + {"name": "Unit (${{ matrix.os }})", "conclusion": "skipped", "html_url": "k3"}, + {"name": "GPU", "conclusion": "skipped", "html_url": "k4"}, + SUMMARY_OK, + ] + + def run_main(self, runs, extra=()): + """runs: [(run id, attempt, created_at, jobs[, run_started_at])], as + the API lists them.""" + listing = [{"id": rid, "run_attempt": att, "created_at": at, + "run_started_at": started[0] if started else at, "html_url": f"r{rid}"} + for rid, att, at, _, *started in runs] + jobs = {(str(rid), str(att)): j for rid, att, _, j, *_ in runs} + + def gh_api(path, jq): + if path == "repos/R/actions/runs/100": + return [7] + if path.startswith("repos/R/actions/workflows/7/runs?event=pull_request&head_sha=abc&"): + return listing + rid, att = path.split("/runs/")[1].split("/attempts/") + return jobs[(rid, att.split("/")[0])] - def run_main(self, jobs, extra=()): tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) - with mock.patch.object(rpt, "list_jobs", return_value=jobs) as listed, \ + with mock.patch.object(rpt, "gh_api", side_effect=gh_api), \ mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ contextlib.redirect_stdout(io.StringIO()) as out: - rc = rpt.main(["--results-dir", tmp.name, *self.LABEL_ARGS, *extra]) - return rc, out.getvalue(), listed + rc = rpt.main(["--results-dir", tmp.name, "--repo", "R", + *self.LABEL_ARGS, *extra]) + return rc, out.getvalue() def test_delivers_the_earlier_run_jobs(self): - rc, text, listed = self.run_main(self.GREEN) + rc, text = self.run_main([(99, 1, "2026-10-15T10:00:00Z", self.GREEN)]) self.assertEqual(rc, 0, text) - listed.assert_called_once_with(mock.ANY, "99", "1") + self.assertIn("[run 99 attempt 1](r99)", text) self.assertIn("| Generated sources | — | [success](c0) |", text) def test_fails_when_the_earlier_run_failed(self): - rc, text, _ = self.run_main(JOBS) # windows unit failed, gfx950 cancelled + rc, text = self.run_main([(99, 1, "2026-10-15T10:00:00Z", JOBS + [self.SUMMARY_RED])]) self.assertEqual(rc, 1) self.assertIn("a label event must not turn it into a pass", text) + def test_fails_when_only_the_earlier_summary_failed(self): + # e.g. a family that never reported: every HRR job is green. + jobs = self.GREEN[:-1] + [self.SUMMARY_RED] + rc, text = self.run_main([(99, 1, "2026-10-15T10:00:00Z", jobs)]) + self.assertEqual(rc, 1) + self.assertIn("HRR CI Summary is failure", text) + + def test_skips_label_runs_and_takes_the_newest_tested_run(self): + rc, text = self.run_main([ + (98, 1, "2026-10-15T09:00:00Z", self.GREEN), + (99, 2, "2026-10-15T10:00:00Z", JOBS + [self.SUMMARY_RED]), # newest that tested + (101, 1, "2026-10-15T11:00:00Z", self.LABEL_ONLY), # an earlier label run + # This run, still in progress, with a green-looking job of its own. + (100, 1, "2026-10-15T12:00:00Z", self.GREEN[:-1]), + ]) + self.assertEqual(rc, 1, text) + self.assertIn("[run 99 attempt 2](r99)", text) + def test_fails_on_a_job_that_has_not_finished(self): - jobs = [{"name": "HRR script tests", "conclusion": "in_progress", "html_url": "u9"}] - rc, text, _ = self.run_main(jobs) + jobs = [{"name": "HRR script tests", "conclusion": None, "status": "in_progress", + "html_url": "u9"}, + {"name": "HRR CI Summary", "conclusion": None, "status": "queued"}] + rc, text = self.run_main([(99, 1, "2026-10-15T10:00:00Z", jobs)]) self.assertEqual(rc, 1) self.assertIn("is in_progress", text) - def test_fails_when_no_earlier_run_was_found(self): - rc, text, _ = self.run_main(self.GREEN, extra=["--upstream-attempt", ""]) - self.assertEqual(rc, 1) - self.assertIn("no earlier run of HRR CI found", text) + def test_fails_when_the_earlier_configure_failed(self): + jobs = [{**j, "conclusion": "failure"} if j["name"] == "Configure" else j + for j in self.LABEL_ONLY if j["name"] != "HRR CI Summary"] + [self.SUMMARY_RED] + rc, text = self.run_main([(99, 1, "2026-10-15T10:00:00Z", jobs)]) + self.assertEqual(rc, 1, text) + self.assertIn("[run 99 attempt 1](r99)", text) + self.assertIn("HRR CI Summary is failure", text) + + def test_an_unfinished_configure_has_a_verdict_to_wait_for(self): + jobs = [{"name": "Configure", "conclusion": None, "status": "in_progress"}, + {"name": "HRR CI Summary", "conclusion": None, "status": "queued"}] + rc, text = self.run_main([(99, 1, "2026-10-15T10:00:00Z", jobs)]) + self.assertEqual(rc, 1, text) + self.assertIn("[run 99 attempt 1](r99)", text) - def test_fails_when_the_earlier_run_has_no_hrr_jobs(self): - rc, text, _ = self.run_main([JOBS[0]]) # only Configure matches nothing + def test_a_rerun_counts_from_its_latest_attempt(self): + rc, text = self.run_main([ + (98, 2, "2026-10-15T09:00:00Z", JOBS + [self.SUMMARY_RED], "2026-10-15T12:00:00Z"), + (99, 1, "2026-10-15T10:00:00Z", self.GREEN), + ]) + self.assertEqual(rc, 1, text) + self.assertIn("[run 98 attempt 2](r98)", text) + + def test_needs_the_head_sha(self): + with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): + rpt.main(["--repo", "R", "--pass-through", "true", "--run-id", "100"]) + + def test_passes_when_no_run_tested_the_commit(self): + # A pull request that touches no HRR path: no run ran an HRR job. + rc, text = self.run_main([(99, 1, "2026-10-15T10:00:00Z", self.LABEL_ONLY)]) + self.assertEqual(rc, 0, text) + self.assertIn("nothing to report", text) + + def test_fails_when_the_lookup_fails(self): + err = subprocess.CalledProcessError(1, "gh", stderr="HTTP 502") + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + with mock.patch.object(rpt, "gh_api", side_effect=err), \ + contextlib.redirect_stdout(io.StringIO()) as out: + rc = rpt.main(["--results-dir", tmp.name, "--repo", "R", *self.LABEL_ARGS]) self.assertEqual(rc, 1) - self.assertIn("lists no HRR jobs", text) + self.assertIn("HTTP 502", out.getvalue()) + + def test_ordinary_run_reports_its_own_jobs(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + with mock.patch.object(rpt, "list_jobs", return_value=self.GREEN) as listed, \ + mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ + contextlib.redirect_stdout(io.StringIO()): + rc = rpt.main(["--results-dir", tmp.name, "--repo", "R", "--pass-through", "false", + "--run-id", "100", "--run-attempt", "2"]) + self.assertEqual(rc, 0) + listed.assert_called_once_with("R", "100", "2") if __name__ == "__main__": From 105abfc334cf8b7738d341e0649acd947081649a Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 15 Oct 2026 22:32:41 +0200 Subject: [PATCH 31/89] ci(hrr): run only gfx94x by default Every family stays listed in ci.yaml so a ci:gfx... label or the workflow_dispatch families input can still request it, but only gfx94x is enabled, so the default run is gfx94x (gfx942) alone. Delete an enabled: false line to put a family back on the default run. targets.arch stays wide: the configure job fails unless it covers every family ci.yaml lists, and a requested family needs its architecture compiled into the binary. That coupling is why this narrows what runs, not what is built. A label-requested disabled family still works, covered by a new test. --- projects/hrr/tests/README.md | 6 +++- projects/hrr/tests/config/ci.yaml | 32 ++++++++++++++----- .../tests/scripts/test_hrr_ci_configure.py | 23 +++++++++++-- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index 8b3b86526e2a..f062de56dfb9 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -152,7 +152,11 @@ applies (`scripts/hrr_ci_configure.py`): only gfx950 uses one, nightly; elsewhere the cases that need two devices skip and show in the summary's Skip column. A `ci:gfx...` label adds a family; the `workflow_dispatch` input `families` replaces the selection. Both can - name a disabled family. + name a disabled family. Only gfx94x is enabled today; the others are listed with + `enabled: false`, so they run only when a label or the input asks for them. + `targets.arch` in `config/integration.yaml` stays wide on purpose: configure fails + unless it covers every family listed here, and a requested family needs its + architecture compiled into the binary. - `rocm_sdk_version`: the ROCm SDK the unit job installs. An unknown key, trigger or family fails the configure job, and so does a family whose diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 9ed00440a4cf..fa4d0d09ae92 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -73,18 +73,34 @@ capture_label: ci:hrr-capture # A family without a runner label is skipped. Every job queues on its own # runner pool and the summary waits for all of them, so to reduce GPU load or # take a pool out, set enabled: false or narrow triggers here. +# +# Only gfx94x runs by default for now; every other family is disabled but +# still listed, so a `ci:gfx...` label or the workflow_dispatch `families` +# input can request it. Delete an `enabled: false` line to make a family run +# by default again. targets.arch in integration.yaml stays wide on purpose: +# configure fails unless it covers every family listed here, and a requested +# family needs its architecture compiled into the binary. families: - gfx90a: {} + gfx90a: + enabled: false gfx94x: {} - # Nightly on an 8-GPU node, so the multi-device cases run once a day. + # Off by default with the other non-gfx94x families. Nightly on an 8-GPU + # node, so once re-enabled the multi-device cases run once a day. gfx950: multi_gpu_triggers: [schedule] - gfx103x: {} - gfx110x: {} - gfx1150: {} - gfx1151: {} - gfx1153: {} - gfx120x: {} + enabled: false + gfx103x: + enabled: false + gfx110x: + enabled: false + gfx1150: + enabled: false + gfx1151: + enabled: false + gfx1153: + enabled: false + gfx120x: + enabled: false # Off until understood: in two runs the MI455 runner picked the job up, but # --arch auto found no GPU, so no case ran. TheRock also reserves this small # pool for designated workflows. A ci:gfx125x label or the workflow_dispatch diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 45067424e1ed..f0abcedced23 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -113,11 +113,30 @@ def family_keys(decision): class RepoConfigTest(unittest.TestCase): - def test_every_family_runs_on_every_event_by_default(self): + def test_only_gfx94x_runs_by_default(self): + """Every family stays listed, but only gfx94x runs unless asked for. + + The others stay in ci.yaml so a ci:gfx... label or the + workflow_dispatch families input can still request them; disabling is + what keeps them off the default run. + """ + self.assertEqual( + sorted(key for key, family in REPO.families.items() if family.enabled), + ["gfx94x"], + ) for key, family in REPO.families.items(): - self.assertEqual(family.enabled, key != "gfx125x", key) self.assertEqual(family.triggers, cfg.EVENTS, key) + def test_a_label_still_requests_a_disabled_family(self): + d = decide( + "pull_request", + ["projects/hrr/playback/x.cpp"], + ["ci:gfx110x"], + config=REPO, + ) + self.assertIn("gfx94x", family_keys(d)) + self.assertIn("gfx110x", family_keys(d)) + def test_multi_gpu_runner_only_nightly(self): for key, family in REPO.families.items(): self.assertIn(family.multi_gpu_triggers, ((), ("schedule",)), key) From 0888597df1aee2c51ddd6222e469ef2c17d502b4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 16 Oct 2026 08:32:12 +0000 Subject: [PATCH 32/89] ci(hrr): run rocJitsu by default and cache the capture build Keep the suite matrix in projects/hrr. The default simulator jobs are gfx950 on the two-GPU config and gfx125x on the four-GPU config. Real GPUs stay on workflow_dispatch or a ci.yaml trigger. The capture build restores from a content hash, and a local Docker entry point builds and runs the suites. --- .github/skip-ci-config.toml | 8 + .github/workflows/hrr-ci.yml | 15 +- .github/workflows/hrr-gpu.yml | 141 +++++++- .github/workflows/therock-multi-arch-ci.yml | 13 +- projects/hrr/tests/config/ci.yaml | 41 ++- projects/hrr/tests/docker/Dockerfile | 18 ++ .../hrr/tests/scripts/hrr_capture_build.py | 193 +++++++++-- .../hrr/tests/scripts/hrr_ci_configure.py | 305 +++++++++++++++--- projects/hrr/tests/scripts/hrr_dev.py | 217 +++++++++++++ projects/hrr/tests/scripts/run_catch2.py | 116 +++++-- .../tests/scripts/test_hrr_capture_build.py | 128 +++++++- .../tests/scripts/test_hrr_ci_configure.py | 277 +++++++++++++--- projects/hrr/tests/scripts/test_hrr_dev.py | 70 ++++ projects/hrr/tests/scripts/test_run_catch2.py | 231 ++++++++++--- 14 files changed, 1536 insertions(+), 237 deletions(-) create mode 100644 projects/hrr/tests/docker/Dockerfile create mode 100644 projects/hrr/tests/scripts/hrr_dev.py create mode 100644 projects/hrr/tests/scripts/test_hrr_dev.py diff --git a/.github/skip-ci-config.toml b/.github/skip-ci-config.toml index 5bcf616dd4e7..c9e1f1f82e8f 100644 --- a/.github/skip-ci-config.toml +++ b/.github/skip-ci-config.toml @@ -45,6 +45,14 @@ common = [ # Changes to experimental code do not run standard build/test workflows. "experimental/*", + + # HRR suite selection and its developer entry point. A source change under + # projects/hrr still builds. + ".github/workflows/hrr-ci.yml", + ".github/workflows/hrr-gpu.yml", + "projects/hrr/tests/config/ci.yaml", + "projects/hrr/tests/docker/*", + "projects/hrr/tests/scripts/hrr_dev.py", ] # Patterns that skip CI only on Linux (e.g., Windows-only projects) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index a992c05557de..5c845af3370a 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -25,6 +25,11 @@ on: description: "Comma-separated GPU families (TheRock keys, e.g. gfx94x,gfx110x). Empty: the families ci.yaml selects." type: string default: "" + backend: + description: "simulator runs rocJitsu on ubuntu-24.04. gpu runs the real runners." + type: choice + options: [simulator, gpu] + default: simulator ignore_test_config: description: "Run the cases the suite config skips" type: boolean @@ -61,6 +66,8 @@ jobs: run_cpu: ${{ steps.decide.outputs.run_cpu }} run_gpu: ${{ steps.decide.outputs.run_gpu }} families: ${{ steps.decide.outputs.families }} + simulator_families: ${{ steps.decide.outputs.simulator_families }} + gpu_families: ${{ steps.decide.outputs.gpu_families }} skipped_families: ${{ steps.decide.outputs.skipped_families }} reason: ${{ steps.decide.outputs.reason }} rocm_sdk_version: ${{ steps.decide.outputs.rocm_sdk_version }} @@ -123,11 +130,13 @@ jobs: # Empty when the head repository was deleted, which counts as a fork. HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} FAMILIES: ${{ inputs.families }} + BACKEND: ${{ inputs.backend }} run: >- python projects/hrr/tests/scripts/hrr_ci_configure.py --event "${EVENT}" --repo "${REPO}" --pr-number "${PR_NUMBER}" --before "${BEFORE}" --after "${AFTER}" --labels "${LABELS}" --action "${ACTION}" --label "${LABEL}" --head-repo "${HEAD_REPO}" --families "${FAMILIES}" + --backend "${BACKEND}" --therock-dir therock codegen: @@ -323,9 +332,11 @@ jobs: needs: configure if: ${{ needs.configure.outputs.run_gpu == 'true' }} uses: ./.github/workflows/hrr-gpu.yml - # No secrets: the GPU workflow reads public artifacts with the job token. + # Note: not using 'secrets: inherit' here; no secrets are needed. with: families: ${{ needs.configure.outputs.families }} + simulator_families: ${{ needs.configure.outputs.simulator_families }} + gpu_families: ${{ needs.configure.outputs.gpu_families }} ignore_test_config: ${{ inputs.ignore_test_config || false }} summary: @@ -364,7 +375,7 @@ jobs: continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: hrr-base-${{ github.sha }} + name: hrr-base-${{ github.run_id }} path: base - name: Summarize diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 0c762868bad4..52045d96fd47 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -13,14 +13,26 @@ on: workflow_call: inputs: families: - description: JSON list of {family, runner} from hrr_ci_configure.py + description: JSON list of {family, runner, backend, arch, config} from hrr_ci_configure.py type: string required: true + simulator_families: + description: Simulator rows from that list. [] skips the rocJitsu jobs. + type: string + default: "[]" + gpu_families: + description: Real-GPU rows from that list. [] skips the GPU jobs. + type: string + default: "[]" ignore_test_config: description: Run the cases the suite config skips type: boolean default: false +env: + THEROCK_PIN: c01e3a3f3f67966dd35612cf363b49964f33cd47 + BUILD_IMAGE: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:cf4f6d9909056906e4a1c0bc4c245658a0af4c6a6174082a86adfa2d518b7aeb + permissions: contents: read @@ -63,7 +75,26 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: python3 projects/hrr/tests/scripts/hrr_capture_build.py fetch-base + - name: Capture cache key + id: cache_key + run: >- + python3 projects/hrr/tests/scripts/hrr_capture_build.py cache-key + --therock-pin "${THEROCK_PIN}" --image "${BUILD_IMAGE}" + + # Exact key. A prefix restore could pair this tree with another base + # or another targets.arch. + - name: Restore capture build + id: capture_cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + build/capture-lib + build/hrr/playback/hrr-playback + build/hrr/tests/integration + key: hrr-capture-v1-${{ steps.cache_key.outputs.digest }} + - name: Build + if: steps.capture_cache.outputs.cache-hit != 'true' run: python3 projects/hrr/tests/scripts/hrr_capture_build.py build - name: Verify @@ -73,7 +104,7 @@ jobs: if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: hrr-base-${{ github.sha }} + name: hrr-base-${{ github.run_id }} path: build/base.json if-no-files-found: error retention-days: 7 @@ -81,7 +112,7 @@ jobs: - name: Upload build uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: hrr-capture-${{ github.sha }} + name: hrr-capture-${{ github.run_id }} path: | build/base.json build/capture-lib/ @@ -91,15 +122,115 @@ jobs: if-no-files-found: error retention-days: 7 + simulator: + name: rocJitsu (${{ matrix.family }}) + needs: build + if: inputs.simulator_families != '[]' + runs-on: ubuntu-24.04 + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.simulator_families) }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + emulation/rocjitsu + projects/hrr/tests + shared/machine-readable-isa + + - name: Checkout TheRock artifact tools + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ROCm/TheRock + ref: ${{ env.THEROCK_PIN }} # 2026-10-12 + path: therock + sparse-checkout: | + build_tools + cmake/therock_amdgpu_targets.cmake + requirements.txt + sparse-checkout-cone-mode: false + + - name: Install rocJitsu build packages + shell: bash + run: | + set -euo pipefail + sudo -n apt-get update + sudo -n DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential ca-certificates clang libdrm-dev ninja-build + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install Python requirements + shell: bash + run: | + set -euo pipefail + python -m venv "${RUNNER_TEMP}/hrr-venv" + "${RUNNER_TEMP}/hrr-venv/bin/python" -m pip install --no-cache-dir \ + -r therock/requirements.txt 'cmake==3.31.10' 'PyYAML==6.0.2' + echo "${RUNNER_TEMP}/hrr-venv/bin" >> "${GITHUB_PATH}" + + - name: Build rocJitsu + shell: bash + run: | + set -euo pipefail + cmake -S emulation/rocjitsu -B build/rocjitsu -G Ninja -DCMAKE_BUILD_TYPE=Release + cmake --build build/rocjitsu --target rocjitsu_bin -j + echo "${GITHUB_WORKSPACE}/build/rocjitsu/tools/rocjitsu" >> "${GITHUB_PATH}" + + - name: Download build + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: hrr-capture-${{ github.run_id }} + path: build + + - name: Install base ROCm artifacts and the capture-enabled amdhip64 + env: + GITHUB_TOKEN: ${{ github.token }} + run: python projects/hrr/tests/scripts/hrr_capture_build.py install-base + + - name: Run integration suite + env: + FAMILY: ${{ matrix.family }} + ARCH: ${{ matrix.arch }} + # The config path stays data. The shell does not assemble the command. + LAUNCHER: ${{ format('rocjitsu --daemon --config {0} --', matrix.config) }} + HRR_TEST_IGNORE_CONFIG: ${{ inputs.ignore_test_config && '1' || '' }} + shell: bash + run: >- + python projects/hrr/tests/scripts/run_catch2.py + --xml "integration-${FAMILY}.xml" + --timeout 1200 + --config projects/hrr/tests/config/integration.yaml + --os linux --arch "${ARCH}" + --launcher "${LAUNCHER}" + -- build/hrr/tests/integration/hrr-integration-tests + + - name: Upload results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: results-integration-${{ matrix.family }} + path: integration-${{ matrix.family }}.xml + if-no-files-found: ignore + retention-days: 7 + test: name: Integration (${{ matrix.family }}) needs: build + if: inputs.gpu_families != '[]' runs-on: ${{ matrix.runner }} timeout-minutes: 45 strategy: fail-fast: false matrix: - include: ${{ fromJSON(inputs.families) }} + include: ${{ fromJSON(inputs.gpu_families) }} # The image and options TheRock's test_component.yml uses for GPU tests. # Running in a container also gives this job the build job's workspace # path, which the test binaries have compiled in. @@ -159,7 +290,7 @@ jobs: - name: Download build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: hrr-capture-${{ github.sha }} + name: hrr-capture-${{ github.run_id }} path: build - name: Install base ROCm artifacts and the capture-enabled amdhip64 diff --git a/.github/workflows/therock-multi-arch-ci.yml b/.github/workflows/therock-multi-arch-ci.yml index a4eeff0e326f..ad87b89060a7 100644 --- a/.github/workflows/therock-multi-arch-ci.yml +++ b/.github/workflows/therock-multi-arch-ci.yml @@ -114,6 +114,7 @@ jobs: - name: Checkout repository config uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + persist-credentials: false sparse-checkout: | .github/repos-config.json .github/scripts/compute_build_stages.py @@ -122,6 +123,7 @@ jobs: - name: Checkout TheRock for configure script uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + persist-credentials: false repository: ROCm/TheRock ref: c01e3a3f3f67966dd35612cf363b49964f33cd47 # 2026-10-12 path: _therock @@ -164,10 +166,13 @@ jobs: # project is disabled on Windows in the topology. - name: Compute build-stage allowlist id: build_stages + env: + CHANGED_PROJECTS: ${{ steps.configure-pr.outputs.changed_projects || steps.configure-manual.outputs.changed_projects }} + RUN_ALL_TESTS: ${{ steps.configure-pr.outputs.run_all_tests || steps.configure-manual.outputs.run_all_tests }} run: | python3 .github/scripts/compute_build_stages.py \ - --changed-projects "${{ steps.configure-pr.outputs.changed_projects || steps.configure-manual.outputs.changed_projects }}" \ - --run-all-tests "${{ steps.configure-pr.outputs.run_all_tests || steps.configure-manual.outputs.run_all_tests }}" \ + --changed-projects "${CHANGED_PROJECTS}" \ + --run-all-tests "${RUN_ALL_TESTS}" \ --therock-path "_therock" setup: @@ -187,6 +192,7 @@ jobs: # other stage outright (no build, no artifact copy). Empty = build all, # the fail-safe for fan-out / unknown-project / CI-infra changes. build_stages: ${{ needs.configure.outputs.build_stages }} + changed_projects: ${{ needs.configure.outputs.changed_projects }} prebuilt_stages: ${{ inputs.prebuilt_stages || '' }} baseline_run_id: ${{ inputs.baseline_run_id || '' }} build_python_packages: ${{ inputs.build_python_packages || false }} @@ -207,7 +213,7 @@ jobs: "repository": "${{ github.repository }}", "ref": "${{ github.sha }}", "base_sha": "${{ github.event.pull_request.base.sha || github.event.before }}", - "head_sha": "${{ github.sha }}", + "head_sha": "${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}", "pr_number": ${{ github.event.pull_request.number || 'null' }}, "event_name": "${{ github.event_name }}", "skip_ci_config": ".github/skip-ci-config.toml", @@ -283,6 +289,7 @@ jobs: - name: Checkout TheRock repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + persist-credentials: false repository: "ROCm/TheRock" ref: c01e3a3f3f67966dd35612cf363b49964f33cd47 # 2026-10-12 sparse-checkout: build_tools/github_actions diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index fa4d0d09ae92..066cfe6115a4 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -33,7 +33,6 @@ paths: - projects/hrr/tests/CMakeLists.txt - projects/hrr/tests/include/ - projects/hrr/tests/integration/ - - projects/hrr/tests/config/ci.yaml - projects/hrr/tests/config/integration.yaml - projects/hrr/tests/scripts/run_catch2.py - projects/hrr/tests/scripts/hrr_test_config.py @@ -43,6 +42,10 @@ paths: - projects/hrr/tests/scripts/hrr_ci_report.py - .github/workflows/hrr-ci.yml - .github/workflows/hrr-gpu.yml + # Runs the suite without changing the capture binary. The capture cache key + # does not include these paths. + suite: + - projects/hrr/tests/config/ci.yaml # Ignored by both lists; hrr-skills.yml covers these. excluded: - projects/hrr/skills/ @@ -52,6 +55,28 @@ paths: # capture path. capture_label: ci:hrr-capture +# simulator runs rocJitsu on ubuntu-24.04. gpu runs TheRock's real runners. +# workflow_dispatch passes its backend input. Any other event uses default, +# unless the event is listed under backend.gpu.triggers. +backend: + default: simulator + gpu: + triggers: [] + +# Default simulator jobs. These do not consult families.*.enabled: gfx125x +# stays disabled for the real MI455 runner and still runs here. +simulator: + runner: ubuntu-24.04 + default_families: [gfx950, gfx125x] + configs: + gfx94x: emulation/rocjitsu/configs/gfx942_cdna3_kmd.json + gfx950: emulation/rocjitsu/configs/gfx950_mi355x_kmd_2gpu.json + gfx90a: emulation/rocjitsu/configs/gfx90a_mi210_kmd.json + gfx110x: emulation/rocjitsu/configs/gfx1100_w7900.json + gfx1151: emulation/rocjitsu/configs/gfx1151.json + gfx120x: emulation/rocjitsu/configs/gfx1201_r9700.json + gfx125x: emulation/rocjitsu/configs/gfx1250_mi455x_kmd_4gpu.json + # The GPU families the suite runs on, as TheRock family keys # (build_tools/github_actions/amdgpu_family_matrix.py). The integration binary # is built for targets.arch in integration.yaml; the configure job fails unless @@ -74,12 +99,14 @@ capture_label: ci:hrr-capture # runner pool and the summary waits for all of them, so to reduce GPU load or # take a pool out, set enabled: false or narrow triggers here. # -# Only gfx94x runs by default for now; every other family is disabled but -# still listed, so a `ci:gfx...` label or the workflow_dispatch `families` -# input can request it. Delete an `enabled: false` line to make a family run -# by default again. targets.arch in integration.yaml stays wide on purpose: -# configure fails unless it covers every family listed here, and a requested -# family needs its architecture compiled into the binary. +# Real GPUs: only gfx94x is enabled. The other families stay listed and +# disabled, so a ci:gfx... label or the workflow_dispatch families input can +# still request one when the backend is gpu. Delete an `enabled: false` line +# to make that family eligible for the GPU backend; it does not add a +# simulator job. Simulator membership is simulator.default_families. +# targets.arch in integration.yaml stays wide on purpose: configure fails +# unless it covers every family listed here, and a requested family needs its +# architecture compiled into the binary. families: gfx90a: enabled: false diff --git a/projects/hrr/tests/docker/Dockerfile b/projects/hrr/tests/docker/Dockerfile new file mode 100644 index 000000000000..b30596b8662d --- /dev/null +++ b/projects/hrr/tests/docker/Dockerfile @@ -0,0 +1,18 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Local HRR development image. The host only needs Docker. +# CMake, Ninja and Python 3 come from the TheRock manylinux build image +# that hrr-gpu.yml pins. This adds PyYAML for the HRR scripts, the GitHub +# CLI that hrr_capture_build.py uses to fetch a base, and GCC 13, the +# minimum emulation/rocjitsu/docs/building.md asks for. +FROM ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:cf4f6d9909056906e4a1c0bc4c245658a0af4c6a6174082a86adfa2d518b7aeb + +USER root +RUN python3 -m pip install --no-cache-dir 'PyYAML==6.0.2' \ + && curl -fsSL https://cli.github.com/packages/rpm/gh-cli.repo -o /etc/yum.repos.d/gh-cli.repo \ + && dnf install -y gh gcc-toolset-13 \ + && dnf clean all \ + && groupadd -f video +ENV PATH=/opt/rh/gcc-toolset-13/root/usr/bin:${PATH} +WORKDIR /workspace diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index 8a8db4c6e70c..e33a59cfa252 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -27,6 +27,7 @@ from __future__ import annotations import argparse +import hashlib import json import os import re @@ -48,19 +49,30 @@ # Artifacts needed to build CLR and projects/hrr. BUILD_ARTIFACTS = ( - "amd-llvm_run", "amd-llvm_lib", "amd-llvm_dev", - "base_run", "base_lib", "base_dev", - "sysdeps_lib", "sysdeps_dev", - "core-runtime_run", "core-runtime_lib", "core-runtime_dev", - "core-hip_run", "core-hip_lib", "core-hip_dev", + "amd-llvm_run", + "amd-llvm_lib", + "amd-llvm_dev", + "base_run", + "base_lib", + "base_dev", + "sysdeps_lib", + "sysdeps_dev", + "core-runtime_run", + "core-runtime_lib", + "core-runtime_dev", + "core-hip_run", + "core-hip_lib", + "core-hip_dev", ) # Artifacts the GPU suite loads at run time. RUNTIME_ARTIFACTS = ( "amd-llvm_lib", - "base_run", "base_lib", + "base_run", + "base_lib", "sysdeps_lib", - "core-runtime_run", "core-runtime_lib", + "core-runtime_run", + "core-runtime_lib", "core-hip_lib", ) @@ -78,6 +90,20 @@ # What the GPU suite runs. The host unit suite is built and run by hrr-ci.yml. HRR_TARGETS = ("hrr-playback", "hrr-integration-tests", "hrr-test-dispatch-tool") +# What the capture cache key hashes. ci.yaml is a suite-selection file and is +# deliberately absent: a family or backend edit must restore this build. +CACHE_SOURCE_PATHS = ( + "projects/clr", + "projects/hip", + "projects/hrr/CMakeLists.txt", + "projects/hrr/include", + "projects/hrr/playback", + "projects/hrr/tests/CMakeLists.txt", + "projects/hrr/tests/include", + "projects/hrr/tests/integration", + "projects/hrr/tests/scripts/hrr_capture_build.py", +) + # Uploaded by hrr-gpu.yml's build job, relative to --build. EXECUTABLES = ( "hrr/playback/hrr-playback", @@ -93,7 +119,9 @@ def run(cmd: Sequence[str]) -> None: def pick_base_run(finder_json: dict) -> dict: """The base run from find_latest_artifacts.py --json output.""" if finder_json.get("status") != "found" or not finder_json.get("results"): - raise SystemExit(f"no {BASE_BRANCH} run of {BASE_WORKFLOW} has the needed artifacts") + raise SystemExit( + f"no {BASE_BRANCH} run of {BASE_WORKFLOW} has the needed artifacts" + ) result = finder_json["results"][0] return { "repo": result["github_repository_name"], @@ -106,28 +134,42 @@ def pick_base_run(finder_json: dict) -> dict: def find_base_run(therock: Path) -> dict: cmd = [ - sys.executable, therock / "build_tools" / "find_latest_artifacts.py", - "--repo", BASE_REPO, - "--workflow", BASE_WORKFLOW, - "--branch", BASE_BRANCH, - "--artifact-group", BASE_ARTIFACT_GROUP, + sys.executable, + therock / "build_tools" / "find_latest_artifacts.py", + "--repo", + BASE_REPO, + "--workflow", + BASE_WORKFLOW, + "--branch", + BASE_BRANCH, + "--artifact-group", + BASE_ARTIFACT_GROUP, "--json", ] for artifact in BUILD_ARTIFACTS: cmd += ["--require-artifact", f"{artifact}_generic"] print("+ " + " ".join(str(c) for c in cmd), flush=True) # Only stdout is captured: it carries the JSON, stderr the finder's log. - out = subprocess.run([str(c) for c in cmd], check=True, stdout=subprocess.PIPE, text=True) + out = subprocess.run( + [str(c) for c in cmd], check=True, stdout=subprocess.PIPE, text=True + ) return pick_base_run(json.loads(out.stdout)) -def fetch_command(therock: Path, base: dict, dist: Path, artifacts: Sequence[str]) -> list: +def fetch_command( + therock: Path, base: dict, dist: Path, artifacts: Sequence[str] +) -> list: return [ - sys.executable, therock / "build_tools" / "fetch_artifacts.py", - "--run-id", base["run_id"], - "--run-github-repo", base["repo"], - "--artifact-group", base["artifact_group"], - "--output-dir", dist, + sys.executable, + therock / "build_tools" / "fetch_artifacts.py", + "--run-id", + base["run_id"], + "--run-github-repo", + base["repo"], + "--artifact-group", + base["artifact_group"], + "--output-dir", + dist, "--flatten", *artifacts, ] @@ -137,7 +179,13 @@ def clr_configure_command(src: Path, build: Path, dist: Path) -> list: # HIP_HRR_CAPTURE_ENABLED is what compiles capture into amdhip64; CLR has # no CMake option for it. return [ - "cmake", "-S", src / "projects" / "clr", "-B", build / "clr", "-GNinja", "-Wno-dev", + "cmake", + "-S", + src / "projects" / "clr", + "-B", + build / "clr", + "-GNinja", + "-Wno-dev", "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_CXX_FLAGS=-DHIP_HRR_CAPTURE_ENABLED", f"-DCMAKE_PREFIX_PATH={dist}", @@ -160,7 +208,13 @@ def hrr_configure_command(src: Path, build: Path, dist: Path, archs: str) -> lis # One compiler for C, C++ and HIP, as TheRock builds HRR: host objects from # the system compiler are not position independent, and the HIP link is. return [ - "cmake", "-S", src / "projects" / "hrr", "-B", build / "hrr", "-GNinja", "-Wno-dev", + "cmake", + "-S", + src / "projects" / "hrr", + "-B", + build / "hrr", + "-GNinja", + "-Wno-dev", "-DCMAKE_BUILD_TYPE=Release", f"-DCMAKE_PREFIX_PATH={dist}", f"-DROCM_PATH={dist}", @@ -221,7 +275,9 @@ def linkage_errors(ldd_output: str, lib_dir: Path) -> list[str]: def check_linkage(binary: Path, lib_dir: Path) -> int: - out = subprocess.run(["ldd", str(binary)], check=True, capture_output=True, text=True).stdout + out = subprocess.run( + ["ldd", str(binary)], check=True, capture_output=True, text=True + ).stdout errors = linkage_errors(out, lib_dir) for error in errors: print(f"::error::{binary.name}: {error}") @@ -263,14 +319,23 @@ def cmd_build(args: argparse.Namespace, runner: Runner = run) -> int: def cmd_verify(args: argparse.Namespace) -> int: exe = args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests" - run([ - sys.executable, args.src / "projects/hrr/tests/scripts/hrr_test_config.py", "check", - "--binary", exe, - "--config", args.src / "projects/hrr/tests/config/integration.yaml", - ]) + run( + [ + sys.executable, + args.src / "projects/hrr/tests/scripts/hrr_test_config.py", + "check", + "--binary", + exe, + "--config", + args.src / "projects/hrr/tests/config/integration.yaml", + ] + ) objdump = args.dist / "lib" / "llvm" / "bin" / "llvm-objdump" out = subprocess.run( - [str(objdump), "--offloading", str(exe)], check=True, capture_output=True, text=True + [str(objdump), "--offloading", str(exe)], + check=True, + capture_output=True, + text=True, ).stdout missing = missing_targets(args.archs, out) print("embedded: " + " ".join(sorted(embedded_targets(out)))) @@ -280,6 +345,58 @@ def cmd_verify(args: argparse.Namespace) -> int: return check_linkage(exe, args.dist / "lib") +def _tree_digest(path: Path) -> str: + digest = hashlib.sha256() + if path.is_file(): + digest.update(path.read_bytes()) + return digest.hexdigest() + if path.is_dir(): + files = sorted(item for item in path.rglob("*") if item.is_file()) + for item in files: + digest.update(item.relative_to(path).as_posix().encode()) + digest.update(b"\0") + digest.update(item.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def cache_key_digest( + src: Path, base: dict, therock_pin: str, image: str, archs: str +) -> str: + """Content key for the capture build. Not the merge SHA.""" + digest = hashlib.sha256() + header = "\n".join( + [ + f"repo={base['repo']}", + f"run_id={base['run_id']}", + f"commit={base['commit']}", + f"artifact_group={base['artifact_group']}", + f"therock={therock_pin}", + f"image={image}", + f"archs={archs}", + ] + ) + digest.update(header.encode()) + for rel in CACHE_SOURCE_PATHS: + digest.update(b"\n") + digest.update(rel.encode()) + digest.update(b"\n") + digest.update(_tree_digest(src / rel).encode()) + return digest.hexdigest() + + +def cmd_cache_key(args: argparse.Namespace) -> int: + base = json.loads(args.base_json.read_text()) + digest = cache_key_digest(args.src, base, args.therock_pin, args.image, args.archs) + line = f"digest={digest}\n" + output = os.environ.get("GITHUB_OUTPUT", "") + if output: + with open(output, "a", encoding="utf-8") as handle: + handle.write(line) + sys.stdout.write(line) + return 0 + + def restore_executable_bits(build: Path) -> None: """Artifact downloads drop the executable bit.""" for rel in EXECUTABLES: @@ -295,13 +412,17 @@ def cmd_install_base(args: argparse.Namespace, runner: Runner = run) -> int: copy_amdhip64(args.capture_lib, args.dist / "lib") restore_executable_bits(args.build) return check_linkage( - args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests", args.dist / "lib" + args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests", + args.dist / "lib", ) def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("command", choices=("fetch-base", "build", "verify", "install-base")) + parser.add_argument( + "command", + choices=("fetch-base", "build", "verify", "install-base", "cache-key"), + ) parser.add_argument("--therock", type=Path, default=Path("therock")) parser.add_argument("--src", type=Path, default=Path.cwd()) parser.add_argument("--dist", type=Path, default=Path("dist")) @@ -309,13 +430,18 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--capture-lib", type=Path, default=Path("build/capture-lib")) parser.add_argument("--base-json", type=Path, default=Path("build/base.json")) parser.add_argument( - "--archs", default="", + "--archs", + default="", help=f"Semicolon-separated GPU targets; default targets.arch of {INTEGRATION_CONFIG}", ) + parser.add_argument( + "--therock-pin", default="", help="TheRock commit the build tools came from" + ) + parser.add_argument("--image", default="", help="Build container image digest") args = parser.parse_args(argv) for name in ("therock", "src", "dist", "build", "capture_lib", "base_json"): setattr(args, name, getattr(args, name).resolve()) - if args.command in ("build", "verify") and not args.archs: + if args.command in ("build", "verify", "cache-key") and not args.archs: config = hrr_test_config.load(args.src / INTEGRATION_CONFIG) args.archs = ";".join(config["targets"]["arch"]) return args @@ -328,6 +454,7 @@ def main(argv: list[str] | None = None) -> int: "build": cmd_build, "verify": cmd_verify, "install-base": cmd_install_base, + "cache-key": cmd_cache_key, }[args.command](args) diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index 9c963dd7a076..083a95a50da3 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -7,22 +7,28 @@ run_cpu "true" when the generated-source, script and unit jobs run. run_gpu "true" when the capture-enabled build and the GPU suite run. - families JSON list of {"family", "runner"} for the GPU suite. + families JSON list of {family, runner, backend, arch, config}. + simulator_families + The subset whose backend is simulator. + gpu_families The subset whose backend is gpu. skipped_families JSON list of {"family", "reason"}: the other ci.yaml - families, when the GPU suite runs. + families, when the suite runs. reason One line explaining the decision, shown in the summary. rocm_sdk_version The ROCm SDK the unit job installs. -The policy lives in projects/hrr/tests/config/ci.yaml. The GPU suite needs an +The policy lives in projects/hrr/tests/config/ci.yaml. The suite needs an amdhip64 with HRR capture compiled in, which only the HRR GPU workflow builds. -It runs when a change touches a capture path, on any pull request with the -capture label, nightly and on workflow_dispatch. - -It runs on every enabled family whose triggers include the event. A ci:gfx... -label adds that family, the same label TheRock CI uses; the workflow_dispatch -families input replaces the selection. Both can name a disabled family. The -runner labels come from TheRock's family matrix unless ci.yaml overrides them; -a family takes TheRock's multi-GPU runner only on its multi_gpu_triggers. +It runs when a change touches a capture or suite path, on any pull request +with the capture label, nightly and on workflow_dispatch. + +The default backend is rocJitsu (simulator). Its jobs are +simulator.default_families, not the families marked enabled. A ci:gfx... label +adds a family, and the workflow_dispatch families input replaces the +selection. A simulator family with no config is an error. The GPU backend +runs every enabled family whose triggers include the event, on TheRock's +runner unless ci.yaml overrides it, and takes the multi-GPU runner only on +its multi_gpu_triggers. workflow_dispatch uses its backend input; any other +event uses backend.default unless the event is in backend.gpu.triggers. """ from __future__ import annotations @@ -64,13 +70,29 @@ class Family: multi_gpu_triggers: tuple[str, ...] = () +@dataclass(frozen=True) +class Backend: + default: str + gpu_triggers: tuple[str, ...] + + +@dataclass(frozen=True) +class Simulator: + runner: str + default_families: tuple[str, ...] + configs: dict[str, str] + + @dataclass(frozen=True) class Config: rocm_sdk_version: str cpu_paths: tuple[str, ...] capture_paths: tuple[str, ...] + suite_paths: tuple[str, ...] excluded_paths: tuple[str, ...] capture_label: str + backend: Backend + simulator: Simulator families: dict[str, Family] @@ -83,7 +105,9 @@ class Decision: skipped: list[dict] = field(default_factory=list) -def _mapping(where: str, value, required: Iterable[str], optional: Iterable[str] = ()) -> dict: +def _mapping( + where: str, value, required: Iterable[str], optional: Iterable[str] = () +) -> dict: if not isinstance(value, dict): raise ConfigError(f"{where}: expected a mapping") unknown = sorted(set(value) - set(required) - set(optional)) @@ -113,7 +137,9 @@ def _bool(where: str, value) -> bool: return value -def _triggers(where: str, value) -> tuple[str, ...]: +def _triggers(where: str, value, allow_empty: bool = False) -> tuple[str, ...]: + if allow_empty and isinstance(value, list) and not value: + return () triggers = _strings(where, value) for trigger in triggers: if trigger not in EVENTS: @@ -123,6 +149,47 @@ def _triggers(where: str, value) -> tuple[str, ...]: return triggers +def _backend(value) -> Backend: + value = _mapping("backend", value, ("default", "gpu")) + default = _string("backend.default", value["default"]) + if default not in ("simulator", "gpu"): + raise ConfigError( + f"backend.default: expected simulator or gpu, got {default!r}" + ) + gpu = _mapping("backend.gpu", value["gpu"], ("triggers",)) + return Backend( + default, _triggers("backend.gpu.triggers", gpu["triggers"], allow_empty=True) + ) + + +def _simulator(value, families: dict) -> Simulator: + value = _mapping("simulator", value, ("runner", "default_families", "configs")) + default_families = _strings("simulator.default_families", value["default_families"]) + unknown = [key for key in default_families if key not in families] + if unknown: + raise ConfigError( + f"simulator.default_families: {', '.join(unknown)} not in families" + ) + configs = value["configs"] + if not isinstance(configs, dict) or not configs: + raise ConfigError("simulator.configs: expected a mapping") + unknown = [key for key in configs if key not in families] + if unknown: + raise ConfigError(f"simulator.configs: {', '.join(unknown)} not in families") + parsed = { + _string("simulator.configs", key): _string(f"simulator.configs.{key}", path) + for key, path in configs.items() + } + missing = [key for key in default_families if key not in parsed] + if missing: + raise ConfigError(f"simulator.configs: missing {', '.join(missing)}") + return Simulator( + runner=_string("simulator.runner", value["runner"]), + default_families=default_families, + configs=parsed, + ) + + def _family(where: str, value) -> Family: value = _mapping( where, value or {}, (), ("enabled", "triggers", "runner", "multi_gpu_triggers") @@ -133,31 +200,46 @@ def _family(where: str, value) -> Family: runner=_string(f"{where}.runner", value["runner"]) if "runner" in value else "", multi_gpu_triggers=( _triggers(f"{where}.multi_gpu_triggers", value["multi_gpu_triggers"]) - if "multi_gpu_triggers" in value else () + if "multi_gpu_triggers" in value + else () ), ) def parse_config(data) -> Config: data = _mapping( - "top level", data, ("version", "rocm_sdk_version", "paths", "capture_label", "families") + "top level", + data, + ( + "version", + "rocm_sdk_version", + "paths", + "capture_label", + "backend", + "simulator", + "families", + ), ) if type(data["version"]) is not int or data["version"] != 1: raise ConfigError(f"version: expected 1, got {data['version']!r}") - paths = _mapping("paths", data["paths"], ("cpu", "capture", "excluded")) + paths = _mapping("paths", data["paths"], ("cpu", "capture", "suite", "excluded")) families = data["families"] if not isinstance(families, dict) or not families: raise ConfigError("families: expected a mapping with at least one family") + parsed_families = { + _string("families", key): _family(f"families.{key}", value) + for key, value in families.items() + } return Config( rocm_sdk_version=_string("rocm_sdk_version", data["rocm_sdk_version"]), cpu_paths=_strings("paths.cpu", paths["cpu"]), capture_paths=_strings("paths.capture", paths["capture"]), + suite_paths=_strings("paths.suite", paths["suite"]), excluded_paths=_strings("paths.excluded", paths["excluded"]), capture_label=_string("capture_label", data["capture_label"]), - families={ - _string("families", key): _family(f"families.{key}", value) - for key, value in families.items() - }, + backend=_backend(data["backend"]), + simulator=_simulator(data["simulator"], parsed_families), + families=parsed_families, ) @@ -233,6 +315,10 @@ def touches_capture(paths: Iterable[str], config: Config) -> bool: return any(_matches(p, config.capture_paths) for p in relevant_paths(paths, config)) +def touches_suite(paths: Iterable[str], config: Config) -> bool: + return any(_matches(p, config.suite_paths) for p in relevant_paths(paths, config)) + + def labelled_families(labels: Iterable[str], matrix: dict) -> list[str]: """Family keys named by ci:gfx... labels, e.g. ci:gfx110x or ci:gfx110X-all.""" wanted = [] @@ -240,7 +326,7 @@ def labelled_families(labels: Iterable[str], matrix: dict) -> list[str]: label = label.strip().lower() if not label.startswith("ci:gfx"): continue - name = label[len("ci:"):] + name = label[len("ci:") :] for key, config in matrix.items(): linux = config.get("linux", {}) if name in (key.lower(), str(linux.get("family", "")).lower()): @@ -270,6 +356,32 @@ def runner_for( return linux.get("test-runs-on", "") +def resolve_backend(event: str, config: Config, backend_input: str) -> str: + """workflow_dispatch uses its input. Any other event uses the default, + unless that event is listed under backend.gpu.triggers.""" + if event == "workflow_dispatch" and backend_input: + if backend_input not in ("simulator", "gpu"): + raise ConfigError(f"backend {backend_input!r} is not simulator or gpu") + return backend_input + if event in config.backend.gpu_triggers: + return "gpu" + return config.backend.default + + +def _arch(matrix: dict, key: str) -> str: + return ",".join(family_targets(matrix.get(key, {}).get("linux", {}))) + + +def _job(family: str, runner: str, backend: str, arch: str, config: str = "") -> dict: + return { + "family": family, + "runner": runner, + "backend": backend, + "arch": arch, + "config": config, + } + + def select_families( event: str, config: Config, @@ -290,7 +402,8 @@ def select_families( chosen = set(requested) else: chosen = { - key for key, family in config.families.items() + key + for key, family in config.families.items() if family.enabled and event in family.triggers } chosen |= set(labelled_families(labels, matrix)) & set(config.families) @@ -312,7 +425,48 @@ def select_families( if not runner: skipped.append({"family": key, "reason": NO_RUNNER}) continue - families.append({"family": key, "runner": runner}) + families.append(_job(key, runner, "gpu", _arch(matrix, key))) + return families, skipped + + +def select_simulator( + config: Config, + requested: list[str], + labels: list[str], + matrix: dict, +) -> tuple[list[dict], list[dict]]: + """rocJitsu families. default_families, unless the dispatch input replaces + them. A ci:gfx... label adds a family. A family with no config is an error.""" + if requested: + unknown = [key for key in requested if key not in config.families] + if unknown: + raise ConfigError( + f"requested families {', '.join(unknown)} are not in ci.yaml" + f" (known: {', '.join(config.families)})" + ) + chosen = set(requested) + else: + chosen = set(config.simulator.default_families) + chosen |= set(labelled_families(labels, matrix)) & set(config.families) + missing = sorted(key for key in chosen if key not in config.simulator.configs) + if missing: + raise ConfigError(f"simulator has no config for {', '.join(missing)}") + + families, skipped = [], [] + for key in config.families: + if key not in chosen: + reason = "not requested" if requested else "not a default simulator family" + skipped.append({"family": key, "reason": reason}) + continue + families.append( + _job( + key, + config.simulator.runner, + "simulator", + _arch(matrix, key), + config.simulator.configs[key], + ) + ) return families, skipped @@ -326,6 +480,7 @@ def decide( matrix: dict, pick_weighted: Callable[[list, str], str], added_label: str | None = None, + backend: str = "", ) -> Decision: """paths is None when the change set could not be listed in full. @@ -335,7 +490,9 @@ def decide( if event not in EVENTS: return Decision(False, False, f"event {event} does not run HRR CI") if added_label is not None and not label_matters(added_label, config): - return Decision(False, False, f"the label {added_label} does not change what HRR CI runs") + return Decision( + False, False, f"the label {added_label} does not change what HRR CI runs" + ) if event in ("schedule", "workflow_dispatch"): run_cpu, capture, why = True, True, f"{event} run" @@ -343,14 +500,25 @@ def decide( run_cpu, capture, why = True, True, "change set too large to list" else: run_cpu = touches_cpu(paths, config) - capture = touches_capture(paths, config) - why = "the change touches capture or replay" if capture else ( - "the change touches HRR" if run_cpu else "the change does not touch HRR" - ) - - labelled = event == "pull_request" and config.capture_label.lower() in {l.strip().lower() for l in labels} + capture = touches_capture(paths, config) or touches_suite(paths, config) + if touches_capture(paths, config): + why = "the change touches capture or replay" + elif touches_suite(paths, config): + why = "the change touches the suite config" + elif run_cpu: + why = "the change touches HRR" + else: + why = "the change does not touch HRR" + + labelled = event == "pull_request" and config.capture_label.lower() in { + l.strip().lower() for l in labels + } if labelled and not capture: - run_cpu, capture, why = True, True, f"the pull request is labelled {config.capture_label}" + run_cpu, capture, why = ( + True, + True, + f"the pull request is labelled {config.capture_label}", + ) # Capture paths that the CPU list omits (hip_context.cpp, hip_code_object.*) # still need the host unit suite. @@ -358,14 +526,23 @@ def decide( if not capture: return Decision(run_cpu, False, why) if is_fork: - return Decision(run_cpu, False, why + "; GPU suite skipped for a fork pull request") + return Decision( + run_cpu, False, why + "; GPU suite skipped for a fork pull request" + ) - families, skipped = select_families(event, config, requested, labels, matrix, pick_weighted) + if resolve_backend(event, config, backend) == "simulator": + families, skipped = select_simulator(config, requested, labels, matrix) + else: + families, skipped = select_families( + event, config, requested, labels, matrix, pick_weighted + ) no_runner = [s["family"] for s in skipped if s["reason"] == NO_RUNNER] if no_runner: why += f"; no runner for {', '.join(no_runner)}" if not families: - return Decision(run_cpu, False, why + "; no GPU family with a runner was selected") + return Decision( + run_cpu, False, why + "; no GPU family with a runner was selected" + ) return Decision(run_cpu, True, why, families, skipped) @@ -377,10 +554,16 @@ def is_fork(event: str, head_repo: str, repo: str) -> bool: def list_pr_files(repo: str, number: str) -> list[str] | None: out = subprocess.run( [ - "gh", "api", "--paginate", f"repos/{repo}/pulls/{number}/files", - "--jq", ".[] | .filename, (.previous_filename // empty)", + "gh", + "api", + "--paginate", + f"repos/{repo}/pulls/{number}/files", + "--jq", + ".[] | .filename, (.previous_filename // empty)", ], - check=True, capture_output=True, text=True, + check=True, + capture_output=True, + text=True, ).stdout files = [line for line in out.splitlines() if line] return None if len(files) >= MAX_LISTED_FILES else files @@ -391,10 +574,15 @@ def list_push_files(repo: str, before: str, after: str) -> list[str] | None: return None out = subprocess.run( [ - "gh", "api", f"repos/{repo}/compare/{before}...{after}", - "--jq", ".files[] | .filename, (.previous_filename // empty)", + "gh", + "api", + f"repos/{repo}/compare/{before}...{after}", + "--jq", + ".files[] | .filename, (.previous_filename // empty)", ], - check=True, capture_output=True, text=True, + check=True, + capture_output=True, + text=True, ).stdout files = [line for line in out.splitlines() if line] # The compare API lists at most 300 files. @@ -409,12 +597,22 @@ def load_matrix(therock_dir: Path): return families, amdgpu_family_matrix.select_weighted_label -def write_outputs(decision: Decision, rocm_sdk_version: str, output_path: str | None) -> None: +def _dump(rows: list[dict]) -> str: + return json.dumps(rows, separators=(",", ":")) + + +def write_outputs( + decision: Decision, rocm_sdk_version: str, output_path: str | None +) -> None: + simulator = [row for row in decision.families if row.get("backend") == "simulator"] + gpu = [row for row in decision.families if row.get("backend") == "gpu"] lines = [ f"run_cpu={'true' if decision.run_cpu else 'false'}", f"run_gpu={'true' if decision.run_gpu else 'false'}", - f"families={json.dumps(decision.families, separators=(',', ':'))}", - f"skipped_families={json.dumps(decision.skipped, separators=(',', ':'))}", + f"families={_dump(decision.families)}", + f"simulator_families={_dump(simulator)}", + f"gpu_families={_dump(gpu)}", + f"skipped_families={_dump(decision.skipped)}", f"reason={decision.reason}", f"rocm_sdk_version={rocm_sdk_version}", ] @@ -436,11 +634,19 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--action", default="", help="The pull_request event's action") parser.add_argument("--label", default="", help="The label a labeled event added") parser.add_argument( - "--head-repo", default="", help="owner/name of the pull request's head repository" + "--head-repo", + default="", + help="owner/name of the pull request's head repository", ) parser.add_argument( "--families", default="", help="Comma-separated family keys (workflow_dispatch)" ) + parser.add_argument( + "--backend", + default="", + choices=("", "simulator", "gpu"), + help="workflow_dispatch backend; empty uses ci.yaml", + ) parser.add_argument("--therock-dir", type=Path, required=True) parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) parser.add_argument("--integration-config", type=Path, default=INTEGRATION_CONFIG) @@ -465,9 +671,16 @@ def main(argv: list[str] | None = None) -> int: paths = list_push_files(args.repo, args.before, args.after) decision = decide( - args.event, paths, labels, is_fork(args.event, args.head_repo, args.repo), requested, - config, matrix, pick_weighted, + args.event, + paths, + labels, + is_fork(args.event, args.head_repo, args.repo), + requested, + config, + matrix, + pick_weighted, added_label=args.label if args.action == "labeled" else None, + backend=args.backend, ) except ConfigError as e: print(f"::error::{e}") diff --git a/projects/hrr/tests/scripts/hrr_dev.py b/projects/hrr/tests/scripts/hrr_dev.py new file mode 100644 index 000000000000..f62477fb623f --- /dev/null +++ b/projects/hrr/tests/scripts/hrr_dev.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Build and run the HRR suites in a local Docker container. + +The host needs Docker. CMake, Ninja, a ROCm toolchain and rocJitsu are inside +the hrr-dev image, which this script builds the first time it is missing. +`emulator` runs the integration drivers under the rocJitsu configs in +ci.yaml. `gpu` runs them on the machine's GPUs. The host unit binary has no +device code, so both commands run it once inside the container and do not +launch it under rocJitsu or on a GPU. + + python3 projects/hrr/tests/scripts/hrr_dev.py emulator + python3 projects/hrr/tests/scripts/hrr_dev.py gpu +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import hrr_ci_configure + +IMAGE = "hrr-dev" +CONTAINER = "hrr-dev" +REPO = Path(__file__).resolve().parents[4] + +# Family keys whose rocJitsu target is not the key itself. +_ARCH = { + "gfx94x": "gfx942", + "gfx103x": "gfx1030", + "gfx110x": "gfx1100", + "gfx120x": "gfx1201", + "gfx125x": "gfx1250", +} + + +@dataclass(frozen=True) +class DockerState: + image: bool + container: str | None # None, "running", or "exited" + + +def arch_for(family: str) -> str: + return _ARCH.get(family, family) + + +def emulator_jobs(config_path: Path | None = None) -> list[dict]: + config = hrr_ci_configure.load_config( + config_path or hrr_ci_configure.DEFAULT_CONFIG + ) + return [ + { + "family": family, + "arch": arch_for(family), + "config": config.simulator.configs[family], + } + for family in config.simulator.default_families + ] + + +def docker_build_command(repo: Path) -> list[str]: + return [ + "docker", + "build", + "-t", + IMAGE, + "-f", + str(repo / "projects/hrr/tests/docker/Dockerfile"), + str(repo), + ] + + +def docker_run_command(repo: Path, backend: str) -> list[str]: + command = [ + "docker", + "run", + "-d", + "--name", + CONTAINER, + "-v", + f"{repo}:{repo}", + "-w", + str(repo), + "--user", + f"{os.getuid()}:{os.getgid()}", + ] + if backend == "gpu": + command += [ + "--device", + "/dev/kfd", + "--device", + "/dev/dri", + "--group-add", + "video", + ] + command += [IMAGE, "sleep", "infinity"] + return command + + +def plan_docker(repo: Path, backend: str, state: DockerState) -> list[list[str]]: + """Commands that make the hrr-dev container ready. An existing container + is started, not recreated.""" + commands = [] + if not state.image: + commands.append(docker_build_command(repo)) + if state.container is None: + commands.append(docker_run_command(repo, backend)) + elif state.container != "running": + commands.append(["docker", "start", CONTAINER]) + return commands + + +def inspect_docker(runner=subprocess.run) -> DockerState: + image = runner( + ["docker", "image", "inspect", IMAGE], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + container = runner( + ["docker", "inspect", "-f", "{{.State.Status}}", CONTAINER], + capture_output=True, + text=True, + ) + status = container.stdout.strip() if container.returncode == 0 else None + return DockerState(image.returncode == 0, status or None) + + +def container_script(repo: Path, backend: str) -> str: + """Bash run inside the container. One rocJitsu family failing still runs + the other.""" + dist = f"{repo}/build/hrr-dev/dist" + work = f"{repo}/build/hrr-dev" + headers = f"{repo}/projects/hip/include;{repo}/projects/clr/hipamd/include" + lines = [ + "set -euo pipefail", + f'export ROCM_PATH="{dist}"', + "python3 projects/hrr/tests/scripts/hrr_capture_build.py fetch-base " + f'--dist "{dist}" --build "{work}" --base-json "{work}/base.json"', + "cmake -S projects/hrr -B build/hrr-dev/unit -G Ninja " + "-DCMAKE_BUILD_TYPE=Release " + f'-DROCM_PATH="{dist}" -DCMAKE_PREFIX_PATH="{dist}" ' + f'"-DHRR_HIP_INCLUDE_DIRS={headers}" ' + "-DHRR_BUILD_PLAYBACK=ON -DHRR_BUILD_TESTS=ON -DHRR_BUILD_INTEGRATION_TESTS=OFF", + "cmake --build build/hrr-dev/unit -j", + "python3 projects/hrr/tests/scripts/hrr_capture_build.py build " + f'--dist "{dist}" --build "{work}"', + "set +e", + "status=0", + "python3 projects/hrr/tests/scripts/run_catch2.py " + "--xml build/hrr-dev/unit.xml --config projects/hrr/tests/config/unit.yaml " + "--arch none -- build/hrr-dev/unit/tests/unit/hrr-unit-tests || status=1", + "if [ -x build/hrr-dev/unit/tests/unit/hrr-handler-tests ]; then " + "python3 projects/hrr/tests/scripts/run_catch2.py " + "--xml build/hrr-dev/handler.xml --config projects/hrr/tests/config/handler.yaml " + "--arch none -- build/hrr-dev/unit/tests/unit/hrr-handler-tests || status=1; fi", + ] + binary = f"{work}/hrr/tests/integration/hrr-integration-tests" + if backend == "emulator": + for job in emulator_jobs(): + launcher = f"rocjitsu --daemon --config {repo}/{job['config']} --" + lines.append( + "python3 projects/hrr/tests/scripts/run_catch2.py " + f"--xml build/hrr-dev/integration-{job['family']}.xml --timeout 1200 " + "--config projects/hrr/tests/config/integration.yaml " + f"--os linux --arch {job['arch']} --launcher \"{launcher}\" " + f"-- {binary} || status=1" + ) + else: + lines.append( + "python3 projects/hrr/tests/scripts/run_catch2.py " + "--xml build/hrr-dev/integration-gpu.xml --timeout 1200 " + "--config projects/hrr/tests/config/integration.yaml " + f"--os linux --arch auto -- {binary} || status=1" + ) + lines.append("exit $status") + return "\n".join(lines) + "\n" + + +def exec_command(repo: Path, backend: str) -> list[str]: + command = ["docker", "exec", "-w", str(repo)] + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + command += ["-e", f"GH_TOKEN={token}"] + command += [CONTAINER, "bash", "-lc", container_script(repo, backend)] + return command + + +def prepare(repo: Path, backend: str, runner=subprocess.run) -> None: + for command in plan_docker(repo, backend, inspect_docker(runner)): + subprocess.run(command, check=True) + subprocess.run(exec_command(repo, backend), check=True) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("backend", choices=("emulator", "gpu")) + parser.add_argument("--repo", type=Path, default=REPO) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + prepare(args.repo.resolve(), args.backend) + except subprocess.CalledProcessError as exc: + return exc.returncode or 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/projects/hrr/tests/scripts/run_catch2.py b/projects/hrr/tests/scripts/run_catch2.py index 12c73144bb50..b6af8604bae5 100644 --- a/projects/hrr/tests/scripts/run_catch2.py +++ b/projects/hrr/tests/scripts/run_catch2.py @@ -7,6 +7,7 @@ import argparse import os +import shlex import signal import subprocess import sys @@ -23,20 +24,38 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=float, help="seconds before the suite and its workloads are killed", ) - parser.add_argument("--config", type=Path, help="suite YAML; decides which cases run") - parser.add_argument("--os", dest="os_name", choices=("auto", "linux", "windows"), - help="OS the rules see; default auto when --config is set") + parser.add_argument( + "--config", type=Path, help="suite YAML; decides which cases run" + ) + parser.add_argument( + "--os", + dest="os_name", + choices=("auto", "linux", "windows"), + help="OS the rules see; default auto when --config is set", + ) parser.add_argument("--arch", help="auto, none, or comma-separated gfx names") - parser.add_argument("--select", help="Catch2 selector; overrides the one in the config") - parser.add_argument("--ignore-config", action="store_true", - help="run the cases the config would skip") + parser.add_argument( + "--select", help="Catch2 selector; overrides the one in the config" + ) + parser.add_argument( + "--ignore-config", + action="store_true", + help="run the cases the config would skip", + ) + parser.add_argument( + "--launcher", + default="", + help="Prefix for the suite process. Case listing still uses the Catch2 binary.", + ) parser.add_argument("command", nargs=argparse.REMAINDER) args = parser.parse_args(argv) if args.command[:1] == ["--"]: args.command = args.command[1:] if not args.command: parser.error("a Catch2 command is required after --") - if args.config is None and (args.os_name or args.arch or args.select or args.ignore_config): + if args.config is None and ( + args.os_name or args.arch or args.select or args.ignore_config + ): parser.error("--os, --arch, --select and --ignore-config require --config") return args @@ -93,7 +112,10 @@ def _ignore_config(args: argparse.Namespace) -> bool: if args.ignore_config: return True return os.environ.get("HRR_TEST_IGNORE_CONFIG", "").strip().lower() in ( - "1", "true", "yes", "on", + "1", + "true", + "yes", + "on", ) @@ -112,16 +134,22 @@ def _archs_for(arch_arg: str, ignore: bool) -> list[str]: return [] if arch_arg == "auto": import hrr_test_config + archs = hrr_test_config.detect_archs() if not archs: - print("error: could not detect a GPU architecture; " - "pass --arch none or --arch gfxNNNN", file=sys.stderr) + print( + "error: could not detect a GPU architecture; " + "pass --arch none or --arch gfxNNNN", + file=sys.stderr, + ) raise SystemExit(1) return archs archs = _parse_arch(arch_arg) if not archs or not all(arch.startswith("gfx") for arch in archs): - print(f"error: --arch must be auto, none, or gfx names, not {arch_arg!r}", - file=sys.stderr) + print( + f"error: --arch must be auto, none, or gfx names, not {arch_arg!r}", + file=sys.stderr, + ) raise SystemExit(1) return archs @@ -145,13 +173,17 @@ def _add_skip(suite: ET.Element, name: str, message: str) -> None: def _write_junit(path: Path, skips: list[tuple[str, str]]) -> None: root = ET.Element("testsuites") - suite = ET.SubElement(root, "testsuite", { - "name": "hrr-config", - "tests": str(len(skips)), - "failures": "0", - "errors": "0", - "skipped": str(len(skips)), - }) + suite = ET.SubElement( + root, + "testsuite", + { + "name": "hrr-config", + "tests": str(len(skips)), + "failures": "0", + "errors": "0", + "skipped": str(len(skips)), + }, + ) for name, message in skips: _add_skip(suite, name, message) tree = ET.ElementTree(root) @@ -177,7 +209,9 @@ def _inject_skips(path: Path, skips: list[tuple[str, str]]) -> None: tree.write(path, encoding="unicode", xml_declaration=True) -def _apply_config(args: argparse.Namespace) -> tuple[list[str], list[tuple[str, str]], str | None]: +def _apply_config( + args: argparse.Namespace, +) -> tuple[list[str], list[tuple[str, str]], str | None]: """Return the Catch2 command, the config skips, and the case-list file. An empty command means every selected case was skipped, so Catch2 is not @@ -212,16 +246,25 @@ def _apply_config(args: argparse.Namespace) -> tuple[list[str], list[tuple[str, # selector is Catch2's. Listing still uses command[0], which is the binary # in CI. Tests mock list_cases. names = [case["name"] for case in listed] - _run, skipped = hrr_test_config.resolve(config, names, os_name, archs, ignore=ignore) + _run, skipped = hrr_test_config.resolve( + config, names, os_name, archs, ignore=ignore + ) records = [ - (item["name"], hrr_test_config.skip_message(os_name, item["archs"], item["rule"])) + ( + item["name"], + hrr_test_config.skip_message(os_name, item["archs"], item["rule"]), + ) for item in skipped ] if not _run: return [], records, None prefix = _strip_filters(args.command) handle = tempfile.NamedTemporaryFile( - "w", prefix="hrr-cases-", suffix=".txt", delete=False, encoding="utf-8", + "w", + prefix="hrr-cases-", + suffix=".txt", + delete=False, + encoding="utf-8", ) with handle: for name in _run: @@ -229,6 +272,13 @@ def _apply_config(args: argparse.Namespace) -> tuple[list[str], list[tuple[str, return prefix + ["--input-file", handle.name], records, handle.name +def launch_command(launcher: str, command: list[str]) -> list[str]: + """Prefix the suite process. Listing uses command[0], the Catch2 binary.""" + if not launcher: + return command + return shlex.split(launcher) + command + + def _report(root: ET.Element) -> tuple[dict[str, int], str]: """Print one line per case. Returns the counts and the joined stdout.""" labels = {"PASS": "PASS", "FAIL": "FAIL", "SKIP": "SKIP", "CONFIG": "SKIP (config)"} @@ -285,14 +335,18 @@ def _run_suite(args: argparse.Namespace, state: dict) -> int: return 0 else: command = list(args.command) - command = command + [ - "--durations", - "yes", - "--reporter", - "console", - "--reporter", - f"junit::out={args.xml}", - ] + command = launch_command( + args.launcher, + command + + [ + "--durations", + "yes", + "--reporter", + "console", + "--reporter", + f"junit::out={args.xml}", + ], + ) # A session of its own, so a timeout also kills the workloads the suite # spawned: they hold the pipe open and would block the read forever. with subprocess.Popen( diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index a6752183b808..22c90657bb8c 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -44,20 +44,30 @@ def test_not_found(self): class CommandTest(unittest.TestCase): def test_fetch_command_uses_recorded_run(self): base = cb.pick_base_run(FINDER_FOUND) - cmd = [str(c) for c in cb.fetch_command(Path("/t"), base, Path("/d"), ("core-hip_lib",))] + cmd = [ + str(c) + for c in cb.fetch_command(Path("/t"), base, Path("/d"), ("core-hip_lib",)) + ] self.assertIn("--flatten", cmd) self.assertEqual(cmd[cmd.index("--run-id") + 1], "123") self.assertEqual(cmd[cmd.index("--run-github-repo") + 1], "ROCm/rocm-systems") self.assertEqual(cmd[-1], "core-hip_lib") def test_clr_compiles_capture_in(self): - cmd = [str(c) for c in cb.clr_configure_command(Path("/s"), Path("/b"), Path("/d"))] + cmd = [ + str(c) for c in cb.clr_configure_command(Path("/s"), Path("/b"), Path("/d")) + ] self.assertIn("-DCMAKE_CXX_FLAGS=-DHIP_HRR_CAPTURE_ENABLED", cmd) self.assertIn("-DCMAKE_INSTALL_PREFIX=/d", cmd) self.assertIn("-DHIP_COMMON_DIR=/s/projects/hip", cmd) def test_hrr_builds_integration_suite_for_archs(self): - cmd = [str(c) for c in cb.hrr_configure_command(Path("/s"), Path("/b"), Path("/d"), "gfx942;gfx950")] + cmd = [ + str(c) + for c in cb.hrr_configure_command( + Path("/s"), Path("/b"), Path("/d"), "gfx942;gfx950" + ) + ] self.assertIn("-DGPU_TARGETS=gfx942;gfx950", cmd) self.assertIn("-DHRR_BUILD_INTEGRATION_TESTS=ON", cmd) self.assertIn("-DCMAKE_HIP_COMPILER=/d/lib/llvm/bin/clang++", cmd) @@ -80,7 +90,9 @@ def test_parses_arches(self): self.assertEqual(cb.embedded_targets(self.OUTPUT), {"gfx942", "gfx1100"}) def test_missing(self): - self.assertEqual(cb.missing_targets("gfx942;gfx950;gfx1100", self.OUTPUT), ["gfx950"]) + self.assertEqual( + cb.missing_targets("gfx942;gfx950;gfx1100", self.OUTPUT), ["gfx950"] + ) def test_none_missing(self): self.assertEqual(cb.missing_targets("gfx942;gfx1100", self.OUTPUT), []) @@ -114,7 +126,12 @@ def test_overlay_replaces_release_library_and_keeps_rocr(self): self.assertEqual( sorted(copied), - ["libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.7.2.1", "libhiprtc.so.7"], + [ + "libamdhip64.so", + "libamdhip64.so.7", + "libamdhip64.so.7.2.1", + "libhiprtc.so.7", + ], ) self.assertFalse((dist / "libamdhip64.so.7.2.0").exists()) self.assertEqual((dist / "libamdhip64.so").read_text(), "capture") @@ -141,16 +158,23 @@ def ldd(self, hsa: str, extra: str = "") -> str: ) def test_both_from_dist(self): - self.assertEqual(cb.linkage_errors(self.ldd("/w/dist/lib/libhsa-runtime64.so.1"), self.LIB), []) + self.assertEqual( + cb.linkage_errors(self.ldd("/w/dist/lib/libhsa-runtime64.so.1"), self.LIB), + [], + ) def test_rocr_from_elsewhere(self): - errors = cb.linkage_errors(self.ldd("/opt/rocm/lib/libhsa-runtime64.so.1"), self.LIB) + errors = cb.linkage_errors( + self.ldd("/opt/rocm/lib/libhsa-runtime64.so.1"), self.LIB + ) self.assertEqual(len(errors), 1) self.assertIn("libhsa-runtime64.so", errors[0]) def test_two_rocr_copies(self): extra = "\tlibhsa-runtime64.so.2 => /opt/rocm/lib/libhsa-runtime64.so.2 (0x0)\n" - errors = cb.linkage_errors(self.ldd("/w/dist/lib/libhsa-runtime64.so.1", extra), self.LIB) + errors = cb.linkage_errors( + self.ldd("/w/dist/lib/libhsa-runtime64.so.1", extra), self.LIB + ) self.assertEqual(len(errors), 1) def test_missing_rocr(self): @@ -174,14 +198,88 @@ def runner(cmd): (dist / "lib" / "libamdhip64.so.7").write_text("capture") args = argparse.Namespace( - src=root / "src", build=root / "build", dist=dist, - capture_lib=root / "build" / "capture-lib", archs="gfx942", + src=root / "src", + build=root / "build", + dist=dist, + capture_lib=root / "build" / "capture-lib", + archs="gfx942", ) cb.cmd_build(args, runner=runner) sources = [c[c.index("-S") + 1] for c in calls if "-S" in c] - self.assertEqual(sources, [f"{root}/src/projects/clr", f"{root}/src/projects/hrr"]) - self.assertEqual([p.name for p in args.capture_lib.iterdir()], ["libamdhip64.so.7"]) - self.assertEqual((dist / "lib" / "libhsa-runtime64.so.1").read_text(), "base") + self.assertEqual( + sources, [f"{root}/src/projects/clr", f"{root}/src/projects/hrr"] + ) + self.assertEqual( + [p.name for p in args.capture_lib.iterdir()], ["libamdhip64.so.7"] + ) + self.assertEqual( + (dist / "lib" / "libhsa-runtime64.so.1").read_text(), "base" + ) + + +class CacheKeyTest(unittest.TestCase): + def test_inputs_are_the_build_and_not_ci_yaml(self): + paths = cb.CACHE_SOURCE_PATHS + for required in ( + "projects/clr", + "projects/hip", + "projects/hrr/CMakeLists.txt", + "projects/hrr/include", + "projects/hrr/playback", + "projects/hrr/tests/CMakeLists.txt", + "projects/hrr/tests/include", + "projects/hrr/tests/integration", + "projects/hrr/tests/scripts/hrr_capture_build.py", + ): + self.assertIn(required, paths) + self.assertNotIn("projects/hrr/tests/config/ci.yaml", paths) + self.assertFalse(any("ci.yaml" in path for path in paths)) + + def test_digest_tracks_base_pin_image_and_archs(self): + with tempfile.TemporaryDirectory() as tmp: + src = Path(tmp) + for rel in cb.CACHE_SOURCE_PATHS: + path = src / rel + path.parent.mkdir(parents=True, exist_ok=True) + if rel.endswith(".py") or rel.endswith(".txt"): + path.write_text("source") + else: + path.mkdir(exist_ok=True) + (path / "file.cpp").write_text("int x;") + (src / "projects/hrr/tests/config").mkdir(parents=True) + (src / "projects/hrr/tests/config/ci.yaml").write_text("families: {}\n") + base = { + "repo": "ROCm/rocm-systems", + "run_id": "1", + "commit": "abc", + "artifact_group": "gfx94X-dcgpu", + } + first = cb.cache_key_digest(src, base, "pin", "image@sha", "gfx950;gfx1250") + self.assertEqual( + first, + cb.cache_key_digest(src, base, "pin", "image@sha", "gfx950;gfx1250"), + ) + self.assertNotEqual( + first, + cb.cache_key_digest( + src, base, "other-pin", "image@sha", "gfx950;gfx1250" + ), + ) + self.assertNotEqual( + first, cb.cache_key_digest(src, base, "pin", "image@sha", "gfx942") + ) + (src / "projects/hrr/tests/config/ci.yaml").write_text( + "backend: simulator\n" + ) + self.assertEqual( + first, + cb.cache_key_digest(src, base, "pin", "image@sha", "gfx950;gfx1250"), + ) + (src / "projects/hrr/playback/file.cpp").write_text("int y;") + self.assertNotEqual( + first, + cb.cache_key_digest(src, base, "pin", "image@sha", "gfx950;gfx1250"), + ) class ArchsTest(unittest.TestCase): @@ -193,7 +291,9 @@ def test_default_is_the_integration_config(self): self.assertEqual(args.archs, ";".join(config["targets"]["arch"])) def test_override(self): - args = cb.parse_args(["verify", "--src", str(self.SRC), "--archs", "gfx942;gfx1152"]) + args = cb.parse_args( + ["verify", "--src", str(self.SRC), "--archs", "gfx942;gfx1152"] + ) self.assertEqual(args.archs, "gfx942;gfx1152") diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index f0abcedced23..b4d1a513a150 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -91,10 +91,32 @@ def config_data(**changes): data = copy.deepcopy(REPO_CONFIG) data["families"] = copy.deepcopy(FAMILIES) data.update(changes) + families = data.get("families") + simulator = data.get("simulator") + if isinstance(families, dict) and isinstance(simulator, dict): + present = [ + key for key in simulator.get("default_families", []) if key in families + ] + if not present and families: + present = [next(iter(families))] + simulator["default_families"] = present + configs = simulator.setdefault("configs", {}) + if isinstance(configs, dict): + for key in [key for key in configs if key not in families]: + del configs[key] + for key in present: + configs.setdefault(key, f"emulation/rocjitsu/configs/{key}.json") return data -CONFIG = cfg.parse_config(config_data()) +def gpu_data(**changes): + """The family-selection tests predate the simulator default.""" + data = config_data(**changes) + data["backend"] = {"default": "gpu", "gpu": {"triggers": []}} + return data + + +CONFIG = cfg.parse_config(gpu_data()) REPO = cfg.load_config(cfg.DEFAULT_CONFIG) @@ -127,15 +149,15 @@ def test_only_gfx94x_runs_by_default(self): for key, family in REPO.families.items(): self.assertEqual(family.triggers, cfg.EVENTS, key) - def test_a_label_still_requests_a_disabled_family(self): + def test_a_label_adds_a_simulator_family(self): d = decide( "pull_request", ["projects/hrr/playback/x.cpp"], ["ci:gfx110x"], config=REPO, ) - self.assertIn("gfx94x", family_keys(d)) - self.assertIn("gfx110x", family_keys(d)) + self.assertEqual(family_keys(d), ["gfx950", "gfx110x", "gfx125x"]) + self.assertTrue(all(row["backend"] == "simulator" for row in d.families)) def test_multi_gpu_runner_only_nightly(self): for key, family in REPO.families.items(): @@ -145,15 +167,22 @@ def test_multi_gpu_runner_only_nightly(self): class PathClassificationTest(unittest.TestCase): def test_playback_change_is_capture(self): - self.assertTrue(cfg.touches_capture(["projects/hrr/playback/hip_playback.cpp"], REPO)) + self.assertTrue( + cfg.touches_capture(["projects/hrr/playback/hip_playback.cpp"], REPO) + ) def test_capture_source_is_capture(self): self.assertTrue( cfg.touches_capture(["projects/clr/hipamd/src/hrr/hip_capture.cpp"], REPO) ) - def test_ci_config_is_capture(self): - self.assertTrue(cfg.touches_capture(["projects/hrr/tests/config/ci.yaml"], REPO)) + def test_ci_config_runs_the_suite_and_is_not_capture(self): + paths = ["projects/hrr/tests/config/ci.yaml"] + self.assertFalse(cfg.touches_capture(paths, REPO)) + self.assertTrue(cfg.touches_suite(paths, REPO)) + decision = decide("pull_request", paths, config=REPO) + self.assertTrue(decision.run_gpu) + self.assertEqual(family_keys(decision), ["gfx950", "gfx125x"]) def test_public_header_is_cpu_only(self): paths = ["projects/hip/include/hip/hip_runtime_api.h"] @@ -195,7 +224,9 @@ def test_wrong_version(self): def test_unknown_family_option(self): families = {"gfx94x": {"runs_on": "x"}} - self.assertRejected(config_data(families=families), "families.gfx94x: unknown key runs_on") + self.assertRejected( + config_data(families=families), "families.gfx94x: unknown key runs_on" + ) def test_unknown_trigger(self): families = {"gfx94x": {"triggers": ["nightly"]}} @@ -204,7 +235,8 @@ def test_unknown_trigger(self): def test_unknown_multi_gpu_trigger(self): families = {"gfx950": {"multi_gpu_triggers": ["nightly"]}} self.assertRejected( - config_data(families=families), "families.gfx950.multi_gpu_triggers: unknown trigger" + config_data(families=families), + "families.gfx950.multi_gpu_triggers: unknown trigger", ) def test_enabled_must_be_a_bool(self): @@ -226,21 +258,36 @@ def test_family_missing_from_matrix(self): self.assertIn("gfx999 not in TheRock's family matrix", str(caught.exception)) def test_family_targets(self): - self.assertEqual(cfg.family_targets(MATRIX["gfx110x"]["linux"]), ["gfx1100", "gfx1101"]) + self.assertEqual( + cfg.family_targets(MATRIX["gfx110x"]["linux"]), ["gfx1100", "gfx1101"] + ) # gfx1150 lists no fetch targets; its family name is the target. self.assertEqual(cfg.family_targets(MATRIX["gfx1150"]["linux"]), ["gfx1150"]) self.assertEqual( - cfg.family_targets({"fetch-gfx-targets": ["gfx942", "gfx942:xnack+"]}), ["gfx942"] + cfg.family_targets({"fetch-gfx-targets": ["gfx942", "gfx942:xnack+"]}), + ["gfx942"], ) self.assertEqual(cfg.family_targets({"family": "gfx101X-dgpu"}), []) def test_targets_cover_every_family(self): - archs = ["gfx90a", "gfx942", "gfx950", "gfx1100", "gfx1101", "gfx1150", "gfx1250"] + archs = [ + "gfx90a", + "gfx942", + "gfx950", + "gfx1100", + "gfx1101", + "gfx1150", + "gfx1250", + ] cfg.check_targets(CONFIG, MATRIX, archs) with self.assertRaises(cfg.ConfigError) as caught: - cfg.check_targets(CONFIG, MATRIX, [a for a in archs if a not in ("gfx1101", "gfx1250")]) + cfg.check_targets( + CONFIG, MATRIX, [a for a in archs if a not in ("gfx1101", "gfx1250")] + ) # gfx125x is disabled, but a label can still request it. - self.assertIn("gfx110x needs gfx1101; gfx125x needs gfx1250", str(caught.exception)) + self.assertIn( + "gfx110x needs gfx1101; gfx125x needs gfx1250", str(caught.exception) + ) def test_family_without_targets_is_an_error(self): config = cfg.parse_config(config_data(families={"gfx94x": {}})) @@ -267,8 +314,16 @@ def test_main_reports_a_bad_config(self): out = io.StringIO() with redirect_stdout(out): status = cfg.main( - ["--event", "schedule", "--repo", "r", "--therock-dir", tmp, - "--config", str(path)] + [ + "--event", + "schedule", + "--repo", + "r", + "--therock-dir", + tmp, + "--config", + str(path), + ] ) self.assertEqual(status, 1) self.assertIn("::error::", out.getvalue()) @@ -291,12 +346,24 @@ def test_capture_pull_request_runs_enabled_families(self): # gfx110x is not triggered by pull requests, gfx125x is disabled and # gfx1150 has no runner. self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950"]) - self.assertEqual(d.families[1], {"family": "gfx94x", "runner": "gpu-94x-a"}) + self.assertEqual( + d.families[1], + { + "family": "gfx94x", + "runner": "gpu-94x-a", + "backend": "gpu", + "arch": "gfx942", + "config": "", + }, + ) self.assertIn("no runner for gfx1150", d.reason) self.assertEqual( d.skipped, [ - {"family": "gfx110x", "reason": "not run on pull_request (ci.yaml triggers)"}, + { + "family": "gfx110x", + "reason": "not run on pull_request (ci.yaml triggers)", + }, {"family": "gfx1150", "reason": cfg.NO_RUNNER}, {"family": "gfx125x", "reason": "disabled in ci.yaml"}, ], @@ -304,16 +371,22 @@ def test_capture_pull_request_runs_enabled_families(self): def test_rename_out_of_capture_counts(self): # The files API lists previous_filename alongside filename. - d = decide("pull_request", ["projects/other/x.cpp", "projects/hrr/playback/x.cpp"]) + d = decide( + "pull_request", ["projects/other/x.cpp", "projects/hrr/playback/x.cpp"] + ) self.assertTrue(d.run_gpu) def test_label_requests_gpu_on_unrelated_pull_request(self): - d = decide("pull_request", ["projects/rocminfo/x.cc"], labels=["ci:hrr-capture"]) + d = decide( + "pull_request", ["projects/rocminfo/x.cc"], labels=["ci:hrr-capture"] + ) self.assertTrue(d.run_cpu) self.assertTrue(d.run_gpu) def test_label_match_ignores_case(self): - d = decide("pull_request", ["projects/rocminfo/x.cc"], labels=["CI:HRR-Capture"]) + d = decide( + "pull_request", ["projects/rocminfo/x.cc"], labels=["CI:HRR-Capture"] + ) self.assertTrue(d.run_gpu) def test_capture_only_path_also_runs_cpu(self): @@ -327,10 +400,14 @@ def test_family_label_adds_family(self): ["projects/hrr/playback/x.cpp"], labels=["ci:gfx110X-all", "ci:gfx125x"], ) - self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950", "gfx110x", "gfx125x"]) + self.assertEqual( + family_keys(d), ["gfx90a", "gfx94x", "gfx950", "gfx110x", "gfx125x"] + ) def test_label_for_family_not_in_config_is_ignored(self): - d = decide("pull_request", ["projects/hrr/playback/x.cpp"], labels=["ci:gfx103x"]) + d = decide( + "pull_request", ["projects/hrr/playback/x.cpp"], labels=["ci:gfx103x"] + ) self.assertNotIn("gfx103x", family_keys(d)) def test_fork_skips_gpu(self): @@ -357,10 +434,12 @@ def test_multi_gpu_runner_on_its_triggers_only(self): nightly = {f["family"]: f["runner"] for f in decide("schedule", []).families} self.assertEqual(nightly["gfx950"], "gpu-950-8") pr = decide("pull_request", ["projects/hrr/playback/x.cpp"]) - self.assertEqual({f["family"]: f["runner"] for f in pr.families}["gfx950"], "gpu-950-1") + self.assertEqual( + {f["family"]: f["runner"] for f in pr.families}["gfx950"], "gpu-950-1" + ) def test_runner_override(self): - config = cfg.parse_config(config_data(families={"gfx94x": {"runner": "my-gpu"}})) + config = cfg.parse_config(gpu_data(families={"gfx94x": {"runner": "my-gpu"}})) d = decide("schedule", [], config=config) self.assertEqual(d.families[0]["runner"], "my-gpu") @@ -381,20 +460,100 @@ def test_unknown_event(self): self.assertFalse(d.run_gpu) +class SimulatorBackendTest(unittest.TestCase): + def test_default_event_is_the_two_rocjitsu_jobs(self): + decision = decide("pull_request", ["projects/hrr/playback/x.cpp"], config=REPO) + self.assertEqual(family_keys(decision), ["gfx950", "gfx125x"]) + by_family = {row["family"]: row for row in decision.families} + self.assertEqual(by_family["gfx950"]["runner"], "ubuntu-24.04") + self.assertEqual(by_family["gfx950"]["backend"], "simulator") + self.assertEqual(by_family["gfx950"]["arch"], "gfx950") + self.assertTrue( + by_family["gfx950"]["config"].endswith("gfx950_mi355x_kmd_2gpu.json") + ) + self.assertEqual(by_family["gfx125x"]["runner"], "ubuntu-24.04") + self.assertEqual(by_family["gfx125x"]["arch"], "gfx1250") + self.assertTrue( + by_family["gfx125x"]["config"].endswith("gfx1250_mi455x_kmd_4gpu.json") + ) + self.assertNotEqual( + by_family["gfx125x"]["runner"], MATRIX["gfx125x"]["linux"]["test-runs-on"] + ) + + def test_gpu_dispatch_selects_the_therock_runner(self): + decision = cfg.decide( + "workflow_dispatch", + [], + [], + False, + ["gfx94x"], + REPO, + MATRIX, + pick_first, + backend="gpu", + ) + self.assertEqual(family_keys(decision), ["gfx94x"]) + self.assertEqual(decision.families[0]["backend"], "gpu") + self.assertEqual(decision.families[0]["runner"], "gpu-94x-a") + self.assertNotIn("gfx125x", family_keys(decision)) + + def test_pull_request_gpu_triggers_select_real_runners(self): + data = config_data() + data["backend"]["gpu"]["triggers"] = ["pull_request"] + config = cfg.parse_config(data) + decision = decide( + "pull_request", ["projects/hrr/playback/x.cpp"], config=config + ) + self.assertEqual(family_keys(decision), ["gfx90a", "gfx94x", "gfx950"]) + self.assertTrue(all(row["backend"] == "gpu" for row in decision.families)) + self.assertNotIn("gfx125x", family_keys(decision)) + + def test_simulator_rejects_a_family_without_a_config(self): + with self.assertRaises(cfg.ConfigError) as caught: + decide( + "pull_request", + ["projects/hrr/playback/x.cpp"], + labels=["ci:gfx103x"], + config=REPO, + ) + self.assertIn("no config for gfx103x", str(caught.exception)) + + def test_unknown_backend_value(self): + data = config_data() + data["backend"]["default"] = "metal" + with self.assertRaises(cfg.ConfigError) as caught: + cfg.parse_config(data) + self.assertIn("backend.default", str(caught.exception)) + + class LabelEventTest(unittest.TestCase): CAPTURE = ["projects/hrr/playback/x.cpp"] - WORKFLOW = Path(__file__).resolve().parents[4] / ".github" / "workflows" / "hrr-ci.yml" + WORKFLOW = ( + Path(__file__).resolve().parents[4] / ".github" / "workflows" / "hrr-ci.yml" + ) def test_which_labels_matter(self): - for label in ("ci:hrr-capture", "CI:HRR-Capture", "ci:gfx110x", "ci:gfx110X-all"): + for label in ( + "ci:hrr-capture", + "CI:HRR-Capture", + "ci:gfx110x", + "ci:gfx110X-all", + ): self.assertTrue(cfg.label_matters(label, REPO), label) for label in ("documentation", "ci:skip", "hrr"): self.assertFalse(cfg.label_matters(label, REPO), label) def test_other_label_runs_nothing(self): d = cfg.decide( - "pull_request", self.CAPTURE, ["ci:hrr-capture", "documentation"], False, [], - CONFIG, MATRIX, pick_first, added_label="documentation", + "pull_request", + self.CAPTURE, + ["ci:hrr-capture", "documentation"], + False, + [], + CONFIG, + MATRIX, + pick_first, + added_label="documentation", ) self.assertFalse(d.run_cpu) self.assertFalse(d.run_gpu) @@ -403,8 +562,15 @@ def test_other_label_runs_nothing(self): def test_capture_or_family_label_reruns(self): for label in ("ci:hrr-capture", "ci:gfx110x"): d = cfg.decide( - "pull_request", self.CAPTURE, [label], False, [], - CONFIG, MATRIX, pick_first, added_label=label, + "pull_request", + self.CAPTURE, + [label], + False, + [], + CONFIG, + MATRIX, + pick_first, + added_label=label, ) self.assertTrue(d.run_gpu, label) @@ -422,15 +588,17 @@ def test_labeled_trigger_and_summary_report_a_verdict(self): types = re.search(r"types:\s*\[([^\]]*)\]", text).group(1) self.assertIn("labeled", types) - summary = text[text.index(" summary:"):] + summary = text[text.index(" summary:") :] # The summary also runs when the label run configured nothing. self.assertIn("github.event.action == 'labeled'", summary) # And it answers with an actual verdict, never a skip of its own. - self.assertNotIn("needs.configure.result != 'success' || needs.configure.outputs.run_cpu == 'true' ||", - summary) + self.assertNotIn( + "needs.configure.result != 'success' || needs.configure.outputs.run_cpu == 'true' ||", + summary, + ) # Only a label run that configured nothing passes the verdict through; # every other run reports its own jobs. - step = summary[summary.index("PASS_THROUGH:"):summary.index("run: >-")] + step = summary[summary.index("PASS_THROUGH:") : summary.index("run: >-")] self.assertIn("github.event.action == 'labeled'", step) self.assertIn("needs.configure.result == 'success'", step) self.assertIn("run_cpu == 'false'", step) @@ -441,7 +609,7 @@ def test_labeled_trigger_and_summary_report_a_verdict(self): def test_concurrency_group_makes_the_same_test(self): text = self.WORKFLOW.read_text(encoding="utf-8") - group = text[text.index("concurrency:"):text.index("cancel-in-progress:")] + group = text[text.index("concurrency:") : text.index("cancel-in-progress:")] self.assertIn(f"github.event.label.name != '{REPO.capture_label}'", group) self.assertIn("!startsWith(github.event.label.name, 'ci:gfx')", group) @@ -462,17 +630,32 @@ def test_only_pull_requests_are_forks(self): self.assertFalse(cfg.is_fork("push", "", self.REPO)) def test_main_skips_gpu_for_a_deleted_head_repository(self): - with tempfile.TemporaryDirectory() as tmp, \ - mock.patch.object(cfg, "list_pr_files", return_value=["projects/hrr/playback/x.cpp"]), \ - mock.patch.object(cfg, "load_matrix", return_value=(MATRIX, pick_first)), \ - mock.patch.dict(os.environ, {"GITHUB_OUTPUT": ""}): + with tempfile.TemporaryDirectory() as tmp, mock.patch.object( + cfg, "list_pr_files", return_value=["projects/hrr/playback/x.cpp"] + ), mock.patch.object( + cfg, "load_matrix", return_value=(MATRIX, pick_first) + ), mock.patch.dict( + os.environ, {"GITHUB_OUTPUT": ""} + ): path = Path(tmp) / "ci.yaml" path.write_text(yaml.safe_dump(config_data())) out = io.StringIO() with redirect_stdout(out): status = cfg.main( - ["--event", "pull_request", "--repo", self.REPO, "--pr-number", "1", - "--head-repo", "", "--therock-dir", tmp, "--config", str(path)] + [ + "--event", + "pull_request", + "--repo", + self.REPO, + "--pr-number", + "1", + "--head-repo", + "", + "--therock-dir", + tmp, + "--config", + str(path), + ] ) self.assertEqual(status, 0) self.assertIn("run_cpu=true\n", out.getvalue()) @@ -483,7 +666,10 @@ def test_main_skips_gpu_for_a_deleted_head_repository(self): class OutputTest(unittest.TestCase): def test_write_outputs(self): d = cfg.Decision( - True, True, "why", [{"family": "gfx94x", "runner": "r"}], + True, + True, + "why", + [{"family": "gfx94x", "runner": "r"}], [{"family": "gfx125x", "reason": "disabled in ci.yaml"}], ) with tempfile.TemporaryDirectory() as tmp: @@ -493,7 +679,10 @@ def test_write_outputs(self): text = out.read_text() self.assertIn("run_cpu=true\n", text) self.assertIn('families=[{"family":"gfx94x","runner":"r"}]\n', text) - self.assertIn('skipped_families=[{"family":"gfx125x","reason":"disabled in ci.yaml"}]\n', text) + self.assertIn( + 'skipped_families=[{"family":"gfx125x","reason":"disabled in ci.yaml"}]\n', + text, + ) self.assertIn("reason=why\n", text) self.assertIn("rocm_sdk_version=1.2.3\n", text) diff --git a/projects/hrr/tests/scripts/test_hrr_dev.py b/projects/hrr/tests/scripts/test_hrr_dev.py new file mode 100644 index 000000000000..e61a5666a1af --- /dev/null +++ b/projects/hrr/tests/scripts/test_hrr_dev.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Tests for hrr_dev.py. Docker is not started.""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import hrr_dev # noqa: E402 + + +class ParseTest(unittest.TestCase): + def test_emulator_and_gpu(self): + self.assertEqual(hrr_dev.parse_args(["emulator"]).backend, "emulator") + self.assertEqual(hrr_dev.parse_args(["gpu"]).backend, "gpu") + + +class DockerPlanTest(unittest.TestCase): + def test_missing_image_is_a_docker_build(self): + repo = Path("/repo") + commands = hrr_dev.plan_docker( + repo, "emulator", hrr_dev.DockerState(image=False, container=None) + ) + self.assertEqual(commands[0][:3], ["docker", "build", "-t"]) + self.assertIn("hrr-dev", commands[0]) + self.assertIn(str(repo / "projects/hrr/tests/docker/Dockerfile"), commands[0]) + + def test_stopped_container_is_started(self): + commands = hrr_dev.plan_docker( + Path("/repo"), "gpu", hrr_dev.DockerState(image=True, container="exited") + ) + self.assertEqual(commands, [["docker", "start", "hrr-dev"]]) + + def test_only_gpu_adds_devices(self): + repo = Path("/repo") + state = hrr_dev.DockerState(image=True, container=None) + emulator = hrr_dev.plan_docker(repo, "emulator", state)[0] + gpu = hrr_dev.plan_docker(repo, "gpu", state)[0] + self.assertNotIn("/dev/kfd", emulator) + self.assertNotIn("/dev/dri", emulator) + self.assertEqual( + gpu[gpu.index("--device") : gpu.index("--device") + 4], + ["--device", "/dev/kfd", "--device", "/dev/dri"], + ) + + +class ScriptTest(unittest.TestCase): + def test_emulator_uses_both_default_configs(self): + script = hrr_dev.container_script(Path("/repo"), "emulator") + self.assertIn("gfx950_mi355x_kmd_2gpu.json", script) + self.assertIn("gfx1250_mi455x_kmd_4gpu.json", script) + self.assertIn("--arch gfx950", script) + self.assertIn("--arch gfx1250", script) + self.assertIn("rocjitsu --daemon", script) + self.assertNotIn("/dev/kfd", script) + self.assertIn("HRR_BUILD_INTEGRATION_TESTS=OFF", script) + self.assertIn("hrr_capture_build.py build", script) + + def test_gpu_uses_arch_auto_and_no_launcher(self): + script = hrr_dev.container_script(Path("/repo"), "gpu") + self.assertIn("--arch auto", script) + self.assertNotIn("rocjitsu", script) + self.assertNotIn("gfx950_mi355x_kmd_2gpu.json", script) + + +if __name__ == "__main__": + unittest.main() diff --git a/projects/hrr/tests/scripts/test_run_catch2.py b/projects/hrr/tests/scripts/test_run_catch2.py index dc2261a89a89..7b99a105d49e 100644 --- a/projects/hrr/tests/scripts/test_run_catch2.py +++ b/projects/hrr/tests/scripts/test_run_catch2.py @@ -66,6 +66,23 @@ def test_timeout_is_optional(self): args = run_catch2.parse_args(["--xml", "r.xml", "--", "./bin"]) self.assertIsNone(args.timeout) + def test_launcher_stays_off_the_catch2_binary(self): + args = run_catch2.parse_args( + [ + "--xml", + "r.xml", + "--launcher", + "rocjitsu --daemon --config c.json --", + "--", + "./bin", + ] + ) + self.assertEqual(args.command, ["./bin"]) + self.assertEqual( + run_catch2.launch_command(args.launcher, args.command), + ["rocjitsu", "--daemon", "--config", "c.json", "--", "./bin"], + ) + def test_requires_a_command(self): with contextlib.redirect_stderr(io.StringIO()): with self.assertRaises(SystemExit): @@ -83,13 +100,25 @@ def parse(self, text: str) -> ET.Element: def test_test_result(self): self.assertEqual(run_catch2.test_result(self.parse(record("a"))), "PASS") - self.assertEqual(run_catch2.test_result(self.parse(record("a", "skipped"))), "SKIP") - self.assertEqual(run_catch2.test_result(self.parse(record("a", "failure"))), "FAIL") - self.assertEqual(run_catch2.test_result(self.parse(record("a", "error"))), "FAIL") + self.assertEqual( + run_catch2.test_result(self.parse(record("a", "skipped"))), "SKIP" + ) + self.assertEqual( + run_catch2.test_result(self.parse(record("a", "failure"))), "FAIL" + ) + self.assertEqual( + run_catch2.test_result(self.parse(record("a", "error"))), "FAIL" + ) def test_group_cases_folds_sections_into_their_case(self): root = self.parse( - junit(record("A"), record("B/x"), record("B/y"), record("C/s/t"), '') + junit( + record("A"), + record("B/x"), + record("B/y"), + record("C/s/t"), + "", + ) ) groups = run_catch2.group_cases(root) self.assertEqual(list(groups), ["A", "B", "C"]) @@ -97,7 +126,9 @@ def test_group_cases_folds_sections_into_their_case(self): def test_group_result_precedence(self): def result(*kinds): - root = self.parse(junit(*(record("A/" + str(i), k) for i, k in enumerate(kinds)))) + root = self.parse( + junit(*(record("A/" + str(i), k) for i, k in enumerate(kinds))) + ) return run_catch2.group_result(run_catch2.group_cases(root)["A"]) self.assertEqual(result(None, None), "PASS") @@ -118,7 +149,15 @@ def setUp(self): def run_main(self, spec: dict, *extra: str): spec_path = self.dir / "spec.json" spec_path.write_text(json.dumps(spec)) - argv = ["--xml", str(self.xml), *extra, "--", sys.executable, str(self.fake), str(spec_path)] + argv = [ + "--xml", + str(self.xml), + *extra, + "--", + sys.executable, + str(self.fake), + str(spec_path), + ] out, err = io.StringIO(), io.StringIO() with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): rc = run_catch2.main(argv) @@ -175,7 +214,9 @@ def test_exit_4_with_a_failure_stays_failed(self): self.assertIn("1 failed", out) def test_crash_after_skips_keeps_the_transcript(self): - rc, out, _ = self.run_main({"xml": junit(record("A", "skipped"), record("B")), "rc": 139}) + rc, out, _ = self.run_main( + {"xml": junit(record("A", "skipped"), record("B")), "rc": 139} + ) self.assertEqual(rc, 139) self.assertIn("Catch2 terminated without a JUnit failure", out) self.assertIn("BANNER", out) @@ -198,38 +239,61 @@ def test_missing_junit_with_clean_exit_still_fails(self): def _config(self, cases): path = self.dir / "suite.yaml" - path.write_text(hrr_test_config.render({ - "version": 1, - "suite": "integration", - "binary": "hrr-integration-tests", - "select": "[hrr]~[direct]", - "targets": {"os": ["linux", "windows"], - "arch": ["gfx90a", "gfx1151"]}, - "cases": cases, - })) + path.write_text( + hrr_test_config.render( + { + "version": 1, + "suite": "integration", + "binary": "hrr-integration-tests", + "select": "[hrr]~[direct]", + "targets": { + "os": ["linux", "windows"], + "arch": ["gfx90a", "gfx1151"], + }, + "cases": cases, + } + ) + ) return path def test_config_skips_a_case_and_records_why(self): - path = self._config({ - "Keep": {}, - "Drop": {"skip": [{ - "when": {"os": "linux", "arch": "gfx1151"}, - "kind": "disabled", - "reason": "npu", - "issue": "https://github.com/ROCm/rocm-systems/issues/1", - }]}, - }) - listed = [{"name": "Keep", "tags": ["[hrr]"]}, {"name": "Drop", "tags": ["[hrr]"]}] + path = self._config( + { + "Keep": {}, + "Drop": { + "skip": [ + { + "when": {"os": "linux", "arch": "gfx1151"}, + "kind": "disabled", + "reason": "npu", + "issue": "https://github.com/ROCm/rocm-systems/issues/1", + } + ] + }, + } + ) + listed = [ + {"name": "Keep", "tags": ["[hrr]"]}, + {"name": "Drop", "tags": ["[hrr]"]}, + ] with mock.patch("hrr_test_config.list_cases", return_value=listed): rc, out, _ = self.run_main( {"xml": junit(record("Keep"))}, - "--config", str(path), "--os", "linux", "--arch", "gfx1151", + "--config", + str(path), + "--os", + "linux", + "--arch", + "gfx1151", ) self.assertEqual(rc, 0) self.assertIn("PASS: Keep", out) self.assertIn("SKIP (config): Drop", out) - self.assertIn("hrr-config: linux/gfx1151: npu " - "(https://github.com/ROCm/rocm-systems/issues/1)", out) + self.assertIn( + "hrr-config: linux/gfx1151: npu " + "(https://github.com/ROCm/rocm-systems/issues/1)", + out, + ) self.assertIn("1 config-skipped", out) root = ET.parse(self.xml).getroot() skipped = root.find(".//testcase[@name='Drop']/skipped") @@ -237,14 +301,31 @@ def test_config_skips_a_case_and_records_why(self): self.assertTrue(skipped.attrib["message"].startswith("hrr-config:")) def test_config_runs_nothing_when_every_case_is_skipped(self): - path = self._config({"Drop": {"skip": [{ - "when": {"os": "linux"}, "kind": "unsupported", "reason": "all", - }]}}) - with mock.patch("hrr_test_config.list_cases", - return_value=[{"name": "Drop", "tags": ["[hrr]"]}]) as listed: + path = self._config( + { + "Drop": { + "skip": [ + { + "when": {"os": "linux"}, + "kind": "unsupported", + "reason": "all", + } + ] + } + } + ) + with mock.patch( + "hrr_test_config.list_cases", + return_value=[{"name": "Drop", "tags": ["[hrr]"]}], + ) as listed: rc, out, _ = self.run_main( {"xml": junit(record("Drop")), "banner": "SHOULD NOT RUN"}, - "--config", str(path), "--os", "linux", "--arch", "none", + "--config", + str(path), + "--os", + "linux", + "--arch", + "none", ) self.assertEqual(rc, 0) self.assertIn("SKIP (config): Drop", out) @@ -253,14 +334,31 @@ def test_config_runs_nothing_when_every_case_is_skipped(self): self.assertEqual(listed.call_args.args[1], "[hrr]~[direct]") def test_ignore_config_runs_the_skipped_case(self): - path = self._config({"Drop": {"skip": [{ - "when": {"os": "linux"}, "kind": "unsupported", "reason": "all", - }]}}) - with mock.patch("hrr_test_config.list_cases", - return_value=[{"name": "Drop", "tags": ["[hrr]"]}]) : + path = self._config( + { + "Drop": { + "skip": [ + { + "when": {"os": "linux"}, + "kind": "unsupported", + "reason": "all", + } + ] + } + } + ) + with mock.patch( + "hrr_test_config.list_cases", + return_value=[{"name": "Drop", "tags": ["[hrr]"]}], + ): rc, out, _ = self.run_main( {"xml": junit(record("Drop")), "rc": 0}, - "--config", str(path), "--os", "linux", "--arch", "gfx1151", + "--config", + str(path), + "--os", + "linux", + "--arch", + "gfx1151", "--ignore-config", ) self.assertEqual(rc, 0) @@ -273,16 +371,36 @@ def test_select_overrides_the_config_and_a_trailing_filter_is_dropped(self): spec_path = self.dir / "spec.json" # A non-zero status with no JUnit failure dumps the transcript, which is # the only place the wrapper's argv (and the case file) are visible. - spec_path.write_text(json.dumps({ - "xml": junit(record("Keep")), "echo_args": True, "rc": 139, - })) + spec_path.write_text( + json.dumps( + { + "xml": junit(record("Keep")), + "echo_args": True, + "rc": 139, + } + ) + ) argv = [ - "--xml", str(self.xml), "--config", str(path), "--os", "linux", - "--arch", "none", "--select", "[cpu]", - "--", sys.executable, str(self.fake), str(spec_path), "[hrr]~[direct]", + "--xml", + str(self.xml), + "--config", + str(path), + "--os", + "linux", + "--arch", + "none", + "--select", + "[cpu]", + "--", + sys.executable, + str(self.fake), + str(spec_path), + "[hrr]~[direct]", ] - with mock.patch("hrr_test_config.list_cases", - return_value=[{"name": "Keep", "tags": ["[cpu]"]}]) as listed: + with mock.patch( + "hrr_test_config.list_cases", + return_value=[{"name": "Keep", "tags": ["[cpu]"]}], + ) as listed: out = io.StringIO() with contextlib.redirect_stdout(out): rc = run_catch2.main(argv) @@ -298,7 +416,12 @@ def test_auto_arch_with_no_device_fails_closed(self): with mock.patch("hrr_test_config.detect_archs", return_value=[]): rc, _, err = self.run_main( {"xml": junit(record("Keep"))}, - "--config", str(path), "--os", "linux", "--arch", "auto", + "--config", + str(path), + "--os", + "linux", + "--arch", + "auto", ) self.assertEqual(rc, 1) self.assertIn("could not detect a GPU architecture", err) @@ -306,11 +429,15 @@ def test_auto_arch_with_no_device_fails_closed(self): def test_config_flags_without_a_config_are_rejected(self): with contextlib.redirect_stderr(io.StringIO()): with self.assertRaises(SystemExit): - run_catch2.parse_args(["--xml", "r.xml", "--arch", "none", "--", "./bin"]) + run_catch2.parse_args( + ["--xml", "r.xml", "--arch", "none", "--", "./bin"] + ) @unittest.skipUnless(os.name == "posix", "process groups are POSIX only") def test_timeout_kills_the_suite_and_reports(self): - rc, out, _ = self.run_main({"sleep": 60, "banner": "LAST CASE"}, "--timeout", "1") + rc, out, _ = self.run_main( + {"sleep": 60, "banner": "LAST CASE"}, "--timeout", "1" + ) self.assertEqual(rc, 1) self.assertIn("timed out after 1 s", out) From 94a3df7b22e09b638d40c48d53e834873f047cfb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 16 Oct 2026 08:59:26 +0000 Subject: [PATCH 33/89] ci(hrr): leave TheRock multi-arch workflow unchanged The simulator default and the capture cache live in the HRR workflows. --- .github/workflows/therock-multi-arch-ci.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/therock-multi-arch-ci.yml b/.github/workflows/therock-multi-arch-ci.yml index ad87b89060a7..a4eeff0e326f 100644 --- a/.github/workflows/therock-multi-arch-ci.yml +++ b/.github/workflows/therock-multi-arch-ci.yml @@ -114,7 +114,6 @@ jobs: - name: Checkout repository config uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - persist-credentials: false sparse-checkout: | .github/repos-config.json .github/scripts/compute_build_stages.py @@ -123,7 +122,6 @@ jobs: - name: Checkout TheRock for configure script uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - persist-credentials: false repository: ROCm/TheRock ref: c01e3a3f3f67966dd35612cf363b49964f33cd47 # 2026-10-12 path: _therock @@ -166,13 +164,10 @@ jobs: # project is disabled on Windows in the topology. - name: Compute build-stage allowlist id: build_stages - env: - CHANGED_PROJECTS: ${{ steps.configure-pr.outputs.changed_projects || steps.configure-manual.outputs.changed_projects }} - RUN_ALL_TESTS: ${{ steps.configure-pr.outputs.run_all_tests || steps.configure-manual.outputs.run_all_tests }} run: | python3 .github/scripts/compute_build_stages.py \ - --changed-projects "${CHANGED_PROJECTS}" \ - --run-all-tests "${RUN_ALL_TESTS}" \ + --changed-projects "${{ steps.configure-pr.outputs.changed_projects || steps.configure-manual.outputs.changed_projects }}" \ + --run-all-tests "${{ steps.configure-pr.outputs.run_all_tests || steps.configure-manual.outputs.run_all_tests }}" \ --therock-path "_therock" setup: @@ -192,7 +187,6 @@ jobs: # other stage outright (no build, no artifact copy). Empty = build all, # the fail-safe for fan-out / unknown-project / CI-infra changes. build_stages: ${{ needs.configure.outputs.build_stages }} - changed_projects: ${{ needs.configure.outputs.changed_projects }} prebuilt_stages: ${{ inputs.prebuilt_stages || '' }} baseline_run_id: ${{ inputs.baseline_run_id || '' }} build_python_packages: ${{ inputs.build_python_packages || false }} @@ -213,7 +207,7 @@ jobs: "repository": "${{ github.repository }}", "ref": "${{ github.sha }}", "base_sha": "${{ github.event.pull_request.base.sha || github.event.before }}", - "head_sha": "${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}", + "head_sha": "${{ github.sha }}", "pr_number": ${{ github.event.pull_request.number || 'null' }}, "event_name": "${{ github.event_name }}", "skip_ci_config": ".github/skip-ci-config.toml", @@ -289,7 +283,6 @@ jobs: - name: Checkout TheRock repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - persist-credentials: false repository: "ROCm/TheRock" ref: c01e3a3f3f67966dd35612cf363b49964f33cd47 # 2026-10-12 sparse-checkout: build_tools/github_actions From 1f35588497de2ac11bb49ef109e6e1148bc4b4dc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 16 Oct 2026 09:05:21 +0000 Subject: [PATCH 34/89] ci(hrr): keep the local Docker helper out of this workflow The image and hrr_dev.py are a projects/hrr development tool, separate from the CI workflow. --- .github/skip-ci-config.toml | 2 - projects/hrr/tests/docker/Dockerfile | 18 -- projects/hrr/tests/scripts/hrr_dev.py | 217 --------------------- projects/hrr/tests/scripts/test_hrr_dev.py | 70 ------- 4 files changed, 307 deletions(-) delete mode 100644 projects/hrr/tests/docker/Dockerfile delete mode 100644 projects/hrr/tests/scripts/hrr_dev.py delete mode 100644 projects/hrr/tests/scripts/test_hrr_dev.py diff --git a/.github/skip-ci-config.toml b/.github/skip-ci-config.toml index c9e1f1f82e8f..ba3e478561ef 100644 --- a/.github/skip-ci-config.toml +++ b/.github/skip-ci-config.toml @@ -51,8 +51,6 @@ common = [ ".github/workflows/hrr-ci.yml", ".github/workflows/hrr-gpu.yml", "projects/hrr/tests/config/ci.yaml", - "projects/hrr/tests/docker/*", - "projects/hrr/tests/scripts/hrr_dev.py", ] # Patterns that skip CI only on Linux (e.g., Windows-only projects) diff --git a/projects/hrr/tests/docker/Dockerfile b/projects/hrr/tests/docker/Dockerfile deleted file mode 100644 index b30596b8662d..000000000000 --- a/projects/hrr/tests/docker/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -# Local HRR development image. The host only needs Docker. -# CMake, Ninja and Python 3 come from the TheRock manylinux build image -# that hrr-gpu.yml pins. This adds PyYAML for the HRR scripts, the GitHub -# CLI that hrr_capture_build.py uses to fetch a base, and GCC 13, the -# minimum emulation/rocjitsu/docs/building.md asks for. -FROM ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:cf4f6d9909056906e4a1c0bc4c245658a0af4c6a6174082a86adfa2d518b7aeb - -USER root -RUN python3 -m pip install --no-cache-dir 'PyYAML==6.0.2' \ - && curl -fsSL https://cli.github.com/packages/rpm/gh-cli.repo -o /etc/yum.repos.d/gh-cli.repo \ - && dnf install -y gh gcc-toolset-13 \ - && dnf clean all \ - && groupadd -f video -ENV PATH=/opt/rh/gcc-toolset-13/root/usr/bin:${PATH} -WORKDIR /workspace diff --git a/projects/hrr/tests/scripts/hrr_dev.py b/projects/hrr/tests/scripts/hrr_dev.py deleted file mode 100644 index f62477fb623f..000000000000 --- a/projects/hrr/tests/scripts/hrr_dev.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""Build and run the HRR suites in a local Docker container. - -The host needs Docker. CMake, Ninja, a ROCm toolchain and rocJitsu are inside -the hrr-dev image, which this script builds the first time it is missing. -`emulator` runs the integration drivers under the rocJitsu configs in -ci.yaml. `gpu` runs them on the machine's GPUs. The host unit binary has no -device code, so both commands run it once inside the container and do not -launch it under rocJitsu or on a GPU. - - python3 projects/hrr/tests/scripts/hrr_dev.py emulator - python3 projects/hrr/tests/scripts/hrr_dev.py gpu -""" - -from __future__ import annotations - -import argparse -import os -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path - -import hrr_ci_configure - -IMAGE = "hrr-dev" -CONTAINER = "hrr-dev" -REPO = Path(__file__).resolve().parents[4] - -# Family keys whose rocJitsu target is not the key itself. -_ARCH = { - "gfx94x": "gfx942", - "gfx103x": "gfx1030", - "gfx110x": "gfx1100", - "gfx120x": "gfx1201", - "gfx125x": "gfx1250", -} - - -@dataclass(frozen=True) -class DockerState: - image: bool - container: str | None # None, "running", or "exited" - - -def arch_for(family: str) -> str: - return _ARCH.get(family, family) - - -def emulator_jobs(config_path: Path | None = None) -> list[dict]: - config = hrr_ci_configure.load_config( - config_path or hrr_ci_configure.DEFAULT_CONFIG - ) - return [ - { - "family": family, - "arch": arch_for(family), - "config": config.simulator.configs[family], - } - for family in config.simulator.default_families - ] - - -def docker_build_command(repo: Path) -> list[str]: - return [ - "docker", - "build", - "-t", - IMAGE, - "-f", - str(repo / "projects/hrr/tests/docker/Dockerfile"), - str(repo), - ] - - -def docker_run_command(repo: Path, backend: str) -> list[str]: - command = [ - "docker", - "run", - "-d", - "--name", - CONTAINER, - "-v", - f"{repo}:{repo}", - "-w", - str(repo), - "--user", - f"{os.getuid()}:{os.getgid()}", - ] - if backend == "gpu": - command += [ - "--device", - "/dev/kfd", - "--device", - "/dev/dri", - "--group-add", - "video", - ] - command += [IMAGE, "sleep", "infinity"] - return command - - -def plan_docker(repo: Path, backend: str, state: DockerState) -> list[list[str]]: - """Commands that make the hrr-dev container ready. An existing container - is started, not recreated.""" - commands = [] - if not state.image: - commands.append(docker_build_command(repo)) - if state.container is None: - commands.append(docker_run_command(repo, backend)) - elif state.container != "running": - commands.append(["docker", "start", CONTAINER]) - return commands - - -def inspect_docker(runner=subprocess.run) -> DockerState: - image = runner( - ["docker", "image", "inspect", IMAGE], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - container = runner( - ["docker", "inspect", "-f", "{{.State.Status}}", CONTAINER], - capture_output=True, - text=True, - ) - status = container.stdout.strip() if container.returncode == 0 else None - return DockerState(image.returncode == 0, status or None) - - -def container_script(repo: Path, backend: str) -> str: - """Bash run inside the container. One rocJitsu family failing still runs - the other.""" - dist = f"{repo}/build/hrr-dev/dist" - work = f"{repo}/build/hrr-dev" - headers = f"{repo}/projects/hip/include;{repo}/projects/clr/hipamd/include" - lines = [ - "set -euo pipefail", - f'export ROCM_PATH="{dist}"', - "python3 projects/hrr/tests/scripts/hrr_capture_build.py fetch-base " - f'--dist "{dist}" --build "{work}" --base-json "{work}/base.json"', - "cmake -S projects/hrr -B build/hrr-dev/unit -G Ninja " - "-DCMAKE_BUILD_TYPE=Release " - f'-DROCM_PATH="{dist}" -DCMAKE_PREFIX_PATH="{dist}" ' - f'"-DHRR_HIP_INCLUDE_DIRS={headers}" ' - "-DHRR_BUILD_PLAYBACK=ON -DHRR_BUILD_TESTS=ON -DHRR_BUILD_INTEGRATION_TESTS=OFF", - "cmake --build build/hrr-dev/unit -j", - "python3 projects/hrr/tests/scripts/hrr_capture_build.py build " - f'--dist "{dist}" --build "{work}"', - "set +e", - "status=0", - "python3 projects/hrr/tests/scripts/run_catch2.py " - "--xml build/hrr-dev/unit.xml --config projects/hrr/tests/config/unit.yaml " - "--arch none -- build/hrr-dev/unit/tests/unit/hrr-unit-tests || status=1", - "if [ -x build/hrr-dev/unit/tests/unit/hrr-handler-tests ]; then " - "python3 projects/hrr/tests/scripts/run_catch2.py " - "--xml build/hrr-dev/handler.xml --config projects/hrr/tests/config/handler.yaml " - "--arch none -- build/hrr-dev/unit/tests/unit/hrr-handler-tests || status=1; fi", - ] - binary = f"{work}/hrr/tests/integration/hrr-integration-tests" - if backend == "emulator": - for job in emulator_jobs(): - launcher = f"rocjitsu --daemon --config {repo}/{job['config']} --" - lines.append( - "python3 projects/hrr/tests/scripts/run_catch2.py " - f"--xml build/hrr-dev/integration-{job['family']}.xml --timeout 1200 " - "--config projects/hrr/tests/config/integration.yaml " - f"--os linux --arch {job['arch']} --launcher \"{launcher}\" " - f"-- {binary} || status=1" - ) - else: - lines.append( - "python3 projects/hrr/tests/scripts/run_catch2.py " - "--xml build/hrr-dev/integration-gpu.xml --timeout 1200 " - "--config projects/hrr/tests/config/integration.yaml " - f"--os linux --arch auto -- {binary} || status=1" - ) - lines.append("exit $status") - return "\n".join(lines) + "\n" - - -def exec_command(repo: Path, backend: str) -> list[str]: - command = ["docker", "exec", "-w", str(repo)] - token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") - if token: - command += ["-e", f"GH_TOKEN={token}"] - command += [CONTAINER, "bash", "-lc", container_script(repo, backend)] - return command - - -def prepare(repo: Path, backend: str, runner=subprocess.run) -> None: - for command in plan_docker(repo, backend, inspect_docker(runner)): - subprocess.run(command, check=True) - subprocess.run(exec_command(repo, backend), check=True) - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("backend", choices=("emulator", "gpu")) - parser.add_argument("--repo", type=Path, default=REPO) - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - args = parse_args(argv) - try: - prepare(args.repo.resolve(), args.backend) - except subprocess.CalledProcessError as exc: - return exc.returncode or 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/projects/hrr/tests/scripts/test_hrr_dev.py b/projects/hrr/tests/scripts/test_hrr_dev.py deleted file mode 100644 index e61a5666a1af..000000000000 --- a/projects/hrr/tests/scripts/test_hrr_dev.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -# SPDX-License-Identifier: MIT -"""Tests for hrr_dev.py. Docker is not started.""" - -import sys -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -import hrr_dev # noqa: E402 - - -class ParseTest(unittest.TestCase): - def test_emulator_and_gpu(self): - self.assertEqual(hrr_dev.parse_args(["emulator"]).backend, "emulator") - self.assertEqual(hrr_dev.parse_args(["gpu"]).backend, "gpu") - - -class DockerPlanTest(unittest.TestCase): - def test_missing_image_is_a_docker_build(self): - repo = Path("/repo") - commands = hrr_dev.plan_docker( - repo, "emulator", hrr_dev.DockerState(image=False, container=None) - ) - self.assertEqual(commands[0][:3], ["docker", "build", "-t"]) - self.assertIn("hrr-dev", commands[0]) - self.assertIn(str(repo / "projects/hrr/tests/docker/Dockerfile"), commands[0]) - - def test_stopped_container_is_started(self): - commands = hrr_dev.plan_docker( - Path("/repo"), "gpu", hrr_dev.DockerState(image=True, container="exited") - ) - self.assertEqual(commands, [["docker", "start", "hrr-dev"]]) - - def test_only_gpu_adds_devices(self): - repo = Path("/repo") - state = hrr_dev.DockerState(image=True, container=None) - emulator = hrr_dev.plan_docker(repo, "emulator", state)[0] - gpu = hrr_dev.plan_docker(repo, "gpu", state)[0] - self.assertNotIn("/dev/kfd", emulator) - self.assertNotIn("/dev/dri", emulator) - self.assertEqual( - gpu[gpu.index("--device") : gpu.index("--device") + 4], - ["--device", "/dev/kfd", "--device", "/dev/dri"], - ) - - -class ScriptTest(unittest.TestCase): - def test_emulator_uses_both_default_configs(self): - script = hrr_dev.container_script(Path("/repo"), "emulator") - self.assertIn("gfx950_mi355x_kmd_2gpu.json", script) - self.assertIn("gfx1250_mi455x_kmd_4gpu.json", script) - self.assertIn("--arch gfx950", script) - self.assertIn("--arch gfx1250", script) - self.assertIn("rocjitsu --daemon", script) - self.assertNotIn("/dev/kfd", script) - self.assertIn("HRR_BUILD_INTEGRATION_TESTS=OFF", script) - self.assertIn("hrr_capture_build.py build", script) - - def test_gpu_uses_arch_auto_and_no_launcher(self): - script = hrr_dev.container_script(Path("/repo"), "gpu") - self.assertIn("--arch auto", script) - self.assertNotIn("rocjitsu", script) - self.assertNotIn("gfx950_mi355x_kmd_2gpu.json", script) - - -if __name__ == "__main__": - unittest.main() From f7f5d34c0b8e3caf9096e9bbae628406b865d9ab Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 16 Oct 2026 09:16:32 +0000 Subject: [PATCH 35/89] ci(hrr): restore librocm_kpack with the capture cache A cache hit skipped the build and verify could not start the integration binary. The build now copies that library into dist/lib, and the cache key is v2 so the incomplete v1 entry is not restored. --- .github/workflows/hrr-gpu.yml | 9 ++-- .../hrr/tests/scripts/hrr_capture_build.py | 48 +++++++++++++++++++ .../tests/scripts/test_hrr_capture_build.py | 14 ++++++ 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 52045d96fd47..4df984639f38 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -89,9 +89,12 @@ jobs: with: path: | build/capture-lib - build/hrr/playback/hrr-playback - build/hrr/tests/integration - key: hrr-capture-v1-${{ steps.cache_key.outputs.digest }} + build/hrr + dist/lib/librocm_kpack.so + dist/lib/librocm_kpack.so.0 + # v2 adds the HRR build tree and librocm_kpack. v1 restored the test + # binary without that library, and verify could not start it. + key: hrr-capture-v2-${{ steps.cache_key.outputs.digest }} - name: Build if: steps.capture_cache.outputs.cache-hit != 'true' diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index e33a59cfa252..45c4dfad13d0 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -274,6 +274,50 @@ def linkage_errors(ldd_output: str, lib_dir: Path) -> list[str]: return errors +def staged_library_sources(ldd_output: str, lib_dir: Path) -> list[Path]: + """Libraries ldd resolved outside lib_dir and outside the system directories. + + The integration binary's RUNPATH searches dist/lib, which is where it loads + amdhip64. A cache restore skips the build, so a dependency that the build + tree had and the base artifacts do not (librocm_kpack.so.0) has to be + copied here for the restored binary to start. + """ + system_roots = (Path("/lib"), Path("/lib64"), Path("/usr/lib"), Path("/usr/lib64")) + sources: list[Path] = [] + for match in re.finditer(r"^\s*\S+ => (\S+)", ldd_output, re.MULTILINE): + raw = match.group(1) + if raw == "not": + continue + src = Path(raw) + if src.parent == lib_dir: + continue + if any(src.is_relative_to(root) for root in system_roots): + continue + sources.append(src) + return sources + + +def stage_runtime_libs(binary: Path, lib_dir: Path) -> list[str]: + """Copy non-system libraries the binary loads into lib_dir. Returns names copied.""" + if not binary.is_file(): + return [] + out = subprocess.run( + ["ldd", str(binary)], check=True, capture_output=True, text=True + ).stdout + lib_dir.mkdir(parents=True, exist_ok=True) + copied: list[str] = [] + for src in staged_library_sources(out, lib_dir): + if not src.is_file(): + continue + dest = lib_dir / src.name + if dest.exists(): + continue + shutil.copy2(src, dest, follow_symlinks=True) + copied.append(dest.name) + print(f"staged {dest.name} from {src}") + return copied + + def check_linkage(binary: Path, lib_dir: Path) -> int: out = subprocess.run( ["ldd", str(binary)], check=True, capture_output=True, text=True @@ -314,6 +358,10 @@ def cmd_build(args: argparse.Namespace, runner: Runner = run) -> int: copy_amdhip64(args.dist / "lib", args.capture_lib) runner(hrr_configure_command(args.src, args.build, args.dist, args.archs)) runner(["cmake", "--build", args.build / "hrr", "--target", *HRR_TARGETS]) + stage_runtime_libs( + args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests", + args.dist / "lib", + ) return 0 diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index 22c90657bb8c..e0d355f1b6eb 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -217,6 +217,20 @@ def runner(cmd): ) +class StageRuntimeLibsTest(unittest.TestCase): + def test_kpack_outside_the_install_is_staged(self): + text = "\n".join( + [ + "libamdhip64.so.7 => /work/dist/lib/libamdhip64.so.7 (0x1)", + "libhsa-runtime64.so.1 => /work/dist/lib/libhsa-runtime64.so.1 (0x2)", + "librocm_kpack.so.0 => /work/build/clr/lib/librocm_kpack.so.0 (0x3)", + "libc.so.6 => /lib64/libc.so.6 (0x4)", + ] + ) + sources = cb.staged_library_sources(text, Path("/work/dist/lib")) + self.assertEqual(sources, [Path("/work/build/clr/lib/librocm_kpack.so.0")]) + + class CacheKeyTest(unittest.TestCase): def test_inputs_are_the_build_and_not_ci_yaml(self): paths = cb.CACHE_SOURCE_PATHS From c5d6628ac97b631231cf38dfeed4da12447aefc4 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 19 Oct 2026 09:54:37 +0200 Subject: [PATCH 36/89] ci(hrr): make the capture build's RUNPATH relocatable for the rocJitsu jobs The integration binary's RUNPATH named the build container's workspace path, so on the bare runner ldd reported libamdhip64 and libhsa-runtime64 as not found and install-base failed. Add relative entries to dist/lib and normalise the paths the linkage check compares. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/scripts/hrr_capture_build.py | 11 +++++++++++ projects/hrr/tests/scripts/test_hrr_capture_build.py | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index 45c4dfad13d0..afe9537480c8 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -203,6 +203,9 @@ def clr_configure_command(src: Path, build: Path, dist: Path) -> list: ] +BUILD_RPATH = "$ORIGIN/../../../../dist/lib;$ORIGIN/../../../dist/lib" + + def hrr_configure_command(src: Path, build: Path, dist: Path, archs: str) -> list: llvm = dist / "lib" / "llvm" # One compiler for C, C++ and HIP, as TheRock builds HRR: host objects from @@ -218,6 +221,12 @@ def hrr_configure_command(src: Path, build: Path, dist: Path, archs: str) -> lis "-DCMAKE_BUILD_TYPE=Release", f"-DCMAKE_PREFIX_PATH={dist}", f"-DROCM_PATH={dist}", + # CMake bakes the absolute dist/lib into RUNPATH, which is the build + # container's workspace path. The rocJitsu jobs run on the bare runner, + # whose workspace path differs. These entries reach dist/lib from the + # integration binary and from the playback binary, wherever the + # workspace sits. + f"-DCMAKE_BUILD_RPATH={BUILD_RPATH}", f"-DCMAKE_C_COMPILER={llvm / 'bin' / 'clang'}", f"-DCMAKE_CXX_COMPILER={llvm / 'bin' / 'clang++'}", f"-DCMAKE_HIP_COMPILER={llvm / 'bin' / 'clang++'}", @@ -261,6 +270,8 @@ def linkage_errors(ldd_output: str, lib_dir: Path) -> list[str]: resolved: dict[str, set[str]] = {} for match in re.finditer(r"^\s*(\S+) => (\S+)", ldd_output, re.MULTILINE): name, path = match.groups() + # A relative RUNPATH entry shows up as .../build/hrr/../../dist/lib. + path = os.path.normpath(path) for stem in LOADED_LIBS: if name.startswith(stem): resolved.setdefault(stem, set()).add(path) diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index e0d355f1b6eb..b6a154209b54 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -71,6 +71,7 @@ def test_hrr_builds_integration_suite_for_archs(self): self.assertIn("-DGPU_TARGETS=gfx942;gfx950", cmd) self.assertIn("-DHRR_BUILD_INTEGRATION_TESTS=ON", cmd) self.assertIn("-DCMAKE_HIP_COMPILER=/d/lib/llvm/bin/clang++", cmd) + self.assertIn(f"-DCMAKE_BUILD_RPATH={cb.BUILD_RPATH}", cmd) def test_build_targets_cover_uploaded_executables(self): names = {Path(rel).name for rel in cb.EXECUTABLES} @@ -163,6 +164,17 @@ def test_both_from_dist(self): [], ) + def test_relative_runpath_entry(self): + text = ( + "\tlibamdhip64.so.7 => /w/build/hrr/tests/integration/../../../../dist/lib/libamdhip64.so.7 (0x1)\n" + "\tlibhsa-runtime64.so.1 => /w/build/hrr/tests/integration/../../../../dist/lib/libhsa-runtime64.so.1 (0x2)\n" + ) + self.assertEqual(cb.linkage_errors(text, self.LIB), []) + + def test_not_found_is_an_error(self): + errors = cb.linkage_errors(self.ldd("not found"), self.LIB) + self.assertEqual(len(errors), 1) + def test_rocr_from_elsewhere(self): errors = cb.linkage_errors( self.ldd("/opt/rocm/lib/libhsa-runtime64.so.1"), self.LIB From 17f8cbc67c0d9a34d611fc59fd28c5ebd5f39fd4 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 19 Oct 2026 10:44:07 +0200 Subject: [PATCH 37/89] ci(hrr): build rocJitsu's interposer and run it in local mode rocjitsu_bin alone does not build librocjitsu.so, so every launch failed with "could not find librocjitsu.so". Daemon mode also fails hipMemCreate and hangs gfx90a replay (#13721, #13722); local mode passes those cases. --- .github/workflows/hrr-gpu.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 4df984639f38..65941b65f4be 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -184,7 +184,7 @@ jobs: run: | set -euo pipefail cmake -S emulation/rocjitsu -B build/rocjitsu -G Ninja -DCMAKE_BUILD_TYPE=Release - cmake --build build/rocjitsu --target rocjitsu_bin -j + cmake --build build/rocjitsu --target rocjitsu_bin rocjitsu_shared -j echo "${GITHUB_WORKSPACE}/build/rocjitsu/tools/rocjitsu" >> "${GITHUB_PATH}" - name: Download build @@ -203,7 +203,9 @@ jobs: FAMILY: ${{ matrix.family }} ARCH: ${{ matrix.arch }} # The config path stays data. The shell does not assemble the command. - LAUNCHER: ${{ format('rocjitsu --daemon --config {0} --', matrix.config) }} + # Local mode: the simulator runs inside each process. Daemon mode + # (rocjitsu --daemon) fails hipMemCreate and hangs gfx90a replay. + LAUNCHER: ${{ format('rocjitsu --config {0} --', matrix.config) }} HRR_TEST_IGNORE_CONFIG: ${{ inputs.ignore_test_config && '1' || '' }} shell: bash run: >- From 6c873d66aadce9c414199fd3d6afcad6985dfac1 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 19 Oct 2026 10:44:07 +0200 Subject: [PATCH 38/89] test(hrr): let a skip rule target runs under a GPU emulator A case can fail only under rocJitsu and pass on hardware. An "emulator: true" key in "when" skips it under run_catch2.py --launcher and keeps it running on real devices. --- projects/hrr/tests/config/handler.yaml | 2 ++ projects/hrr/tests/config/integration.yaml | 2 ++ projects/hrr/tests/config/unit.yaml | 2 ++ projects/hrr/tests/scripts/hrr_test_config.py | 25 +++++++++---- projects/hrr/tests/scripts/run_catch2.py | 2 +- .../hrr/tests/scripts/test_hrr_test_config.py | 26 ++++++++++++++ projects/hrr/tests/scripts/test_run_catch2.py | 35 +++++++++++++++++++ 7 files changed, 87 insertions(+), 7 deletions(-) diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml index 267370f807d6..2a1c0a2d754c 100644 --- a/projects/hrr/tests/config/handler.yaml +++ b/projects/hrr/tests/config/handler.yaml @@ -3,6 +3,8 @@ # # A skip rule matches when every key under "when" matches (a list is OR, a # missing key matches anything, and a value may be a glob such as gfx11*). +# "emulator: true" matches only a run under a GPU emulator (run_catch2.py +# --launcher), "emulator: false" only a run on a real device. # kind: disabled is a temporary regression and requires a public issue URL. # kind: unsupported is permanent and needs only a reason. # diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index fbae575b8726..3d02ff264e2b 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -3,6 +3,8 @@ # # A skip rule matches when every key under "when" matches (a list is OR, a # missing key matches anything, and a value may be a glob such as gfx11*). +# "emulator: true" matches only a run under a GPU emulator (run_catch2.py +# --launcher), "emulator: false" only a run on a real device. # kind: disabled is a temporary regression and requires a public issue URL. # kind: unsupported is permanent and needs only a reason. # diff --git a/projects/hrr/tests/config/unit.yaml b/projects/hrr/tests/config/unit.yaml index e53e576ab044..2dacd2b2c8bd 100644 --- a/projects/hrr/tests/config/unit.yaml +++ b/projects/hrr/tests/config/unit.yaml @@ -3,6 +3,8 @@ # # A skip rule matches when every key under "when" matches (a list is OR, a # missing key matches anything, and a value may be a glob such as gfx11*). +# "emulator: true" matches only a run under a GPU emulator (run_catch2.py +# --launcher), "emulator: false" only a run on a real device. # kind: disabled is a temporary regression and requires a public issue URL. # kind: unsupported is permanent and needs only a reason. # diff --git a/projects/hrr/tests/scripts/hrr_test_config.py b/projects/hrr/tests/scripts/hrr_test_config.py index bbd4a66a62c3..82faa9caa3bd 100644 --- a/projects/hrr/tests/scripts/hrr_test_config.py +++ b/projects/hrr/tests/scripts/hrr_test_config.py @@ -11,6 +11,8 @@ A rule matches when every key it sets matches. Keys in one ``when`` are ANDed; a list under a key is ORed; a missing key matches anything. Glob patterns (``gfx11*``) are allowed. Several visible GPUs match when any of them does. +``emulator: true`` matches only a run under a GPU emulator (run_catch2.py +``--launcher``), ``emulator: false`` only a run on a real device. ``kind: disabled`` is a temporary regression and needs a public issue URL. ``kind: unsupported`` is permanent and needs only a reason. @@ -37,7 +39,7 @@ ARCH_TOKEN = re.compile(r"gfx[0-9][0-9a-z]*") WORKLOAD_TAGS = {"hrr-direct", "direct"} KINDS = {"disabled", "unsupported"} -WHEN_KEYS = {"os", "arch"} +WHEN_KEYS = {"os", "arch", "emulator"} RULE_KEYS = {"when", "kind", "reason", "issue"} CASE_KEYS = {"skip"} TOP_KEYS = {"version", "suite", "binary", "select", "targets", "skip", "cases"} @@ -47,6 +49,8 @@ # # A skip rule matches when every key under "when" matches (a list is OR, a # missing key matches anything, and a value may be a glob such as gfx11*). +# "emulator: true" matches only a run under a GPU emulator (run_catch2.py +# --launcher), "emulator: false" only a run on a real device. # kind: disabled is a temporary regression and requires a public issue URL. # kind: unsupported is permanent and needs only a reason. # @@ -96,16 +100,21 @@ def value_matches(pattern, value: str) -> bool: return any(fnmatch.fnmatchcase(value, str(item)) for item in _as_list(pattern)) -def rule_matches(rule: dict, os_name: str, archs: list[str]) -> list[str] | None: +def rule_matches( + rule: dict, os_name: str, archs: list[str], emulated: bool = False +) -> list[str] | None: """The architectures that made the rule match, or None. An ``arch`` key matches when any visible architecture matches, and does - not match when no architecture was supplied. An empty ``when`` matches - every machine. + not match when no architecture was supplied. An ``emulator`` key matches + a run under a GPU emulator (true) or on a real device (false). An empty + ``when`` matches every machine. """ when = rule.get("when") or {} if "os" in when and not value_matches(when["os"], os_name): return None + if "emulator" in when and when["emulator"] != emulated: + return None matched = list(archs) if "arch" in when: matched = [arch for arch in archs if value_matches(when["arch"], arch)] @@ -120,7 +129,7 @@ def _rules_for(config: dict, name: str) -> list[dict]: def resolve(config: dict, names: list[str], os_name: str, archs: list[str], - ignore: bool = False) -> tuple[list[str], list[dict]]: + ignore: bool = False, emulated: bool = False) -> tuple[list[str], list[dict]]: """Split ``names`` into cases to run and cases a rule skips. The first matching rule wins, suite rules before the case's own. Each @@ -135,7 +144,7 @@ def resolve(config: dict, names: list[str], os_name: str, archs: list[str], hit = None matched: list[str] = [] for rule in _rules_for(config, name): - matched = rule_matches(rule, os_name, archs) + matched = rule_matches(rule, os_name, archs, emulated) if matched is not None: hit = rule break @@ -195,6 +204,8 @@ def _check_rule(errors: list[str], where: str, rule, targets: dict) -> None: for key in ("os", "arch"): if key in when: _check_pattern(errors, where, key, when[key], targets.get(key, [])) + if "emulator" in when and not isinstance(when["emulator"], bool): + _error(errors, where, "'emulator' must be true or false") kind = rule.get("kind") if kind not in KINDS: _error(errors, where, "'kind' must be disabled or unsupported") @@ -345,6 +356,8 @@ def _yaml_pattern(value) -> str: def _emit_rule(lines: list[str], rule: dict, indent: str) -> None: when = rule.get("when") or {} parts = [f"{key}: {_yaml_pattern(when[key])}" for key in ("os", "arch") if key in when] + if "emulator" in when: + parts.append(f"emulator: {'true' if when['emulator'] else 'false'}") lines.append(f"{indent}- when: {{{', '.join(parts)}}}") lines.append(f"{indent} kind: {rule['kind']}") lines.append(f"{indent} reason: {_yaml_str(rule['reason'])}") diff --git a/projects/hrr/tests/scripts/run_catch2.py b/projects/hrr/tests/scripts/run_catch2.py index b6af8604bae5..606296cf6083 100644 --- a/projects/hrr/tests/scripts/run_catch2.py +++ b/projects/hrr/tests/scripts/run_catch2.py @@ -247,7 +247,7 @@ def _apply_config( # in CI. Tests mock list_cases. names = [case["name"] for case in listed] _run, skipped = hrr_test_config.resolve( - config, names, os_name, archs, ignore=ignore + config, names, os_name, archs, ignore=ignore, emulated=bool(args.launcher) ) records = [ ( diff --git a/projects/hrr/tests/scripts/test_hrr_test_config.py b/projects/hrr/tests/scripts/test_hrr_test_config.py index cd264df59059..ffd11cd78062 100644 --- a/projects/hrr/tests/scripts/test_hrr_test_config.py +++ b/projects/hrr/tests/scripts/test_hrr_test_config.py @@ -110,6 +110,25 @@ def test_no_device_does_not_match_an_arch_rule(self): run, _ = cfg.resolve(suite(cases), ["Filtered"], "linux", []) self.assertEqual(run, ["Filtered"]) + def test_emulator_rule_matches_only_an_emulated_run(self): + cases = {"Filtered": {"skip": [rule({"emulator": True, "arch": "gfx1151"})]}} + config = suite(cases) + run, skipped = cfg.resolve( + config, ["Filtered"], "linux", ["gfx1151"], emulated=True + ) + self.assertEqual(run, []) + self.assertEqual(skipped[0]["archs"], ["gfx1151"]) + run, _ = cfg.resolve(config, ["Filtered"], "linux", ["gfx1151"]) + self.assertEqual(run, ["Filtered"]) + + def test_emulator_false_rule_matches_only_a_real_device(self): + cases = {"Filtered": {"skip": [rule({"emulator": False})]}} + config = suite(cases) + run, _ = cfg.resolve(config, ["Filtered"], "linux", ["gfx90a"], emulated=True) + self.assertEqual(run, ["Filtered"]) + run, _ = cfg.resolve(config, ["Filtered"], "linux", ["gfx90a"]) + self.assertEqual(run, []) + def test_ignore_runs_everything(self): cases = {"Filtered": {"skip": [rule({"os": "linux"})]}} config = suite(cases, skip=[rule({"arch": "gfx90a"})]) @@ -126,6 +145,13 @@ def test_a_valid_config_is_quiet(self): ]}} self.assertEqual(cfg.lint(suite(cases)), []) + def test_emulator_must_be_a_bool(self): + cases = {"Filtered": {"skip": [rule({"emulator": "yes"})]}} + errors = cfg.lint(suite(cases)) + self.assertTrue(any("emulator" in error for error in errors)) + cases = {"Filtered": {"skip": [rule({"emulator": True, "arch": "gfx90a"})]}} + self.assertEqual(cfg.lint(suite(cases)), []) + def test_disabled_needs_an_issue_and_every_rule_needs_a_reason(self): cases = {"Filtered": {"skip": [{"when": {"os": "linux"}, "kind": "disabled"}]}} errors = cfg.lint(suite(cases)) diff --git a/projects/hrr/tests/scripts/test_run_catch2.py b/projects/hrr/tests/scripts/test_run_catch2.py index 7b99a105d49e..c8468f492f1d 100644 --- a/projects/hrr/tests/scripts/test_run_catch2.py +++ b/projects/hrr/tests/scripts/test_run_catch2.py @@ -300,6 +300,41 @@ def test_config_skips_a_case_and_records_why(self): self.assertIsNotNone(skipped) self.assertTrue(skipped.attrib["message"].startswith("hrr-config:")) + def test_emulator_rule_applies_only_with_a_launcher(self): + path = self._config( + { + "Keep": {}, + "Drop": { + "skip": [ + { + "when": {"emulator": True}, + "kind": "unsupported", + "reason": "emulator only", + } + ] + }, + } + ) + listed = [ + {"name": "Keep", "tags": ["[hrr]"]}, + {"name": "Drop", "tags": ["[hrr]"]}, + ] + with mock.patch("hrr_test_config.list_cases", return_value=listed): + rc, out, _ = self.run_main( + {"xml": junit(record("Keep"))}, + "--config", str(path), "--os", "linux", "--arch", "gfx1151", + "--launcher", "env", + ) + self.assertEqual(rc, 0) + self.assertIn("SKIP (config): Drop", out) + with mock.patch("hrr_test_config.list_cases", return_value=listed): + rc, out, _ = self.run_main( + {"xml": junit(record("Keep"), record("Drop"))}, + "--config", str(path), "--os", "linux", "--arch", "gfx1151", + ) + self.assertEqual(rc, 0) + self.assertNotIn("SKIP (config)", out) + def test_config_runs_nothing_when_every_case_is_skipped(self): path = self._config( { From 3aa6f9512455e2efe2c86734364215a4f277a908 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 19 Oct 2026 10:44:07 +0200 Subject: [PATCH 39/89] ci(hrr): run every rocJitsu family by default, no real-GPU job Default events run the rocJitsu job on gfx90a, gfx94x, gfx950, gfx110x, gfx1151, gfx120x and gfx125x. Real GPUs run only from workflow_dispatch with backend gpu. Cases that fail only under the emulator are skipped with "emulator: true" rules that cite their issue (#13723, #13724, #13726). --- projects/hrr/tests/config/ci.yaml | 26 +++++---- projects/hrr/tests/config/integration.yaml | 48 ++++++++++++++--- .../tests/scripts/test_hrr_ci_configure.py | 54 +++++++++++++++++-- 3 files changed, 108 insertions(+), 20 deletions(-) diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 066cfe6115a4..997cd7f21a1f 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -57,17 +57,25 @@ capture_label: ci:hrr-capture # simulator runs rocJitsu on ubuntu-24.04. gpu runs TheRock's real runners. # workflow_dispatch passes its backend input. Any other event uses default, -# unless the event is listed under backend.gpu.triggers. +# unless the event is listed under backend.gpu.triggers. No event is, so a +# pull request, a push and the nightly schedule never queue on a real GPU +# runner; a person asks for one with workflow_dispatch (backend gpu). backend: default: simulator gpu: triggers: [] -# Default simulator jobs. These do not consult families.*.enabled: gfx125x -# stays disabled for the real MI455 runner and still runs here. +# Default simulator jobs. These do not consult families.*.enabled. Each row is +# a rocJitsu KMD config that exposes the family's GPU to HIP, and the whole +# integration suite passes on it, less the cases integration.yaml skips with +# "emulator: true" (each names its reason). Checked 2026-10-19 on a +# two-GPU gfx950 config and the single-GPU configs of the others. +# +# Not listed: gfx103x, gfx1150 and gfx1153, which rocJitsu has no config for. +# A ci:gfx... label for one of them fails the configure job. simulator: runner: ubuntu-24.04 - default_families: [gfx950, gfx125x] + default_families: [gfx90a, gfx94x, gfx950, gfx110x, gfx1151, gfx120x, gfx125x] configs: gfx94x: emulation/rocjitsu/configs/gfx942_cdna3_kmd.json gfx950: emulation/rocjitsu/configs/gfx950_mi355x_kmd_2gpu.json @@ -99,11 +107,11 @@ simulator: # runner pool and the summary waits for all of them, so to reduce GPU load or # take a pool out, set enabled: false or narrow triggers here. # -# Real GPUs: only gfx94x is enabled. The other families stay listed and -# disabled, so a ci:gfx... label or the workflow_dispatch families input can -# still request one when the backend is gpu. Delete an `enabled: false` line -# to make that family eligible for the GPU backend; it does not add a -# simulator job. Simulator membership is simulator.default_families. +# Real GPUs run only when a person starts workflow_dispatch with backend gpu. +# Then only gfx94x is enabled; the other families stay listed and disabled, so +# the families input can still request one. Delete an `enabled: false` line to +# make that family eligible for the GPU backend; it does not add a simulator +# job. Simulator membership is simulator.default_families. # targets.arch in integration.yaml stays wide on purpose: configure fails # unless it covers every family listed here, and a requested family needs its # architecture compiled into the binary. diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 3d02ff264e2b..3fb7e78530c7 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -44,7 +44,12 @@ cases: Unit_HRR_ApiMatrix_T1_Roundtrip: {} Unit_HRR_ApiMatrix_T2_Roundtrip: {} Unit_HRR_ApiMatrix_T3_Roundtrip: {} - Unit_HRR_ApiMatrix_T4_Roundtrip: {} + Unit_HRR_ApiMatrix_T4_Roundtrip: + skip: + - when: {arch: ["gfx11*", "gfx120*"], emulator: true} + kind: disabled + reason: "hipMemcpyToArray and hipMemcpyFromArray copy no data under the RDNA3 and RDNA4 rocJitsu configs" + issue: "https://github.com/ROCm/rocm-systems/issues/13724" Unit_HRR_ApiMatrix_T5_Roundtrip: {} Unit_HRR_ApiMatrix_UC1Dense_Direct: {} Unit_HRR_BlankCaptureOutputCapturesNothing: {} @@ -110,7 +115,11 @@ cases: Unit_HRR_DiskSpace_StopsBeforeReserve: {} Unit_HRR_DiskSpace_StopsBeforeReserve_Direct: {} Unit_HRR_DispatchSwap_Direct: {} - Unit_HRR_DispatchSwap_NoUnreplayableAfterTrailer: {} + Unit_HRR_DispatchSwap_NoUnreplayableAfterTrailer: + skip: + - when: {emulator: true} + kind: unsupported + reason: "The case replaces LD_PRELOAD in its child, which drops the rocJitsu interposer and leaves the child with no device" Unit_HRR_DispatchSwap_Unreplayable_Direct: {} Unit_HRR_DispatchSwap_WrapperOutlivesCapture: {} Unit_HRR_DivergenceAbortRoundtrip: {} @@ -129,7 +138,12 @@ cases: Unit_HRR_FailedBlobClose_Direct: {} Unit_HRR_FailedBlobClose_LeavesCaptureIncomplete: {} Unit_HRR_FailedBlobWrite_Direct: {} - Unit_HRR_FailedBlobWrite_LeavesCaptureIncomplete: {} + Unit_HRR_FailedBlobWrite_LeavesCaptureIncomplete: + skip: + - when: {arch: ["gfx94*", "gfx95*", "gfx125*"], emulator: true} + kind: disabled + reason: "Local-mode rocJitsu runs inside the process, and the case lowers RLIMIT_FSIZE, which also limits the simulator" + issue: "https://github.com/ROCm/rocm-systems/issues/13726" Unit_HRR_FailedMemcpy3DNotRecorded: {} Unit_HRR_FailedMemcpy3D_Direct: {} Unit_HRR_FailedShimCallsNotRecorded: {} @@ -148,8 +162,18 @@ cases: Unit_HRR_ForkWhileRecording_Direct: {} Unit_HRR_ForkWhileWriterHoldsLock: {} Unit_HRR_ForkWhileWriterHoldsLock_Direct: {} - Unit_HRR_Fork_ChildArchiveCompleteAfterParentFailure: {} - Unit_HRR_Fork_ChildListsOnlyItsOwnUnreplayableApis: {} + Unit_HRR_Fork_ChildArchiveCompleteAfterParentFailure: + skip: + - when: {arch: ["gfx94*", "gfx95*", "gfx125*"], emulator: true} + kind: disabled + reason: "Local-mode rocJitsu runs inside the process, and the case lowers RLIMIT_FSIZE, which also limits the simulator" + issue: "https://github.com/ROCm/rocm-systems/issues/13726" + Unit_HRR_Fork_ChildListsOnlyItsOwnUnreplayableApis: + skip: + - when: {arch: ["gfx94*", "gfx95*", "gfx125*"], emulator: true} + kind: disabled + reason: "Local-mode rocJitsu runs inside the process, and the case lowers RLIMIT_FSIZE, which also limits the simulator" + issue: "https://github.com/ROCm/rocm-systems/issues/13726" Unit_HRR_Fork_Direct: {} Unit_HRR_ForkedChildRecordsAfterShutdown: {} Unit_HRR_ForkedChildRecordsAfterShutdown_Direct: {} @@ -178,7 +202,12 @@ cases: Unit_HRR_MemPoolExtended_Direct: {} Unit_HRR_Memcpy3DSptRoundtrip: {} Unit_HRR_Memcpy3DSpt_Direct: {} - Unit_HRR_MemcpyExtraRoundtrip: {} + Unit_HRR_MemcpyExtraRoundtrip: + skip: + - when: {arch: ["gfx11*", "gfx120*"], emulator: true} + kind: disabled + reason: "hipMemcpyToArray and hipMemcpyFromArray copy no data under the RDNA3 and RDNA4 rocJitsu configs" + issue: "https://github.com/ROCm/rocm-systems/issues/13724" Unit_HRR_MemcpyExtra_Direct: {} Unit_HRR_MemcpyPeerRoundtrip: {} Unit_HRR_MemcpyPeer_Direct: {} @@ -266,6 +295,11 @@ cases: Unit_HRR_VMMRoundtrip: {} Unit_HRR_VMM_Direct: {} Unit_HRR_VaPlacement_Direct: {} - Unit_HRR_VaPlacement_StoredPointer: {} + Unit_HRR_VaPlacement_StoredPointer: + skip: + - when: {arch: ["gfx95*", "gfx125*"], emulator: true} + kind: disabled + reason: "The multi-GPU rocJitsu configs reject overlapping GEM_VA ranges, so replay cannot place allocations at their capture addresses" + issue: "https://github.com/ROCm/rocm-systems/issues/13723" Unit_HRR_ZeroInitRead_Direct: {} Unit_HRR_ZeroInitRoundtrip: {} diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index b4d1a513a150..9e980790c1db 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -150,11 +150,14 @@ def test_only_gfx94x_runs_by_default(self): self.assertEqual(family.triggers, cfg.EVENTS, key) def test_a_label_adds_a_simulator_family(self): + """A family outside simulator.default_families joins on its label.""" + data = copy.deepcopy(REPO_CONFIG) + data["simulator"]["default_families"] = ["gfx950", "gfx125x"] d = decide( "pull_request", ["projects/hrr/playback/x.cpp"], ["ci:gfx110x"], - config=REPO, + config=cfg.parse_config(data), ) self.assertEqual(family_keys(d), ["gfx950", "gfx110x", "gfx125x"]) self.assertTrue(all(row["backend"] == "simulator" for row in d.families)) @@ -182,7 +185,7 @@ def test_ci_config_runs_the_suite_and_is_not_capture(self): self.assertTrue(cfg.touches_suite(paths, REPO)) decision = decide("pull_request", paths, config=REPO) self.assertTrue(decision.run_gpu) - self.assertEqual(family_keys(decision), ["gfx950", "gfx125x"]) + self.assertEqual(family_keys(decision), list(REPO.simulator.default_families)) def test_public_header_is_cpu_only(self): paths = ["projects/hip/include/hip/hip_runtime_api.h"] @@ -461,9 +464,52 @@ def test_unknown_event(self): class SimulatorBackendTest(unittest.TestCase): - def test_default_event_is_the_two_rocjitsu_jobs(self): + def test_default_runs_no_real_gpu_job_on_any_event(self): + """Pull requests, pushes and the nightly run only rocJitsu jobs, on every + family that has a rocJitsu config; none queues on a GPU runner.""" + for event in ("pull_request", "push", "schedule", "workflow_dispatch"): + decision = decide(event, ["projects/hrr/playback/x.cpp"], config=REPO) + self.assertTrue(decision.run_gpu, event) + self.assertEqual( + family_keys(decision), + [ + "gfx90a", + "gfx94x", + "gfx950", + "gfx110x", + "gfx1151", + "gfx120x", + "gfx125x", + ], + event, + ) + self.assertEqual( + {row["backend"] for row in decision.families}, {"simulator"}, event + ) + self.assertEqual( + {row["runner"] for row in decision.families}, {"ubuntu-24.04"}, event + ) + + def test_every_default_simulator_family_has_a_config_file(self): + root = Path(__file__).resolve().parents[4] + for key in REPO.simulator.default_families: + self.assertTrue((root / REPO.simulator.configs[key]).is_file(), key) + + def test_the_label_does_not_move_a_run_to_a_real_gpu(self): + for label, path in ( + ("ci:hrr-capture", "projects/hrr/README.md"), + ("ci:gfx94x", "projects/hrr/playback/x.cpp"), + ): + decision = decide("pull_request", [path], [label], config=REPO) + self.assertTrue(decision.run_gpu, label) + self.assertEqual( + {row["backend"] for row in decision.families}, {"simulator"}, label + ) + + def test_default_event_is_the_rocjitsu_jobs(self): decision = decide("pull_request", ["projects/hrr/playback/x.cpp"], config=REPO) - self.assertEqual(family_keys(decision), ["gfx950", "gfx125x"]) + self.assertIn("gfx950", family_keys(decision)) + self.assertIn("gfx125x", family_keys(decision)) by_family = {row["family"]: row for row in decision.families} self.assertEqual(by_family["gfx950"]["runner"], "ubuntu-24.04") self.assertEqual(by_family["gfx950"]["backend"], "simulator") From bddf0cdfdf489dc79c4553a4cc2b1c9201cbeb90 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 19 Oct 2026 12:38:16 +0200 Subject: [PATCH 40/89] ci(hrr): check out the rocJitsu configs for script tests, map /__w for emulator jobs The HRR script tests sparse checkout lacked emulation/rocjitsu/configs, which test_every_default_simulator_family_has_a_config_file reads. The test binary is built in a container and spawns itself and the playback tool by absolute /__w/... paths. The rocJitsu jobs run it on the host, so Unit_HRR_CaptureCapabilities and Unit_HRR_KernelArgs_DebugArgsOnlyInDebugRuntime failed with exit 127. Link /__w/rocm-systems/rocm-systems to the workspace. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-ci.yml | 3 ++- .github/workflows/hrr-gpu.yml | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 5c845af3370a..b89992ef1e4b 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -183,13 +183,14 @@ jobs: # Cone mode also checks out projects/clr/hipamd/src/amdhip.def.in, # which gen_hrr_api_args.py reads on import. A test reads hrr-ci.yml. # The generator-rule tests in tools/tests regenerate the handlers from - # the HIP headers. + # the HIP headers. A configure test checks the rocJitsu config files exist. sparse-checkout: | projects/hrr projects/hip/include projects/clr/hipamd/include projects/clr/hipamd/src/hrr .github/workflows + emulation/rocjitsu/configs - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 65941b65f4be..824581ba7de9 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -198,6 +198,14 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: python projects/hrr/tests/scripts/hrr_capture_build.py install-base + # The build job compiled the test binary in a container, where the + # workspace is /__w//, and the binary spawns itself and the + # playback tool by those absolute paths. Make them resolve on this host. + - name: Map the container build path to the workspace + run: | + sudo mkdir -p /__w/rocm-systems + sudo ln -sfn "${GITHUB_WORKSPACE}" /__w/rocm-systems/rocm-systems + - name: Run integration suite env: FAMILY: ${{ matrix.family }} From 403a6914106110c0f0f163ff8f860860be1c681a Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 19 Oct 2026 12:46:05 +0200 Subject: [PATCH 41/89] ci(hrr): make a capture-build cache hit start the test binary A cache hit restored the binary without the libraries the build staged into dist/lib (libLLVM, the sysdeps) and, in v2, without the files librocm_kpack.so.0 needs, so verify failed with 'librocm_kpack.so.0: cannot open shared object file'. Cache every librocm_kpack file (v3) and stage the runtime libraries again after a restore. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-gpu.yml | 15 ++++++--- .../hrr/tests/scripts/hrr_capture_build.py | 31 +++++++++++++++++-- .../tests/scripts/test_hrr_capture_build.py | 13 ++++++++ 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 824581ba7de9..4805eef6cc3c 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -90,16 +90,21 @@ jobs: path: | build/capture-lib build/hrr - dist/lib/librocm_kpack.so - dist/lib/librocm_kpack.so.0 - # v2 adds the HRR build tree and librocm_kpack. v1 restored the test - # binary without that library, and verify could not start it. - key: hrr-capture-v2-${{ steps.cache_key.outputs.digest }} + dist/lib/librocm_kpack* + # v2 added the HRR build tree and librocm_kpack. v3 caches every + # librocm_kpack file. A restore of v2 left librocm_kpack.so.0 missing. + key: hrr-capture-v3-${{ steps.cache_key.outputs.digest }} - name: Build if: steps.capture_cache.outputs.cache-hit != 'true' run: python3 projects/hrr/tests/scripts/hrr_capture_build.py build + # The build stages copies of libLLVM and the sysdeps into dist/lib, and + # the cache does not keep them. + - name: Stage runtime libraries + if: steps.capture_cache.outputs.cache-hit == 'true' + run: python3 projects/hrr/tests/scripts/hrr_capture_build.py stage + - name: Verify run: python3 projects/hrr/tests/scripts/hrr_capture_build.py verify diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index afe9537480c8..087b99e4fb64 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -308,12 +308,14 @@ def staged_library_sources(ldd_output: str, lib_dir: Path) -> list[Path]: return sources -def stage_runtime_libs(binary: Path, lib_dir: Path) -> list[str]: +def stage_runtime_libs( + binary: Path, lib_dir: Path, env: dict | None = None +) -> list[str]: """Copy non-system libraries the binary loads into lib_dir. Returns names copied.""" if not binary.is_file(): return [] out = subprocess.run( - ["ldd", str(binary)], check=True, capture_output=True, text=True + ["ldd", str(binary)], check=True, capture_output=True, text=True, env=env ).stdout lib_dir.mkdir(parents=True, exist_ok=True) copied: list[str] = [] @@ -376,6 +378,28 @@ def cmd_build(args: argparse.Namespace, runner: Runner = run) -> int: return 0 +def cmd_stage(args: argparse.Namespace) -> int: + """Stage the runtime libraries again after a cache restore. + + The cache keeps the binary but not the copies build staged into dist/lib. + Without the build tree, ldd finds the libraries only in the installed + subtrees (dist/lib/llvm/lib, dist/lib/rocm_sysdeps/lib), so put those on + its search path. + """ + lib_dir = args.dist / "lib" + search = [str(p) for p in sorted(lib_dir.glob("*/lib")) if p.is_dir()] + env = dict(os.environ) + env["LD_LIBRARY_PATH"] = os.pathsep.join( + search + ([env["LD_LIBRARY_PATH"]] if env.get("LD_LIBRARY_PATH") else []) + ) + stage_runtime_libs( + args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests", + lib_dir, + env, + ) + return 0 + + def cmd_verify(args: argparse.Namespace) -> int: exe = args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests" run( @@ -480,7 +504,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "command", - choices=("fetch-base", "build", "verify", "install-base", "cache-key"), + choices=("fetch-base", "build", "stage", "verify", "install-base", "cache-key"), ) parser.add_argument("--therock", type=Path, default=Path("therock")) parser.add_argument("--src", type=Path, default=Path.cwd()) @@ -511,6 +535,7 @@ def main(argv: list[str] | None = None) -> int: return { "fetch-base": cmd_fetch_base, "build": cmd_build, + "stage": cmd_stage, "verify": cmd_verify, "install-base": cmd_install_base, "cache-key": cmd_cache_key, diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index b6a154209b54..9a12a390fa1e 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -243,6 +243,19 @@ def test_kpack_outside_the_install_is_staged(self): self.assertEqual(sources, [Path("/work/build/clr/lib/librocm_kpack.so.0")]) +class StageCommandTest(unittest.TestCase): + def test_stage_is_a_command(self): + args = cb.parse_args(["stage"]) + self.assertEqual(args.command, "stage") + + def test_stage_without_a_binary_copies_nothing(self): + with tempfile.TemporaryDirectory() as tmp: + args = cb.parse_args( + ["stage", "--dist", f"{tmp}/dist", "--build", f"{tmp}/build"] + ) + self.assertEqual(cb.cmd_stage(args), 0) + + class CacheKeyTest(unittest.TestCase): def test_inputs_are_the_build_and_not_ci_yaml(self): paths = cb.CACHE_SOURCE_PATHS From 05e7d70543aac760a095fa6ce30580b486372011 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 20 Oct 2026 07:13:49 +0200 Subject: [PATCH 42/89] test(hrr): skip the secure-exec case under the emulator A set-group-ID copy starts in secure-execution mode, where the loader ignores LD_PRELOAD. The copy then loses the rocJitsu interposer, sees no device, and Unit_HRR_GpuWorkload_Direct fails with hipSetDevice error 100. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/integration.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index ec8b895735c4..b8026f54c5d6 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -270,7 +270,11 @@ cases: Unit_HRR_ReplaceKernelBadSpec: {} Unit_HRR_ReplaceKernelMissingCO: {} Unit_HRR_ReplaceKernelRoundtrip: {} - Unit_HRR_SecureExecIgnoresCaptureOutput: {} + Unit_HRR_SecureExecIgnoresCaptureOutput: + skip: + - when: {emulator: true} + kind: unsupported + reason: "The loader ignores LD_PRELOAD in the set-group-ID copy, which drops the rocJitsu interposer and leaves the copy with no device" Unit_HRR_SecureExecProbe_Direct: {} Unit_HRR_ShutdownWhileChildOpensArchive: {} Unit_HRR_ShutdownWhileChildOpensArchive_Direct: {} From dbd3b6d27f5754992a710a7d9c70eb2df1081177 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 20 Oct 2026 07:27:04 +0200 Subject: [PATCH 43/89] ci(hrr): check out runtimes/api-headers for the rocJitsu build rocJitsu on develop now takes the HSA and UAPI headers from runtimes/api-headers and fails to configure without it. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-gpu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 4805eef6cc3c..34c64bb88827 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -148,6 +148,7 @@ jobs: sparse-checkout: | emulation/rocjitsu projects/hrr/tests + runtimes/api-headers shared/machine-readable-isa - name: Checkout TheRock artifact tools From 4713ba07525ae68c3c53441bea7dbcd8f34cf175 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 20 Oct 2026 07:41:58 +0200 Subject: [PATCH 44/89] ci(hrr): build the capture tree every run, no cache restore A cache hit left librocm_kpack.so.0 out of the tree, so Verify failed on the restored binary. Drop the restore and the key step until that is understood. The rocJitsu job takes its libraries from the uploaded build and the base artifacts, not from the cache. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-gpu.yml | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 34c64bb88827..88c66440fc54 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -75,36 +75,13 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: python3 projects/hrr/tests/scripts/hrr_capture_build.py fetch-base - - name: Capture cache key - id: cache_key - run: >- - python3 projects/hrr/tests/scripts/hrr_capture_build.py cache-key - --therock-pin "${THEROCK_PIN}" --image "${BUILD_IMAGE}" - - # Exact key. A prefix restore could pair this tree with another base - # or another targets.arch. - - name: Restore capture build - id: capture_cache - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - build/capture-lib - build/hrr - dist/lib/librocm_kpack* - # v2 added the HRR build tree and librocm_kpack. v3 caches every - # librocm_kpack file. A restore of v2 left librocm_kpack.so.0 missing. - key: hrr-capture-v3-${{ steps.cache_key.outputs.digest }} - + # No cache restore: a cache hit left librocm_kpack.so.0 out of the tree and + # Verify failed. Off until that failure is understood, so the build always + # runs. The `cache-key` and `stage` commands of hrr_capture_build.py are + # the pieces to reuse. - name: Build - if: steps.capture_cache.outputs.cache-hit != 'true' run: python3 projects/hrr/tests/scripts/hrr_capture_build.py build - # The build stages copies of libLLVM and the sysdeps into dist/lib, and - # the cache does not keep them. - - name: Stage runtime libraries - if: steps.capture_cache.outputs.cache-hit == 'true' - run: python3 projects/hrr/tests/scripts/hrr_capture_build.py stage - - name: Verify run: python3 projects/hrr/tests/scripts/hrr_capture_build.py verify From d24aff22acaa985ac55b1f6702ef16888f4cae92 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 20 Oct 2026 09:31:25 +0200 Subject: [PATCH 45/89] test(hrr): skip the two IPC handle cases under the emulator hipIpcGetMemHandle fails under rocJitsu since #13035 (ROCm/rocm-systems#13803), so Unit_HRR_CaptureCapabilities exports no handle and Unit_HRR_ApiMatrix_T1_Roundtrip falls below its evidence floor. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/integration.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index b8026f54c5d6..8c31abe46fd5 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -41,7 +41,12 @@ cases: Unit_HRR_ApiMatrix_SptCoopLaunch_Direct: {} Unit_HRR_ApiMatrix_SptKernelLaunch_Direct: {} Unit_HRR_ApiMatrix_T0_Roundtrip: {} - Unit_HRR_ApiMatrix_T1_Roundtrip: {} + Unit_HRR_ApiMatrix_T1_Roundtrip: + skip: + - when: {emulator: true} + kind: disabled + reason: "hipIpcGetMemHandle fails under rocJitsu since the shared runtime API headers change (hsa_amd_ipc_memory_create returns HSA_STATUS_ERROR)" + issue: "https://github.com/ROCm/rocm-systems/issues/13803" Unit_HRR_ApiMatrix_T2_Roundtrip: {} Unit_HRR_ApiMatrix_T3_Roundtrip: {} Unit_HRR_ApiMatrix_T4_Roundtrip: @@ -57,7 +62,12 @@ cases: Unit_HRR_CaptureActiveMarker: {} Unit_HRR_CaptureActiveMarker_Direct: {} Unit_HRR_CaptureArchiveIsPrivate: {} - Unit_HRR_CaptureCapabilities: {} + Unit_HRR_CaptureCapabilities: + skip: + - when: {emulator: true} + kind: disabled + reason: "hipIpcGetMemHandle fails under rocJitsu since the shared runtime API headers change (hsa_amd_ipc_memory_create returns HSA_STATUS_ERROR)" + issue: "https://github.com/ROCm/rocm-systems/issues/13803" Unit_HRR_CaptureCapabilitiesImport_Direct: {} Unit_HRR_CaptureCapabilities_Direct: {} Unit_HRR_CaptureCrashDuringFork: {} From d3f731ed7748978c8dfd8609fe8296ea0189d3b9 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 20 Oct 2026 10:09:10 +0200 Subject: [PATCH 46/89] ci(hrr): count the rocJitsu jobs as the family integration jobs in the summary The summary looked only for 'Integration ()' jobs, so with the rocJitsu jobs it listed every family as 'did not run' and failed with all jobs green. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/scripts/hrr_ci_report.py | 3 ++- projects/hrr/tests/scripts/test_hrr_ci_report.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index 690375523e62..895374912740 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -36,7 +36,8 @@ ("Script tests", re.compile(r"^HRR script tests$")), ("Unit", re.compile(r"(?:^|/ )Unit \((?P[^)]+)\)$")), ("Build", re.compile(r"(?:^|/ )Build capture-enabled HIP and HRR$")), - ("Integration", re.compile(r"(?:^|/ )Integration \((?P[^)]+)\)$")), + # The real-GPU job is "Integration ()", the rocJitsu one "rocJitsu ()". + ("Integration", re.compile(r"(?:^|/ )(?:Integration|rocJitsu) \((?P[^)]+)\)$")), ) BAD_CONCLUSIONS = {"failure", "cancelled", "timed_out", "startup_failure"} diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index ec132b3f5910..a4d49632adf8 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -54,6 +54,22 @@ def test_queued_job_reports_status(self): self.assertEqual(rows[0]["conclusion"], "queued") +class RocJitsuJobTest(unittest.TestCase): + def test_rocjitsu_job_counts_as_the_family_integration_job(self): + rows = rpt.classify_jobs( + [ + {"name": "GPU / rocJitsu (gfx90a)", "conclusion": "success", "html_url": "u1"}, + {"name": "GPU / rocJitsu (gfx94x)", "conclusion": "failure", "html_url": "u2"}, + ] + ) + self.assertEqual( + [(r["kind"], r["name"], r["conclusion"]) for r in rows], + [("Integration", "gfx90a", "success"), ("Integration", "gfx94x", "failure")], + ) + families = [{"family": "gfx90a"}, {"family": "gfx94x"}] + self.assertEqual(rpt.missing_families(families, rows), []) + + class OverallTest(unittest.TestCase): def test_failure_or_cancel_fails(self): self.assertFalse(rpt.overall_ok(rpt.classify_jobs(JOBS), [])) From d97e49a242f34c9c4955fe31a89f663bf43601a3 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 21 Oct 2026 12:07:17 +0200 Subject: [PATCH 47/89] test(hrr): accept the active marker name without a leading slash hip_capture_writer.cpp now names the marker "active"; the privacy-doc test still required kActiveMarker = "/name". Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tools/tests/test_privacy_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/hrr/tools/tests/test_privacy_doc.py b/projects/hrr/tools/tests/test_privacy_doc.py index 8d49326c05f6..16dd36e4d654 100644 --- a/projects/hrr/tools/tests/test_privacy_doc.py +++ b/projects/hrr/tools/tests/test_privacy_doc.py @@ -64,7 +64,7 @@ def test_device_table_matches_the_manifest_capture_writes(self): def test_inventory_lists_the_active_marker_capture_writes(self): writer = (CAPTURE_SRC / "hip_capture_writer.cpp").read_text(encoding="utf-8") - name = re.search(r'kActiveMarker = "/(\w+)"', writer).group(1) + name = re.search(r'kActiveMarker = "/?(\w+)"', writer).group(1) self.assertTrue( any(row.split("|")[2].strip() == f"`{name}`" for row in inventory_rows()) ) From fdb733bcb483ee341dad376ef37972b24251b15e Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 21 Oct 2026 14:00:29 +0200 Subject: [PATCH 48/89] ci(hrr): build rocJitsu once and share it with the simulator jobs The seven rocJitsu (family) jobs built identical binaries. A new Build rocJitsu job builds once and uploads a tarball of tools/rocjitsu and librocjitsu.so; the simulator jobs unpack it and drop the build packages and cmake. The CI summary counts the new job as a Build job. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-gpu.yml | 103 ++++++++++++++++-- projects/hrr/tests/scripts/hrr_ci_report.py | 2 + .../hrr/tests/scripts/test_hrr_ci_report.py | 15 +++ 3 files changed, 110 insertions(+), 10 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 88c66440fc54..bfb6fe7af420 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -107,16 +107,16 @@ jobs: if-no-files-found: error retention-days: 7 - simulator: - name: rocJitsu (${{ matrix.family }}) - needs: build + # rocJitsu has no family or architecture argument: every simulator job would + # build the same binary, so build it once here. The simulator jobs download + # it from the artifact. The launcher finds librocjitsu.so at ../../ from its + # own path, so the tarball keeps the build/rocjitsu layout, and tar keeps the + # exec bit that upload-artifact would drop. + rocjitsu-build: + name: Build rocJitsu if: inputs.simulator_families != '[]' runs-on: ubuntu-24.04 - timeout-minutes: 180 - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(inputs.simulator_families) }} + timeout-minutes: 60 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -124,7 +124,6 @@ jobs: persist-credentials: false sparse-checkout: | emulation/rocjitsu - projects/hrr/tests runtimes/api-headers shared/machine-readable-isa @@ -168,7 +167,91 @@ jobs: set -euo pipefail cmake -S emulation/rocjitsu -B build/rocjitsu -G Ninja -DCMAKE_BUILD_TYPE=Release cmake --build build/rocjitsu --target rocjitsu_bin rocjitsu_shared -j - echo "${GITHUB_WORKSPACE}/build/rocjitsu/tools/rocjitsu" >> "${GITHUB_PATH}" + + - name: Package rocJitsu + shell: bash + run: | + set -euo pipefail + tar -C build/rocjitsu -cf rocjitsu.tar tools/rocjitsu librocjitsu.so* + tar -tvf rocjitsu.tar + + - name: Upload rocJitsu + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rocjitsu-${{ github.run_id }} + path: rocjitsu.tar + if-no-files-found: error + retention-days: 7 + + simulator: + name: rocJitsu (${{ matrix.family }}) + needs: [build, rocjitsu-build] + if: inputs.simulator_families != '[]' + runs-on: ubuntu-24.04 + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.simulator_families) }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + emulation/rocjitsu/configs + projects/hrr/tests + + - name: Checkout TheRock artifact tools + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ROCm/TheRock + ref: ${{ env.THEROCK_PIN }} # 2026-10-12 + path: therock + sparse-checkout: | + build_tools + cmake/therock_amdgpu_targets.cmake + requirements.txt + sparse-checkout-cone-mode: false + + # The prebuilt rocJitsu and the capture-enabled amdhip64 load libdrm at run + # time; the build packages are no longer needed. + - name: Install libdrm + shell: bash + run: | + set -euo pipefail + sudo -n apt-get update + sudo -n DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libdrm2 libdrm-amdgpu1 + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install Python requirements + shell: bash + run: | + set -euo pipefail + python -m venv "${RUNNER_TEMP}/hrr-venv" + "${RUNNER_TEMP}/hrr-venv/bin/python" -m pip install --no-cache-dir \ + -r therock/requirements.txt 'PyYAML==6.0.2' + echo "${RUNNER_TEMP}/hrr-venv/bin" >> "${GITHUB_PATH}" + + - name: Download rocJitsu + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: rocjitsu-${{ github.run_id }} + path: rocjitsu-artifact + + - name: Unpack rocJitsu + shell: bash + run: | + set -euo pipefail + mkdir -p build/rocjitsu + tar -C build/rocjitsu -xf rocjitsu-artifact/rocjitsu.tar + ls -l build/rocjitsu/tools build/rocjitsu/librocjitsu.so* + echo "${GITHUB_WORKSPACE}/build/rocjitsu/tools" >> "${GITHUB_PATH}" - name: Download build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index 895374912740..1ffb8a79fde1 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -36,6 +36,8 @@ ("Script tests", re.compile(r"^HRR script tests$")), ("Unit", re.compile(r"(?:^|/ )Unit \((?P[^)]+)\)$")), ("Build", re.compile(r"(?:^|/ )Build capture-enabled HIP and HRR$")), + # The simulator jobs download this one build of rocJitsu; its failure fails them too. + ("Build", re.compile(r"(?:^|/ )Build rocJitsu$")), # The real-GPU job is "Integration ()", the rocJitsu one "rocJitsu ()". ("Integration", re.compile(r"(?:^|/ )(?:Integration|rocJitsu) \((?P[^)]+)\)$")), ) diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index a4d49632adf8..41676a7fca69 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -70,6 +70,21 @@ def test_rocjitsu_job_counts_as_the_family_integration_job(self): self.assertEqual(rpt.missing_families(families, rows), []) + def test_a_failed_rocjitsu_build_fails_the_summary_and_is_no_missing_family(self): + rows = rpt.classify_jobs( + [ + {"name": "GPU / Build rocJitsu", "conclusion": "failure", "html_url": "u0"}, + {"name": "GPU / rocJitsu (gfx90a)", "conclusion": "skipped", "html_url": "u1"}, + ] + ) + self.assertEqual( + [(r["kind"], r["name"], r["conclusion"]) for r in rows], + [("Build", "", "failure"), ("Integration", "gfx90a", "skipped")], + ) + self.assertEqual(rpt.missing_families([{"family": "gfx90a"}], rows), []) + self.assertFalse(rpt.overall_ok(rows, [])) + + class OverallTest(unittest.TestCase): def test_failure_or_cancel_fails(self): self.assertFalse(rpt.overall_ok(rpt.classify_jobs(JOBS), [])) From 6ea2f9229635b9936028d579495cce4e5d7abaca Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 21 Oct 2026 14:25:28 +0200 Subject: [PATCH 49/89] ci(hrr): expand the librocjitsu glob inside build/rocjitsu when packaging The glob expanded in the workspace root, matched nothing and failed tar. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-gpu.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index bfb6fe7af420..a9971d553add 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -172,7 +172,10 @@ jobs: shell: bash run: | set -euo pipefail - tar -C build/rocjitsu -cf rocjitsu.tar tools/rocjitsu librocjitsu.so* + # The glob must expand inside build/rocjitsu, so cd there. tar keeps + # symlinks, so a versioned librocjitsu.so chain survives. + ls -l build/rocjitsu/tools/rocjitsu build/rocjitsu/librocjitsu.so* + (cd build/rocjitsu && tar -cf "${GITHUB_WORKSPACE}/rocjitsu.tar" tools/rocjitsu librocjitsu.so*) tar -tvf rocjitsu.tar - name: Upload rocJitsu From 861ae53f24c89d0f65461ff834ac57abde59e95a Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 21 Oct 2026 14:47:25 +0200 Subject: [PATCH 50/89] ci(hrr): package the rocjitsu executable and put its directory on PATH build/rocjitsu/tools/rocjitsu is a directory; the launcher is inside it. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-gpu.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index a9971d553add..6ef141fb7890 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -109,8 +109,8 @@ jobs: # rocJitsu has no family or architecture argument: every simulator job would # build the same binary, so build it once here. The simulator jobs download - # it from the artifact. The launcher finds librocjitsu.so at ../../ from its - # own path, so the tarball keeps the build/rocjitsu layout, and tar keeps the + # it from the artifact. The launcher is tools/rocjitsu/rocjitsu and finds + # librocjitsu.so at ../../ from its own path, so the tarball keeps the build/rocjitsu layout, and tar keeps the # exec bit that upload-artifact would drop. rocjitsu-build: name: Build rocJitsu @@ -174,8 +174,8 @@ jobs: set -euo pipefail # The glob must expand inside build/rocjitsu, so cd there. tar keeps # symlinks, so a versioned librocjitsu.so chain survives. - ls -l build/rocjitsu/tools/rocjitsu build/rocjitsu/librocjitsu.so* - (cd build/rocjitsu && tar -cf "${GITHUB_WORKSPACE}/rocjitsu.tar" tools/rocjitsu librocjitsu.so*) + ls -l build/rocjitsu/tools/rocjitsu/rocjitsu build/rocjitsu/librocjitsu.so* + (cd build/rocjitsu && tar -cf "${GITHUB_WORKSPACE}/rocjitsu.tar" tools/rocjitsu/rocjitsu librocjitsu.so*) tar -tvf rocjitsu.tar - name: Upload rocJitsu @@ -253,8 +253,8 @@ jobs: set -euo pipefail mkdir -p build/rocjitsu tar -C build/rocjitsu -xf rocjitsu-artifact/rocjitsu.tar - ls -l build/rocjitsu/tools build/rocjitsu/librocjitsu.so* - echo "${GITHUB_WORKSPACE}/build/rocjitsu/tools" >> "${GITHUB_PATH}" + ls -l build/rocjitsu/tools/rocjitsu build/rocjitsu/librocjitsu.so* + echo "${GITHUB_WORKSPACE}/build/rocjitsu/tools/rocjitsu" >> "${GITHUB_PATH}" - name: Download build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 From d371abed6db3b97699bdfe96e715f3ecd289d026 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 22 Oct 2026 11:30:56 +0200 Subject: [PATCH 51/89] HRR CI: let fork pull requests run the rocJitsu simulator jobs The fork check returned before the backend was resolved, so a fork PR got no GPU-side jobs at all. Only the real-GPU backend needs runners a fork cannot use; the simulator jobs need neither GPU runners nor secrets (hrr-gpu.yml has permissions: contents: read). Apply the fork refusal to the gpu backend only. Cost: the capture-enabled build job (about 90 minutes) now also runs for fork pull requests that touch capture paths. Whether a fork's read-only token can fetch the base artifacts is not verified. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/scripts/hrr_ci_configure.py | 8 ++++++-- .../hrr/tests/scripts/test_hrr_ci_configure.py | 18 +++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index 083a95a50da3..4e00a3d30a22 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -525,12 +525,16 @@ def decide( run_cpu = run_cpu or capture if not capture: return Decision(run_cpu, False, why) - if is_fork: + # A fork pull request gets no secrets and cannot use the real GPU runners, + # but the simulator jobs need neither (hrr-gpu.yml has read-only + # permissions), so only the gpu backend is refused. + resolved = resolve_backend(event, config, backend) + if is_fork and resolved == "gpu": return Decision( run_cpu, False, why + "; GPU suite skipped for a fork pull request" ) - if resolve_backend(event, config, backend) == "simulator": + if resolved == "simulator": families, skipped = select_simulator(config, requested, labels, matrix) else: families, skipped = select_families( diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 9e980790c1db..f9e4efbac50d 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -413,7 +413,19 @@ def test_label_for_family_not_in_config_is_ignored(self): ) self.assertNotIn("gfx103x", family_keys(d)) - def test_fork_skips_gpu(self): + def test_fork_still_runs_the_simulator(self): + d = decide( + "pull_request", + ["projects/hrr/playback/x.cpp"], + is_fork=True, + config=cfg.parse_config(config_data()), + ) + self.assertTrue(d.run_cpu) + self.assertTrue(d.run_gpu) + self.assertTrue(d.families) + self.assertTrue(all(row["backend"] == "simulator" for row in d.families)) + + def test_fork_skips_the_real_gpu_backend(self): d = decide("pull_request", ["projects/hrr/playback/x.cpp"], is_fork=True) self.assertTrue(d.run_cpu) self.assertFalse(d.run_gpu) @@ -675,7 +687,7 @@ def test_deleted_head_repository_is_a_fork(self): def test_only_pull_requests_are_forks(self): self.assertFalse(cfg.is_fork("push", "", self.REPO)) - def test_main_skips_gpu_for_a_deleted_head_repository(self): + def test_main_skips_real_gpu_for_a_deleted_head_repository(self): with tempfile.TemporaryDirectory() as tmp, mock.patch.object( cfg, "list_pr_files", return_value=["projects/hrr/playback/x.cpp"] ), mock.patch.object( @@ -684,7 +696,7 @@ def test_main_skips_gpu_for_a_deleted_head_repository(self): os.environ, {"GITHUB_OUTPUT": ""} ): path = Path(tmp) / "ci.yaml" - path.write_text(yaml.safe_dump(config_data())) + path.write_text(yaml.safe_dump(gpu_data())) out = io.StringIO() with redirect_stdout(out): status = cfg.main( From 10279970794049b7abf641bce751482ff040a821 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 22 Oct 2026 11:31:12 +0200 Subject: [PATCH 52/89] HRR CI: run the CPU jobs when amdhip.def.in changes gen_hrr_api_args.py reads projects/clr/hipamd/src/amdhip.def.in, so a change to it can stale the generated sources. Add it to paths.cpu. It stays out of paths.capture, like hip_runtime_api.h: the generated-source job checks it, the capture build does not read it. Also note in ci.yaml that emulation/rocjitsu, runtimes/api-headers and shared/machine-readable-isa are not path triggers; the nightly schedule and workflow_dispatch cover them. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/ci.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 997cd7f21a1f..c3a6f9fb8398 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -11,16 +11,21 @@ rocm_sdk_version: "10.1.0a20260822" # everything under it; any other path matches only itself. paths: # A change under any of these runs the generated-source, script and unit jobs. + # Changes under emulation/rocjitsu, runtimes/api-headers and + # shared/machine-readable-isa are not path triggers; the nightly schedule and + # workflow_dispatch cover them. cpu: - projects/hrr/ - projects/clr/hipamd/src/hrr/ - projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp + - projects/clr/hipamd/src/amdhip.def.in - projects/hip/include/hip/hip_runtime_api.h - .github/workflows/hrr-ci.yml - .github/workflows/hrr-gpu.yml # A change under any of these also runs the capture-enabled build and the GPU # suite. hip_runtime_api.h is deliberately absent: almost every HIP API change # touches it, and the generated-source job already checks HRR against it. + # amdhip.def.in is absent for the same reason (cpu only). capture: - projects/clr/hipamd/src/hrr/ - projects/clr/hipamd/src/hip_context.cpp From 43ea1d169f83f5804a6225faf610ba492dbd97c3 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 22 Oct 2026 11:31:18 +0200 Subject: [PATCH 53/89] HRR CI report: drop job rows with an unexpanded matrix name The skipped real-GPU matrix job keeps the name 'GPU / Integration (${{ matrix.family }})' and showed up as a fake family in HRR CI Summary. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/scripts/hrr_ci_report.py | 3 +++ projects/hrr/tests/scripts/test_hrr_ci_report.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index 1ffb8a79fde1..810a9ae4b198 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -48,6 +48,9 @@ def classify_jobs(jobs: list[dict]) -> list[dict]: """HRR jobs from the run's job list, as {kind, name, conclusion, url}.""" rows = [] for job in jobs: + # A skipped matrix job keeps its unexpanded name; it is no family. + if "${{" in job.get("name", ""): + continue for kind, pattern in JOB_PATTERNS: match = pattern.search(job.get("name", "")) if match: diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index 41676a7fca69..82b0a8fa8d54 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -49,6 +49,15 @@ def test_missing_family(self): rows = rpt.classify_jobs(JOBS) self.assertEqual(rpt.missing_families(FAMILIES, rows), ["gfx110x"]) + def test_skipped_matrix_job_with_unexpanded_name_is_dropped(self): + rows = rpt.classify_jobs( + [ + {"name": "GPU / Integration (${{ matrix.family }})", "conclusion": "skipped"}, + {"name": "GPU / Integration (gfx94x)", "conclusion": "success"}, + ] + ) + self.assertEqual([(r["kind"], r["name"]) for r in rows], [("Integration", "gfx94x")]) + def test_queued_job_reports_status(self): rows = rpt.classify_jobs([{"name": "gpu / Integration (gfx90a)", "status": "queued"}]) self.assertEqual(rows[0]["conclusion"], "queued") From 3df0078e8cbddd9a00b275c2740a44a860055598 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 22 Oct 2026 11:31:28 +0200 Subject: [PATCH 54/89] HRR CI: drop the unsupported if-no-files-found input on download-artifact actions/download-artifact@v8.0.1 has no such input (the runner warned 'Unexpected input(s)'); it was ignored, so behaviour is unchanged. With a pattern that matches nothing the action does not fail the step. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index b89992ef1e4b..558b99a3f2b4 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -369,7 +369,6 @@ jobs: pattern: results-* merge-multiple: true path: results - if-no-files-found: ignore - name: Download base build record if: ${{ needs.configure.outputs.run_gpu == 'true' }} From 52478cbe4c65479f66c9b0a53038fdf4dabf3384 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 22 Oct 2026 12:01:35 +0200 Subject: [PATCH 55/89] HRR CI: list the two unmapped-note handler cases develop's #13894 added Unit_HRR_UnmappedNote_MemMap and Unit_HRR_UnmappedNote_IpcCloseOfSkippedImport to hrr-handler-tests. CI tests the merge ref, so handler.yaml must carry them or the case-list check fails. Both are host-only (no HIP call), so an empty entry. Names taken from source; the binary was not built or listed locally. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Daniel Olsson --- projects/hrr/tests/config/handler.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml index 2a1c0a2d754c..cf1253ff7aca 100644 --- a/projects/hrr/tests/config/handler.yaml +++ b/projects/hrr/tests/config/handler.yaml @@ -50,3 +50,5 @@ cases: Unit_HRR_Refusals_LinkCreateRefusesPointerCarryingOptions: {} Unit_HRR_Refusals_Memcpy2DNoOpCopiesAreNotRefused: {} Unit_HRR_Refusals_Memcpy2DWithUnmappedEndIsRefused: {} + Unit_HRR_UnmappedNote_IpcCloseOfSkippedImport: {} + Unit_HRR_UnmappedNote_MemMap: {} From 41fc59adb696162cc38ff9059cbb865a09f99495 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 22 Oct 2026 16:13:20 +0200 Subject: [PATCH 56/89] HRR CI: drop the skip-ci-config.toml entries Use the ci:skip label on the PR instead. Co-Authored-By: Claude Sonnet 5.5 --- .github/skip-ci-config.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/skip-ci-config.toml b/.github/skip-ci-config.toml index ba3e478561ef..5bcf616dd4e7 100644 --- a/.github/skip-ci-config.toml +++ b/.github/skip-ci-config.toml @@ -45,12 +45,6 @@ common = [ # Changes to experimental code do not run standard build/test workflows. "experimental/*", - - # HRR suite selection and its developer entry point. A source change under - # projects/hrr still builds. - ".github/workflows/hrr-ci.yml", - ".github/workflows/hrr-gpu.yml", - "projects/hrr/tests/config/ci.yaml", ] # Patterns that skip CI only on Linux (e.g., Windows-only projects) From e4fb61d3abe0a063a5d842d5e7bb936e391d5897 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 22 Oct 2026 18:07:53 +0200 Subject: [PATCH 57/89] HRR CI: fail the summary when a job or a case fails The summary read job conclusions from the GitHub API, which could still say queued after a rocJITsu job had failed, and it ignored JUnit failures. Take the verdict from the workflow needs and the result files. ci:gfx labels belong to TheRock and no longer select an HRR family. The simulator job runs in TheRock's test image, and the TheRock pin matches the multi-arch workflow. Signed-off-by: Daniel Olsson Co-authored-by: Cursor --- .github/workflows/hrr-ci.yml | 14 ++- .github/workflows/hrr-gpu.yml | 41 +++---- projects/hrr/tests/README.md | 21 +++- projects/hrr/tests/config/ci.yaml | 11 +- projects/hrr/tests/config/integration.yaml | 7 +- .../hrr/tests/scripts/hrr_capture_build.py | 8 +- .../hrr/tests/scripts/hrr_ci_configure.py | 37 ++---- projects/hrr/tests/scripts/hrr_ci_report.py | 75 +++++++++++- projects/hrr/tests/scripts/summarize_junit.py | 18 ++- .../tests/scripts/test_hrr_ci_configure.py | 106 +++++++++++------ .../hrr/tests/scripts/test_hrr_ci_report.py | 111 ++++++++++++++++++ .../hrr/tests/scripts/test_summarize_junit.py | 12 ++ 12 files changed, 345 insertions(+), 116 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 558b99a3f2b4..488c4a95c649 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -6,8 +6,8 @@ name: HRR CI -# labeled stays in the types so that adding ci:hrr-capture or ci:gfx* can -# still start a run; hrr_ci_configure.py decides from the label itself +# labeled stays in the types so that adding ci:hrr-capture can still start a +# run; hrr_ci_configure.py decides from the label itself # whether anything runs, and the summary job reports the verdict of the run # that actually tested the commit (see `pass_through_ok`). A label that # changes nothing therefore cannot mask a failing run on the same commit. @@ -46,7 +46,6 @@ concurrency: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.sha) || github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'ci:hrr-capture' - && !startsWith(github.event.label.name, 'ci:gfx') && format('-label-{0}', github.event.label.name) || '' }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -59,6 +58,7 @@ jobs: configure: name: Configure runs-on: ubuntu-24.04 + timeout-minutes: 10 permissions: contents: read pull-requests: read @@ -88,7 +88,7 @@ jobs: persist-credentials: false repository: ROCm/TheRock # Keep in sync with the TheRock ref in therock-multi-arch-ci.yml. - ref: c01e3a3f3f67966dd35612cf363b49964f33cd47 # 2026-10-12 + ref: 488e66e7c2220df926c2762e9b97a50becfb4239 # 2026-10-21 path: therock sparse-checkout: build_tools/github_actions @@ -144,6 +144,7 @@ jobs: needs: configure if: ${{ needs.configure.outputs.run_cpu == 'true' }} runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -175,6 +176,7 @@ jobs: needs: configure if: ${{ needs.configure.outputs.run_cpu == 'true' }} runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -214,6 +216,7 @@ jobs: needs: configure if: ${{ needs.configure.outputs.run_cpu == 'true' }} runs-on: ${{ matrix.os }} + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -350,6 +353,7 @@ jobs: || needs.configure.outputs.run_gpu == 'true' || (github.event_name == 'pull_request' && github.event.action == 'labeled')) }} runs-on: ubuntu-24.04 + timeout-minutes: 10 permissions: actions: read contents: read @@ -396,10 +400,12 @@ jobs: && needs.configure.outputs.run_cpu == 'false' && needs.configure.outputs.run_gpu == 'false' }} HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + NEEDS_JSON: ${{ toJSON(needs) }} run: >- python3 projects/hrr/tests/scripts/hrr_ci_report.py --configure-result "${CONFIGURE_RESULT}" --reason "${REASON}" --run-gpu "${RUN_GPU}" --families "${FAMILIES}" --skipped-families "${SKIPPED}" --results-dir results --base-json base/base.json + --needs-json "${NEEDS_JSON}" --pass-through "${PASS_THROUGH}" --head-sha "${HEAD_SHA}" diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 6ef141fb7890..bb09b4f171be 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -4,8 +4,10 @@ # Called by hrr-ci.yml, which decides when this runs and on which families. # # Interim: the build job and projects/hrr/tests/scripts/hrr_capture_build.py -# exist only until TheRock provides an hrr-capture build variant. After that, -# this file calls TheRock's multi-arch workflows with that variant instead. +# rebuild CLR with capture enabled on top of TheRock release artifacts. They +# stay until TheRock can build and run the HRR integration suite. Defining the +# hrr-capture variant is not enough: nothing selects it, and it does not +# install the test binaries. name: HRR GPU @@ -29,10 +31,6 @@ on: type: boolean default: false -env: - THEROCK_PIN: c01e3a3f3f67966dd35612cf363b49964f33cd47 - BUILD_IMAGE: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:cf4f6d9909056906e4a1c0bc4c245658a0af4c6a6174082a86adfa2d518b7aeb - permissions: contents: read @@ -59,7 +57,7 @@ jobs: persist-credentials: false repository: ROCm/TheRock # Keep in sync with the TheRock ref in therock-multi-arch-ci.yml. - ref: c01e3a3f3f67966dd35612cf363b49964f33cd47 # 2026-10-12 + ref: 488e66e7c2220df926c2762e9b97a50becfb4239 # 2026-10-21 path: therock sparse-checkout: | build_tools @@ -132,7 +130,7 @@ jobs: with: persist-credentials: false repository: ROCm/TheRock - ref: ${{ env.THEROCK_PIN }} # 2026-10-12 + ref: 488e66e7c2220df926c2762e9b97a50becfb4239 # 2026-10-21 path: therock sparse-checkout: | build_tools @@ -191,11 +189,17 @@ jobs: needs: [build, rocjitsu-build] if: inputs.simulator_families != '[]' runs-on: ubuntu-24.04 - timeout-minutes: 180 + timeout-minutes: 45 strategy: fail-fast: false matrix: include: ${{ fromJSON(inputs.simulator_families) }} + # The same image as the real-GPU job, without devices. The build job's + # workspace path is this container's workspace, so the paths compiled into + # the test binaries resolve without a symlink on the runner. + container: + image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98 + options: --user 0:0 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -210,7 +214,7 @@ jobs: with: persist-credentials: false repository: ROCm/TheRock - ref: ${{ env.THEROCK_PIN }} # 2026-10-12 + ref: 488e66e7c2220df926c2762e9b97a50becfb4239 # 2026-10-21 path: therock sparse-checkout: | build_tools @@ -219,13 +223,12 @@ jobs: sparse-checkout-cone-mode: false # The prebuilt rocJitsu and the capture-enabled amdhip64 load libdrm at run - # time; the build packages are no longer needed. + # time. The container runs as root, so this does not use sudo. - name: Install libdrm - shell: bash run: | set -euo pipefail - sudo -n apt-get update - sudo -n DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libdrm2 libdrm-amdgpu1 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -267,14 +270,6 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: python projects/hrr/tests/scripts/hrr_capture_build.py install-base - # The build job compiled the test binary in a container, where the - # workspace is /__w//, and the binary spawns itself and the - # playback tool by those absolute paths. Make them resolve on this host. - - name: Map the container build path to the workspace - run: | - sudo mkdir -p /__w/rocm-systems - sudo ln -sfn "${GITHUB_WORKSPACE}" /__w/rocm-systems/rocm-systems - - name: Run integration suite env: FAMILY: ${{ matrix.family }} @@ -347,7 +342,7 @@ jobs: with: persist-credentials: false repository: ROCm/TheRock - ref: c01e3a3f3f67966dd35612cf363b49964f33cd47 # 2026-10-12 + ref: 488e66e7c2220df926c2762e9b97a50becfb4239 # 2026-10-21 path: therock sparse-checkout: | build_tools diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index d8e324fbbf70..897a0a2874c9 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -155,8 +155,8 @@ applies (`scripts/hrr_ci_configure.py`): jobs. `paths.capture` also runs the capture-enabled build and the GPU suite. `paths.excluded` counts for neither. - `capture_label` (`ci:hrr-capture`) runs the GPU suite on a pull request that touches - no capture path. Adding it or a `ci:gfx...` label restarts the run; adding any - other label leaves the run in progress alone. + no capture path. Adding it restarts the run; adding any other label, including + TheRock's `ci:gfx...` labels, leaves the run in progress alone. - `families`: the GPU families the suite runs on, as TheRock family keys, on the runner TheRock's family matrix names. By default each runs on every event (`pull_request`, `push`, `schedule`, `workflow_dispatch`). Per family, `enabled: false` leaves it to @@ -164,10 +164,11 @@ applies (`scripts/hrr_ci_configure.py`): and `multi_gpu_triggers` names the events on which it takes TheRock's multi-GPU runner instead of the single-GPU one. TheRock's multi-GPU pools are 8-GPU nodes, so only gfx950 uses one, nightly; elsewhere the cases that need two devices skip and - show in the summary's Skip column. A `ci:gfx...` label adds a - family; the `workflow_dispatch` input `families` replaces the selection. Both can - name a disabled family. Only gfx94x is enabled today; the others are listed with - `enabled: false`, so they run only when a label or the input asks for them. + show in the summary's Skip column. The `workflow_dispatch` input `families` + replaces the selection and can name a disabled family. Only gfx94x is enabled + today; the others are listed with `enabled: false`, so a real-GPU run includes + them only when that input names them. `ci:gfx...` labels are TheRock's and do + not select an HRR family. `targets.arch` in `config/integration.yaml` stays wide on purpose: configure fails unless it covers every family listed here, and a requested family needs its architecture compiled into the binary. @@ -178,6 +179,14 @@ GPU targets in TheRock's family matrix are not all in `targets.arch` of `config/integration.yaml`. To take load off the GPU pools, disable families or narrow their triggers here. +Until the TheRock commit pinned by `.github/workflows/therock-multi-arch-ci.yml` +includes [TheRock #8965](https://github.com/ROCm/TheRock/pull/8965), TheRock +treats `hrr-ci.yml`, `hrr-gpu.yml`, and `projects/hrr` as unclassified and +builds every stage. A pull request that changes only those workflow files or +`config/ci.yaml` should carry the `ci:skip` label. Remove that label when the +change should build HRR sources, and drop this note once the pin includes +#8965. + ## Configuring where a case runs The build is one fat binary for every architecture in `targets.arch` of diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index c3a6f9fb8398..5af95fcf060b 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -77,7 +77,7 @@ backend: # two-GPU gfx950 config and the single-GPU configs of the others. # # Not listed: gfx103x, gfx1150 and gfx1153, which rocJitsu has no config for. -# A ci:gfx... label for one of them fails the configure job. +# A workflow_dispatch families input that names one fails the configure job. simulator: runner: ubuntu-24.04 default_families: [gfx90a, gfx94x, gfx950, gfx110x, gfx1151, gfx120x, gfx125x] @@ -96,8 +96,9 @@ simulator: # that covers every GPU target TheRock lists for each family here. # # Per family, all optional: -# enabled false: run only when requested by a ci:gfx... label or the -# workflow_dispatch families input. Default true. +# enabled false: run only when the workflow_dispatch families input names +# it. Default true. ci:gfx labels belong to TheRock and are not +# read here. # triggers Events that run this family: pull_request, push, schedule, # workflow_dispatch. Default all four. # runner Runner label to use instead of TheRock's. @@ -143,7 +144,7 @@ families: enabled: false # Off until understood: in two runs the MI455 runner picked the job up, but # --arch auto found no GPU, so no case ran. TheRock also reserves this small - # pool for designated workflows. A ci:gfx125x label or the workflow_dispatch - # families input still runs it. + # pool for designated workflows. The workflow_dispatch families input still + # runs it. gfx125x: enabled: false diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 2c7b0640ed5f..5c2ad47097e6 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -272,7 +272,12 @@ cases: Unit_HRR_OverflowingMemcpy3DNotRecorded: {} Unit_HRR_OverflowingMemcpy3D_Direct: {} Unit_HRR_PinnedHostBlobRefused: {} - Unit_HRR_PinnedHostFork: {} + Unit_HRR_PinnedHostFork: + skip: + - when: {arch: ["gfx95*"], emulator: true} + kind: disabled + reason: "The capture subprocess exited 42 under the two-GPU gfx950 rocJITsu configuration; the same case passed on the other rocJITsu families" + issue: "https://github.com/ROCm/rocm-systems/issues/14033" Unit_HRR_PinnedHostFork_Direct: {} Unit_HRR_PinnedHostFreedRoundtrip: {} Unit_HRR_PinnedHostFreed_Direct: {} diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index 087b99e4fb64..c064a633417d 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -3,9 +3,11 @@ """Build an amdhip64 with HRR capture compiled in, for the HRR GPU suite. -Interim: TheRock has no capture-enabled build yet. Once rocm-systems CI can -request TheRock's hrr-capture build variant, hrr-gpu.yml calls TheRock's -workflows instead and this script and test_hrr_capture_build.py are deleted. +Interim: this rebuilds CLR with capture enabled on top of TheRock release +artifacts. It stays until TheRock can build and run the HRR integration +suite. The hrr-capture variant alone does not: no workflow selects it, and +it does not install the test binaries. This script and +test_hrr_capture_build.py go away once that suite runs from TheRock artifacts. Release builds compile capture out (HIP_HRR_CAPTURE_ENABLED undefined). This script installs the ROCm artifacts that rocm-systems' own TheRock CI built for diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index 4e00a3d30a22..d7107db69475 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -22,9 +22,10 @@ with the capture label, nightly and on workflow_dispatch. The default backend is rocJitsu (simulator). Its jobs are -simulator.default_families, not the families marked enabled. A ci:gfx... label -adds a family, and the workflow_dispatch families input replaces the -selection. A simulator family with no config is an error. The GPU backend +simulator.default_families, not the families marked enabled. The +workflow_dispatch families input replaces the selection. A simulator family +with no config is an error. ci:gfx labels are TheRock's, so this script does +not read them. The GPU backend runs every enabled family whose triggers include the event, on TheRock's runner unless ci.yaml overrides it, and takes the multi-GPU runner only on its multi_gpu_triggers. workflow_dispatch uses its backend input; any other @@ -319,26 +320,12 @@ def touches_suite(paths: Iterable[str], config: Config) -> bool: return any(_matches(p, config.suite_paths) for p in relevant_paths(paths, config)) -def labelled_families(labels: Iterable[str], matrix: dict) -> list[str]: - """Family keys named by ci:gfx... labels, e.g. ci:gfx110x or ci:gfx110X-all.""" - wanted = [] - for label in labels: - label = label.strip().lower() - if not label.startswith("ci:gfx"): - continue - name = label[len("ci:") :] - for key, config in matrix.items(): - linux = config.get("linux", {}) - if name in (key.lower(), str(linux.get("family", "")).lower()): - wanted.append(key) - return wanted - - def label_matters(label: str, config: Config) -> bool: """Whether adding this label can change what runs. hrr-ci.yml's concurrency - group makes the same test, so that other labels do not cancel a run.""" + group makes the same test, so that other labels do not cancel a run. + ci:gfx labels belong to TheRock and must not restart this workflow.""" label = label.strip().lower() - return label == config.capture_label.lower() or label.startswith("ci:gfx") + return label == config.capture_label.lower() def runner_for( @@ -386,7 +373,6 @@ def select_families( event: str, config: Config, requested: list[str], - labels: list[str], matrix: dict, pick_weighted: Callable[[list, str], str], ) -> tuple[list[dict], list[dict]]: @@ -406,7 +392,6 @@ def select_families( for key, family in config.families.items() if family.enabled and event in family.triggers } - chosen |= set(labelled_families(labels, matrix)) & set(config.families) families, skipped = [], [] for key, family in config.families.items(): @@ -432,11 +417,10 @@ def select_families( def select_simulator( config: Config, requested: list[str], - labels: list[str], matrix: dict, ) -> tuple[list[dict], list[dict]]: """rocJitsu families. default_families, unless the dispatch input replaces - them. A ci:gfx... label adds a family. A family with no config is an error.""" + them. A family with no config is an error.""" if requested: unknown = [key for key in requested if key not in config.families] if unknown: @@ -447,7 +431,6 @@ def select_simulator( chosen = set(requested) else: chosen = set(config.simulator.default_families) - chosen |= set(labelled_families(labels, matrix)) & set(config.families) missing = sorted(key for key in chosen if key not in config.simulator.configs) if missing: raise ConfigError(f"simulator has no config for {', '.join(missing)}") @@ -535,10 +518,10 @@ def decide( ) if resolved == "simulator": - families, skipped = select_simulator(config, requested, labels, matrix) + families, skipped = select_simulator(config, requested, matrix) else: families, skipped = select_families( - event, config, requested, labels, matrix, pick_weighted + event, config, requested, matrix, pick_weighted ) no_runner = [s["family"] for s in skipped if s["reason"] == NO_RUNNER] if no_runner: diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index 810a9ae4b198..70ca7866c98e 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -11,8 +11,12 @@ * the per-suite results table from summarize_junit.py, with a row for every ci.yaml family, saying why when one did not run. -Exits 1 when the configure job did not succeed, any HRR job failed or was -cancelled, or a results file could not be read. +Exits 1 when the configure job did not succeed, a needed job failed, was +cancelled, or was skipped after configure selected it, any listed HRR job is +not in a terminal success state, a JUnit suite has failed cases, or a results +file could not be read. Job conclusions from the Actions API are links in the +table. The verdict uses --needs-json (the workflow's toJSON(needs)) and the +JUnit files, because the jobs API can still say queued after those exist. With --pass-through true (a label run that ran no HRR job) it reports the newest other run on --head-sha that has a verdict instead, and exits 1 unless @@ -41,7 +45,10 @@ # The real-GPU job is "Integration ()", the rocJitsu one "rocJitsu ()". ("Integration", re.compile(r"(?:^|/ )(?:Integration|rocJitsu) \((?P[^)]+)\)$")), ) -BAD_CONCLUSIONS = {"failure", "cancelled", "timed_out", "startup_failure"} +# Anything else (queued, in_progress, waiting, unknown) has not finished. +TERMINAL_OK = {"success", "neutral"} +BAD_NEEDS = {"failure", "cancelled"} +CPU_NEEDS = ("codegen", "scripts", "unit") def classify_jobs(jobs: list[dict]) -> list[dict]: @@ -110,7 +117,39 @@ def render_jobs(rows: list[dict], missing: list[str]) -> str: def overall_ok(rows: list[dict], missing: list[str]) -> bool: - return not missing and not any(r["conclusion"] in BAD_CONCLUSIONS for r in rows) + """True when every requested family ran and every listed job finished + successfully. A skipped, failed, or still-queued job is not a pass: the + jobs API has reported queued while the suite had already failed.""" + if missing: + return False + return all(r["conclusion"] in TERMINAL_OK for r in rows) + + +def need_result(needs: dict, name: str) -> str: + info = needs.get(name) + if isinstance(info, dict) and info.get("result"): + return str(info["result"]) + return "missing" + + +def check_needs(needs: dict, *, run_gpu: bool) -> list[str]: + """Problems in GitHub's toJSON(needs). A failure or cancellation fails the + run. On a run whose configure succeeded, the CPU jobs were selected, and + the GPU workflow was selected when run_gpu is true; a skip there is a job + that was supposed to run.""" + problems = [] + for name in needs: + result = need_result(needs, name) + if result in BAD_NEEDS: + problems.append(f"{name} is {result}") + selected = list(CPU_NEEDS) + if run_gpu: + selected.append("gpu") + for name in selected: + result = need_result(needs, name) + if result != "success" and f"{name} is {result}" not in problems: + problems.append(f"{name} is {result}") + return problems SUMMARY_JOB = "HRR CI Summary" @@ -233,6 +272,8 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "")) parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID", "")) parser.add_argument("--run-attempt", default=os.environ.get("GITHUB_RUN_ATTEMPT", "1")) + parser.add_argument("--needs-json", default="", + help="GitHub toJSON(needs). Empty skips the needs check") parser.add_argument("--pass-through", choices=("true", "false"), default="false", help="true: report the earlier run that tested --head-sha instead") # Not GITHUB_SHA: on a pull request that is the merge commit, which no @@ -265,10 +306,13 @@ def main(argv: list[str] | None = None) -> int: markdown = render_header(args.reason, run_gpu, families, base, args.configure_result) markdown += render_jobs(rows, missing) errors: list[str] = [] + failed_suites: list[str] = [] args.results_dir.mkdir(parents=True, exist_ok=True) table_families = [(f["family"], "") for f in families] table_families += [(s["family"], s["reason"]) for s in skipped] - markdown += summarize_junit.render(args.results_dir, errors, table_families) + markdown += summarize_junit.render( + args.results_dir, errors, table_families, failures=failed_suites + ) sys.stdout.write(markdown) summary = os.environ.get("GITHUB_STEP_SUMMARY") @@ -277,7 +321,26 @@ def main(argv: list[str] | None = None) -> int: out.write(markdown) for error in errors: print(f"::error::unreadable JUnit XML {error}") - return 0 if configured and overall_ok(rows, missing) and not errors else 1 + for suite in failed_suites: + print(f"::error::failed cases in {suite}") + + needs_problems: list[str] = [] + if args.needs_json and configured: + try: + needs = json.loads(args.needs_json) + except json.JSONDecodeError as error: + print(f"::error::could not read --needs-json: {error}") + return 1 + if not isinstance(needs, dict): + print("::error::--needs-json must be an object") + return 1 + needs_problems = check_needs(needs, run_gpu=run_gpu) + for problem in needs_problems: + print(f"::error::{problem}") + return 0 if ( + configured and not needs_problems and overall_ok(rows, missing) + and not errors and not failed_suites + ) else 1 if __name__ == "__main__": diff --git a/projects/hrr/tests/scripts/summarize_junit.py b/projects/hrr/tests/scripts/summarize_junit.py index 4fe32e391c2f..58ab8f081d10 100644 --- a/projects/hrr/tests/scripts/summarize_junit.py +++ b/projects/hrr/tests/scripts/summarize_junit.py @@ -106,10 +106,15 @@ def failed_cell(names: list[str]) -> str: def render( - directory: Path, errors: list[str], families: list[tuple[str, str]] | None = None + directory: Path, + errors: list[str], + families: list[tuple[str, str]] | None = None, + *, + failures: list[str] | None = None, ) -> str: """families: (family, why it did not run, or "") per integration row. By - default, one row per integration-.xml present.""" + default, one row per integration-.xml present. Suites with failed + cases are appended to failures as "suite platform".""" if families is None: families = [ (path.stem[len("integration-"):], "") @@ -158,6 +163,8 @@ def render( ) continue counts = data["counts"] + if failures is not None and counts["FAIL"]: + failures.append(f"{suite} {platform}") lines.append( "| {suite} | {platform} | {passed} | {failed} | {skipped} | {config} | {names} |".format( suite=suite, @@ -206,14 +213,17 @@ def main(argv: list[str] | None = None) -> int: print(f"not a directory: {args.dir}", file=sys.stderr) return 1 errors: list[str] = [] - markdown = render(args.dir, errors) + failures: list[str] = [] + markdown = render(args.dir, errors, failures=failures) sys.stdout.write(markdown) if args.summary is not None: with args.summary.open("a", encoding="utf-8") as out: out.write(markdown) for error in errors: print(f"::error::unreadable JUnit XML {error}") - return 1 if errors else 0 + for suite in failures: + print(f"::error::failed cases in {suite}") + return 1 if errors or failures else 0 if __name__ == "__main__": diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index f9e4efbac50d..f11c8ea6cfff 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -138,9 +138,9 @@ class RepoConfigTest(unittest.TestCase): def test_only_gfx94x_runs_by_default(self): """Every family stays listed, but only gfx94x runs unless asked for. - The others stay in ci.yaml so a ci:gfx... label or the - workflow_dispatch families input can still request them; disabling is - what keeps them off the default run. + The others stay in ci.yaml so the workflow_dispatch families input can + still request them; disabling is what keeps them off the default run. + ci:gfx labels are TheRock's and do not select an HRR family. """ self.assertEqual( sorted(key for key, family in REPO.families.items() if family.enabled), @@ -149,8 +149,8 @@ def test_only_gfx94x_runs_by_default(self): for key, family in REPO.families.items(): self.assertEqual(family.triggers, cfg.EVENTS, key) - def test_a_label_adds_a_simulator_family(self): - """A family outside simulator.default_families joins on its label.""" + def test_a_gfx_label_does_not_add_a_simulator_family(self): + """ci:gfx is a TheRock label. It must not widen the rocJITsu matrix.""" data = copy.deepcopy(REPO_CONFIG) data["simulator"]["default_families"] = ["gfx950", "gfx125x"] d = decide( @@ -159,7 +159,7 @@ def test_a_label_adds_a_simulator_family(self): ["ci:gfx110x"], config=cfg.parse_config(data), ) - self.assertEqual(family_keys(d), ["gfx950", "gfx110x", "gfx125x"]) + self.assertEqual(family_keys(d), ["gfx950", "gfx125x"]) self.assertTrue(all(row["backend"] == "simulator" for row in d.families)) def test_multi_gpu_runner_only_nightly(self): @@ -397,15 +397,13 @@ def test_capture_only_path_also_runs_cpu(self): self.assertTrue(d.run_cpu) self.assertTrue(d.run_gpu) - def test_family_label_adds_family(self): + def test_gfx_label_does_not_add_a_family(self): d = decide( "pull_request", ["projects/hrr/playback/x.cpp"], labels=["ci:gfx110X-all", "ci:gfx125x"], ) - self.assertEqual( - family_keys(d), ["gfx90a", "gfx94x", "gfx950", "gfx110x", "gfx125x"] - ) + self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950"]) def test_label_for_family_not_in_config_is_ignored(self): d = decide( @@ -566,13 +564,10 @@ def test_pull_request_gpu_triggers_select_real_runners(self): self.assertTrue(all(row["backend"] == "gpu" for row in decision.families)) self.assertNotIn("gfx125x", family_keys(decision)) - def test_simulator_rejects_a_family_without_a_config(self): + def test_simulator_rejects_a_requested_family_without_a_config(self): with self.assertRaises(cfg.ConfigError) as caught: decide( - "pull_request", - ["projects/hrr/playback/x.cpp"], - labels=["ci:gfx103x"], - config=REPO, + "workflow_dispatch", [], requested=["gfx103x"], config=REPO ) self.assertIn("no config for gfx103x", str(caught.exception)) @@ -591,14 +586,9 @@ class LabelEventTest(unittest.TestCase): ) def test_which_labels_matter(self): - for label in ( - "ci:hrr-capture", - "CI:HRR-Capture", - "ci:gfx110x", - "ci:gfx110X-all", - ): + for label in ("ci:hrr-capture", "CI:HRR-Capture"): self.assertTrue(cfg.label_matters(label, REPO), label) - for label in ("documentation", "ci:skip", "hrr"): + for label in ("documentation", "ci:skip", "hrr", "ci:gfx110x", "ci:gfx110X-all"): self.assertFalse(cfg.label_matters(label, REPO), label) def test_other_label_runs_nothing(self): @@ -617,20 +607,32 @@ def test_other_label_runs_nothing(self): self.assertFalse(d.run_gpu) self.assertIn("documentation", d.reason) - def test_capture_or_family_label_reruns(self): - for label in ("ci:hrr-capture", "ci:gfx110x"): - d = cfg.decide( - "pull_request", - self.CAPTURE, - [label], - False, - [], - CONFIG, - MATRIX, - pick_first, - added_label=label, - ) - self.assertTrue(d.run_gpu, label) + def test_capture_label_reruns_and_a_gfx_label_does_not(self): + d = cfg.decide( + "pull_request", + self.CAPTURE, + ["ci:hrr-capture"], + False, + [], + CONFIG, + MATRIX, + pick_first, + added_label="ci:hrr-capture", + ) + self.assertTrue(d.run_gpu) + d = cfg.decide( + "pull_request", + self.CAPTURE, + ["ci:gfx110x"], + False, + [], + CONFIG, + MATRIX, + pick_first, + added_label="ci:gfx110x", + ) + self.assertFalse(d.run_cpu) + self.assertFalse(d.run_gpu) def test_labeled_trigger_and_summary_report_a_verdict(self): """Regression for the label that can turn a failing HRR CI green. @@ -669,7 +671,7 @@ def test_concurrency_group_makes_the_same_test(self): text = self.WORKFLOW.read_text(encoding="utf-8") group = text[text.index("concurrency:") : text.index("cancel-in-progress:")] self.assertIn(f"github.event.label.name != '{REPO.capture_label}'", group) - self.assertIn("!startsWith(github.event.label.name, 'ci:gfx')", group) + self.assertNotIn("ci:gfx", group) class ForkTest(unittest.TestCase): @@ -745,5 +747,35 @@ def test_write_outputs(self): self.assertIn("rocm_sdk_version=1.2.3\n", text) +class WorkflowPinTest(unittest.TestCase): + ROOT = Path(__file__).resolve().parents[4] + SHA = re.compile(r"(?:@|ref:\s*)([0-9a-f]{40})") + + def therock_shas(self, path: Path) -> list[str]: + lines = path.read_text(encoding="utf-8").splitlines() + found = [] + for index, line in enumerate(lines): + if "ROCm/TheRock" not in line: + continue + window = "\n".join(lines[index:index + 6]) + found.extend(self.SHA.findall(window)) + return found + + def test_hrr_workflows_use_the_multi_arch_therock_pin(self): + pinned = self.therock_shas( + self.ROOT / ".github/workflows/therock-multi-arch-ci.yml" + ) + self.assertTrue(pinned) + self.assertEqual(set(pinned), {pinned[0]}) + for name in ("hrr-ci.yml", "hrr-gpu.yml"): + path = self.ROOT / ".github/workflows" / name + text = path.read_text(encoding="utf-8") + self.assertNotIn("THEROCK_PIN", text, name) + self.assertNotIn("BUILD_IMAGE", text, name) + found = self.therock_shas(path) + self.assertTrue(found, name) + self.assertEqual(set(found), {pinned[0]}, name) + + if __name__ == "__main__": unittest.main() diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index 82b0a8fa8d54..a3c1711d5df5 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -5,6 +5,7 @@ import contextlib import io +import json import os import sys import subprocess @@ -61,6 +62,7 @@ def test_skipped_matrix_job_with_unexpanded_name_is_dropped(self): def test_queued_job_reports_status(self): rows = rpt.classify_jobs([{"name": "gpu / Integration (gfx90a)", "status": "queued"}]) self.assertEqual(rows[0]["conclusion"], "queued") + self.assertFalse(rpt.overall_ok(rows, [])) class RocJitsuJobTest(unittest.TestCase): @@ -169,6 +171,115 @@ def test_main_fails_when_configure_did_not_succeed(self): self.assertEqual(rc, 1, result) self.assertIn(f"Configure: {result}, so nothing else ran", out.getvalue()) + def test_run_39793856571_does_not_pass(self): + """The jobs API was still queued, needs.gpu had failed, and the gfx950 + JUnit file already contained Unit_HRR_PinnedHostFork. All three must + fail the summary; any one of them is enough.""" + jobs = [ + {"name": "Generated sources", "conclusion": "success", "html_url": "u0"}, + {"name": "HRR script tests", "conclusion": "success"}, + {"name": "Unit (ubuntu-24.04)", "conclusion": "success"}, + {"name": "Unit (windows-2022)", "conclusion": "success"}, + {"name": "GPU / Build capture-enabled HIP and HRR", "conclusion": "success"}, + {"name": "GPU / Build rocJitsu", "status": "in_progress", "html_url": "u1"}, + {"name": "GPU / rocJitsu (gfx950)", "status": "queued", "html_url": "u2"}, + ] + needs = { + "configure": {"result": "success"}, + "codegen": {"result": "success"}, + "scripts": {"result": "success"}, + "unit": {"result": "success"}, + "gpu": {"result": "failure"}, + } + failed = ( + '' + '' + "" + ) + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(rpt, "list_jobs", return_value=jobs), \ + mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ + contextlib.redirect_stdout(io.StringIO()) as out: + (Path(tmp) / "integration-gfx950.xml").write_text(failed, encoding="utf-8") + rc = rpt.main([ + "--run-gpu", "true", "--results-dir", tmp, + "--families", '[{"family":"gfx950","runner":"ubuntu-24.04"}]', + "--needs-json", json.dumps(needs), + ]) + self.assertEqual(rc, 1) + text = out.getvalue() + self.assertIn("gpu is failure", text) + self.assertIn("failed cases in Integration gfx950", text) + self.assertIn("| Integration | gfx950 | [queued](u2) |", text) + + def test_junit_failure_fails_when_jobs_and_needs_succeeded(self): + jobs = [ + {"name": "Generated sources", "conclusion": "success"}, + {"name": "HRR script tests", "conclusion": "success"}, + {"name": "Unit (ubuntu-24.04)", "conclusion": "success"}, + {"name": "GPU / rocJitsu (gfx94x)", "conclusion": "success"}, + ] + needs = {name: {"result": "success"} for name in ( + "configure", "codegen", "scripts", "unit", "gpu")} + failed = ( + '' + '' + "" + ) + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(rpt, "list_jobs", return_value=jobs), \ + mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ + contextlib.redirect_stdout(io.StringIO()): + (Path(tmp) / "integration-gfx94x.xml").write_text(failed, encoding="utf-8") + rc = rpt.main([ + "--run-gpu", "true", "--results-dir", tmp, + "--families", '[{"family":"gfx94x"}]', + "--needs-json", json.dumps(needs), + ]) + self.assertEqual(rc, 1) + + def test_a_selected_cpu_job_that_was_skipped_fails(self): + needs = { + "configure": {"result": "success"}, + "codegen": {"result": "skipped"}, + "scripts": {"result": "success"}, + "unit": {"result": "success"}, + "gpu": {"result": "skipped"}, + } + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(rpt, "list_jobs", return_value=JOBS[:3]), \ + mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ + contextlib.redirect_stdout(io.StringIO()) as out: + rc = rpt.main(["--results-dir", tmp, "--needs-json", json.dumps(needs)]) + self.assertEqual(rc, 1) + self.assertIn("codegen is skipped", out.getvalue()) + + def test_green_needs_jobs_and_junit_pass(self): + jobs = [ + {"name": "Generated sources", "conclusion": "success"}, + {"name": "HRR script tests", "conclusion": "success"}, + {"name": "Unit (ubuntu-24.04)", "conclusion": "success"}, + {"name": "Unit (windows-2022)", "conclusion": "success"}, + {"name": "GPU / rocJitsu (gfx94x)", "conclusion": "success"}, + ] + needs = {name: {"result": "success"} for name in ( + "configure", "codegen", "scripts", "unit", "gpu")} + passed = ( + '' + '' + ) + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch.object(rpt, "list_jobs", return_value=jobs), \ + mock.patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": ""}), \ + contextlib.redirect_stdout(io.StringIO()): + (Path(tmp) / "integration-gfx94x.xml").write_text(passed, encoding="utf-8") + rc = rpt.main([ + "--run-gpu", "true", "--results-dir", tmp, + "--families", '[{"family":"gfx94x"}]', + "--needs-json", json.dumps(needs), + ]) + self.assertEqual(rc, 0) + class GhApiTest(unittest.TestCase): def test_reads_every_value_across_pages(self): diff --git a/projects/hrr/tests/scripts/test_summarize_junit.py b/projects/hrr/tests/scripts/test_summarize_junit.py index e2a2d49a8089..2df153c39158 100644 --- a/projects/hrr/tests/scripts/test_summarize_junit.py +++ b/projects/hrr/tests/scripts/test_summarize_junit.py @@ -179,6 +179,18 @@ def test_writes_stdout_and_appends_to_the_summary_file(self): self.assertTrue(written.startswith("existing\n")) self.assertIn("## HRR results by platform", written) + def test_failed_cases_are_reported_and_exit_1(self): + self.write( + "integration-gfx950.xml", + junit(record("Unit_HRR_PinnedHostFork", "failure", "42 == 0")), + ) + failures: list[str] = [] + summarize_junit.render(self.dir, [], [("gfx950", "")], failures=failures) + self.assertEqual(failures, ["Integration gfx950"]) + rc, out, _ = self.run_main("--dir", str(self.dir)) + self.assertEqual(rc, 1) + self.assertIn("::error::failed cases in Integration gfx950", out) + def test_unreadable_xml_exits_1_with_an_annotation(self): self.write("integration-gfx950.xml", "") rc, out, _ = self.run_main("--dir", str(self.dir)) From 4500d089afdca34371ebe6212bf35c95905cf2de Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 22 Oct 2026 18:45:33 +0200 Subject: [PATCH 58/89] HRR CI: run the simulator libdrm install under bash The no_rocm image defaults to dash, which rejects set -o pipefail, so every rocJITsu job exited before apt-get. Signed-off-by: Daniel Olsson Co-authored-by: Cursor --- .github/workflows/hrr-gpu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index bb09b4f171be..6aa2e6e6fcc1 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -225,6 +225,7 @@ jobs: # The prebuilt rocJitsu and the capture-enabled amdhip64 load libdrm at run # time. The container runs as root, so this does not use sudo. - name: Install libdrm + shell: bash run: | set -euo pipefail apt-get update From b21f84b33adb8d17bf59fd746791211079f742ce Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 23 Oct 2026 08:32:53 +0200 Subject: [PATCH 59/89] HRR CI: let TheRock CI skip HRR-only CI and test changes TheRock never reads the HRR workflows or projects/hrr/tests, tools and skills: CLR compiles against projects/hrr/include, and TheRock #8965 adds hrr-playback from CMakeLists.txt, include/ and playback/ with HRR_BUILD_TESTS=OFF. List those paths in skip-ci-config.toml, which TheRock's configure_multi_arch_ci.py already reads at the pinned ref, so a pull request that changes only them skips the TheRock build without the ci:skip label. Signed-off-by: Daniel Olsson --- .github/skip-ci-config.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/skip-ci-config.toml b/.github/skip-ci-config.toml index 5bcf616dd4e7..48ea3c3d0bcd 100644 --- a/.github/skip-ci-config.toml +++ b/.github/skip-ci-config.toml @@ -45,6 +45,15 @@ common = [ # Changes to experimental code do not run standard build/test workflows. "experimental/*", + + # HRR's own CI and the HRR paths TheRock never reads. CLR compiles against + # projects/hrr/include, and TheRock #8965 adds hrr-playback (CMakeLists.txt, + # include/ and playback/, with HRR_BUILD_TESTS=OFF). Remove + # projects/hrr/tests/* once TheRock builds the HRR test suites. + ".github/workflows/hrr-*.yml", + "projects/hrr/tests/*", + "projects/hrr/tools/*", + "projects/hrr/skills/*", ] # Patterns that skip CI only on Linux (e.g., Windows-only projects) From ce8482540216ef67f7302e02914c85b28fd78432 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 23 Oct 2026 08:34:31 +0200 Subject: [PATCH 60/89] HRR CI: start runs only for pull requests that touch HRR paths Without a paths filter every pull request event in the repository, each label included, started HRR CI, and Configure then ran nothing. Filter pull_request and push on the union of the ci.yaml cpu, capture and suite lists, less paths.excluded. A test fails when the filter and ci.yaml disagree. black also reformats three existing lines of that test file. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 31 +++++++++ projects/hrr/tests/config/ci.yaml | 3 +- .../hrr/tests/scripts/hrr_ci_configure.py | 5 +- .../tests/scripts/test_hrr_ci_configure.py | 63 +++++++++++++++++-- 4 files changed, 94 insertions(+), 8 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 488c4a95c649..f16616c6df83 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -11,11 +11,42 @@ name: HRR CI # whether anything runs, and the summary job reports the verdict of the run # that actually tested the commit (see `pass_through_ok`). A label that # changes nothing therefore cannot mask a failing run on the same commit. +# +# The paths filters are the union of the paths.cpu, paths.capture and +# paths.suite lists of projects/hrr/tests/config/ci.yaml, less paths.excluded, +# so a pull request that touches no HRR path starts no run. +# test_hrr_ci_configure.py fails when the two disagree. on: pull_request: types: [opened, synchronize, reopened, labeled] + paths: + - ".github/workflows/hrr-ci.yml" + - ".github/workflows/hrr-gpu.yml" + - "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp" + - "projects/clr/hipamd/src/amdhip.def.in" + - "projects/clr/hipamd/src/hip_code_object.cpp" + - "projects/clr/hipamd/src/hip_code_object.hpp" + - "projects/clr/hipamd/src/hip_context.cpp" + - "projects/clr/hipamd/src/hrr/**" + - "projects/hip/include/hip/hip_runtime_api.h" + - "projects/hrr/**" + - "!projects/hrr/README.md" + - "!projects/hrr/skills/**" push: branches: [develop] + paths: + - ".github/workflows/hrr-ci.yml" + - ".github/workflows/hrr-gpu.yml" + - "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp" + - "projects/clr/hipamd/src/amdhip.def.in" + - "projects/clr/hipamd/src/hip_code_object.cpp" + - "projects/clr/hipamd/src/hip_code_object.hpp" + - "projects/clr/hipamd/src/hip_context.cpp" + - "projects/clr/hipamd/src/hrr/**" + - "projects/hip/include/hip/hip_runtime_api.h" + - "projects/hrr/**" + - "!projects/hrr/README.md" + - "!projects/hrr/skills/**" schedule: # Nightly: capture-enabled build and the GPU suite on every HRR family. - cron: "0 3 * * *" diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 5af95fcf060b..419c37cf655c 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -57,7 +57,8 @@ paths: - projects/hrr/README.md # A pull request with this label runs the GPU suite even when it touches no -# capture path. +# capture path. It must still touch a cpu, capture or suite path: hrr-ci.yml's +# paths filter, which is those lists, starts no run otherwise. capture_label: ci:hrr-capture # simulator runs rocJitsu on ubuntu-24.04. gpu runs TheRock's real runners. diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index d7107db69475..38b2a8de6435 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -18,8 +18,9 @@ The policy lives in projects/hrr/tests/config/ci.yaml. The suite needs an amdhip64 with HRR capture compiled in, which only the HRR GPU workflow builds. -It runs when a change touches a capture or suite path, on any pull request -with the capture label, nightly and on workflow_dispatch. +It runs when a change touches a capture or suite path, on a pull request with +the capture label that touches any HRR path (hrr-ci.yml's paths filter starts +no run otherwise), nightly and on workflow_dispatch. The default backend is rocJitsu (simulator). Its jobs are simulator.default_families, not the families marked enabled. The diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index f11c8ea6cfff..139af11fb2b2 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -566,9 +566,7 @@ def test_pull_request_gpu_triggers_select_real_runners(self): def test_simulator_rejects_a_requested_family_without_a_config(self): with self.assertRaises(cfg.ConfigError) as caught: - decide( - "workflow_dispatch", [], requested=["gfx103x"], config=REPO - ) + decide("workflow_dispatch", [], requested=["gfx103x"], config=REPO) self.assertIn("no config for gfx103x", str(caught.exception)) def test_unknown_backend_value(self): @@ -588,7 +586,13 @@ class LabelEventTest(unittest.TestCase): def test_which_labels_matter(self): for label in ("ci:hrr-capture", "CI:HRR-Capture"): self.assertTrue(cfg.label_matters(label, REPO), label) - for label in ("documentation", "ci:skip", "hrr", "ci:gfx110x", "ci:gfx110X-all"): + for label in ( + "documentation", + "ci:skip", + "hrr", + "ci:gfx110x", + "ci:gfx110X-all", + ): self.assertFalse(cfg.label_matters(label, REPO), label) def test_other_label_runs_nothing(self): @@ -757,7 +761,7 @@ def therock_shas(self, path: Path) -> list[str]: for index, line in enumerate(lines): if "ROCm/TheRock" not in line: continue - window = "\n".join(lines[index:index + 6]) + window = "\n".join(lines[index : index + 6]) found.extend(self.SHA.findall(window)) return found @@ -777,5 +781,54 @@ def test_hrr_workflows_use_the_multi_arch_therock_pin(self): self.assertEqual(set(found), {pinned[0]}, name) +class WorkflowPathsFilterTest(unittest.TestCase): + ROOT = Path(__file__).resolve().parents[4] + + @staticmethod + def glob(path: str) -> str: + return path + "**" if path.endswith("/") else path + + def expected(self, config: cfg.Config) -> tuple[set[str], set[str]]: + """ci.yaml's path lists as a GitHub paths filter: entries under a listed + directory fold into that directory's glob.""" + union = set(config.cpu_paths + config.capture_paths + config.suite_paths) + dirs = [p for p in union if p.endswith("/")] + kept = {p for p in union if not any(p != d and p.startswith(d) for d in dirs)} + return ( + {self.glob(p) for p in kept}, + {"!" + self.glob(p) for p in config.excluded_paths}, + ) + + def test_hrr_ci_paths_filter_matches_ci_yaml(self): + config = cfg.load_config(self.ROOT / "projects/hrr/tests/config/ci.yaml") + # ci.yaml matches every entry as a prefix; the filter matches an entry + # without a trailing "/" only as that exact file. + for path in ( + config.cpu_paths + + config.capture_paths + + config.suite_paths + + config.excluded_paths + ): + if not path.endswith("/"): + self.assertTrue((self.ROOT / path).is_file(), path) + include, exclude = self.expected(config) + workflow = yaml.safe_load( + (self.ROOT / ".github/workflows/hrr-ci.yml").read_text(encoding="utf-8") + ) + # PyYAML reads the bare key `on` as True. + triggers = workflow.get("on", workflow.get(True)) + for event in ("pull_request", "push"): + with self.subTest(event=event): + paths = triggers[event]["paths"] + self.assertEqual(len(paths), len(set(paths))) + self.assertEqual({p for p in paths if not p.startswith("!")}, include) + self.assertEqual({p for p in paths if p.startswith("!")}, exclude) + # A negated pattern only drops what an earlier pattern matched. + first_negated = next( + i for i, p in enumerate(paths) if p.startswith("!") + ) + self.assertTrue(all(p.startswith("!") for p in paths[first_negated:])) + + if __name__ == "__main__": unittest.main() From c7969417dd1956d46444b5e4615065a9fcf44c17 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 23 Oct 2026 08:39:44 +0200 Subject: [PATCH 61/89] HRR CI: take TheRock's rocJitsu for the simulator jobs Building rocJitsu from source took about 23 of the roughly 31 minutes a simulator run spends on its critical path. The rocJitsu job now fetches rocjitsu_run and rocjitsu_lib from the newest develop TheRock run and uses that launcher, with the same build's KMD configs, when the run ships bin/rocjitsu. It still builds rocJitsu from this tree on schedule and workflow_dispatch, when a change touches simulator.source_paths, and when no run ships the launcher. Until rocm-systems pins a TheRock that packages bin/rocjitsu, that is every run. Both paths package bin/rocjitsu and lib/librocjitsu.so*; the launcher finds librocjitsu.so at ../lib from its own path. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 2 + .github/workflows/hrr-gpu.yml | 69 ++++++++++---- projects/hrr/tests/config/ci.yaml | 10 ++ .../hrr/tests/scripts/hrr_capture_build.py | 82 ++++++++++++++-- .../hrr/tests/scripts/hrr_ci_configure.py | 33 ++++++- .../tests/scripts/test_hrr_capture_build.py | 93 +++++++++++++++++++ .../tests/scripts/test_hrr_ci_configure.py | 92 ++++++++++++++++++ 7 files changed, 352 insertions(+), 29 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index f16616c6df83..d007326a84f7 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -100,6 +100,7 @@ jobs: simulator_families: ${{ steps.decide.outputs.simulator_families }} gpu_families: ${{ steps.decide.outputs.gpu_families }} skipped_families: ${{ steps.decide.outputs.skipped_families }} + build_rocjitsu: ${{ steps.decide.outputs.build_rocjitsu }} reason: ${{ steps.decide.outputs.reason }} rocm_sdk_version: ${{ steps.decide.outputs.rocm_sdk_version }} steps: @@ -372,6 +373,7 @@ jobs: families: ${{ needs.configure.outputs.families }} simulator_families: ${{ needs.configure.outputs.simulator_families }} gpu_families: ${{ needs.configure.outputs.gpu_families }} + build_rocjitsu: ${{ needs.configure.outputs.build_rocjitsu == 'true' }} ignore_test_config: ${{ inputs.ignore_test_config || false }} summary: diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 6aa2e6e6fcc1..0a774e4a8176 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -26,6 +26,10 @@ on: description: Real-GPU rows from that list. [] skips the GPU jobs. type: string default: "[]" + build_rocjitsu: + description: Build rocJitsu from this tree instead of taking TheRock's + type: boolean + default: true ignore_test_config: description: Run the cases the suite config skips type: boolean @@ -105,11 +109,12 @@ jobs: if-no-files-found: error retention-days: 7 - # rocJitsu has no family or architecture argument: every simulator job would - # build the same binary, so build it once here. The simulator jobs download - # it from the artifact. The launcher is tools/rocjitsu/rocjitsu and finds - # librocjitsu.so at ../../ from its own path, so the tarball keeps the build/rocjitsu layout, and tar keeps the - # exec bit that upload-artifact would drop. + # rocJitsu has no family or architecture argument, so every simulator job + # uses the same launcher, prepared once here: TheRock's from a develop run + # unless build_rocjitsu is set or no run ships it, else built from this tree. + # The tarball has bin/rocjitsu and lib/librocjitsu.so*, which the launcher + # finds at ../lib from its own path, plus share/rocjitsu/configs when it is + # TheRock's. tar keeps the exec bit that upload-artifact would drop. rocjitsu-build: name: Build rocJitsu if: inputs.simulator_families != '[]' @@ -122,6 +127,7 @@ jobs: persist-credentials: false sparse-checkout: | emulation/rocjitsu + projects/hrr/tests/scripts runtimes/api-headers shared/machine-readable-isa @@ -138,14 +144,6 @@ jobs: requirements.txt sparse-checkout-cone-mode: false - - name: Install rocJitsu build packages - shell: bash - run: | - set -euo pipefail - sudo -n apt-get update - sudo -n DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential ca-certificates clang libdrm-dev ninja-build - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" @@ -159,7 +157,24 @@ jobs: -r therock/requirements.txt 'cmake==3.31.10' 'PyYAML==6.0.2' echo "${RUNNER_TEMP}/hrr-venv/bin" >> "${GITHUB_PATH}" + - name: Fetch TheRock's rocJitsu + id: prebuilt + if: ${{ !inputs.build_rocjitsu }} + env: + GITHUB_TOKEN: ${{ github.token }} + run: python projects/hrr/tests/scripts/hrr_capture_build.py fetch-rocjitsu + + - name: Install rocJitsu build packages + if: ${{ steps.prebuilt.outputs.prebuilt != 'true' }} + shell: bash + run: | + set -euo pipefail + sudo -n apt-get update + sudo -n DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential ca-certificates clang libdrm-dev ninja-build + - name: Build rocJitsu + if: ${{ steps.prebuilt.outputs.prebuilt != 'true' }} shell: bash run: | set -euo pipefail @@ -168,12 +183,23 @@ jobs: - name: Package rocJitsu shell: bash + env: + PREBUILT: ${{ steps.prebuilt.outputs.prebuilt }} run: | set -euo pipefail - # The glob must expand inside build/rocjitsu, so cd there. tar keeps - # symlinks, so a versioned librocjitsu.so chain survives. - ls -l build/rocjitsu/tools/rocjitsu/rocjitsu build/rocjitsu/librocjitsu.so* - (cd build/rocjitsu && tar -cf "${GITHUB_WORKSPACE}/rocjitsu.tar" tools/rocjitsu/rocjitsu librocjitsu.so*) + mkdir -p pkg/bin pkg/lib + # cp -a keeps the symlinks of a versioned librocjitsu.so chain. + if [[ "${PREBUILT}" == "true" ]]; then + src=build/rocjitsu-prebuilt + cp -a "${src}/bin/rocjitsu" pkg/bin/ + cp -a "${src}"/lib/librocjitsu.so* pkg/lib/ + mkdir -p pkg/share/rocjitsu + cp -a "${src}/share/rocjitsu/configs" pkg/share/rocjitsu/ + else + cp -a build/rocjitsu/tools/rocjitsu/rocjitsu pkg/bin/ + cp -a build/rocjitsu/librocjitsu.so* pkg/lib/ + fi + tar -C pkg -cf rocjitsu.tar . tar -tvf rocjitsu.tar - name: Upload rocJitsu @@ -257,8 +283,13 @@ jobs: set -euo pipefail mkdir -p build/rocjitsu tar -C build/rocjitsu -xf rocjitsu-artifact/rocjitsu.tar - ls -l build/rocjitsu/tools/rocjitsu build/rocjitsu/librocjitsu.so* - echo "${GITHUB_WORKSPACE}/build/rocjitsu/tools/rocjitsu" >> "${GITHUB_PATH}" + ls -l build/rocjitsu/bin build/rocjitsu/lib + # TheRock's launcher runs with the configs of the same build. + configs=build/rocjitsu/share/rocjitsu/configs + if [[ -d "${configs}" ]]; then + cp -a "${configs}/." emulation/rocjitsu/configs/ + fi + echo "${GITHUB_WORKSPACE}/build/rocjitsu/bin" >> "${GITHUB_PATH}" - name: Download build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 419c37cf655c..0398c4e6dd51 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -79,8 +79,18 @@ backend: # # Not listed: gfx103x, gfx1150 and gfx1153, which rocJitsu has no config for. # A workflow_dispatch families input that names one fails the configure job. +# +# The simulator jobs take rocJitsu, with that build's configs, from the newest +# develop TheRock run that has the rocJitsu artifacts, if those include the +# launcher (hrr_capture_build.py fetch-rocjitsu). They build it from this tree +# instead on schedule and workflow_dispatch, when a change touches +# source_paths or cannot be listed, and when that run lacks the launcher. simulator: runner: ubuntu-24.04 + source_paths: + - emulation/rocjitsu/ + - shared/machine-readable-isa/ + - runtimes/api-headers/ default_families: [gfx90a, gfx94x, gfx950, gfx110x, gfx1151, gfx120x, gfx125x] configs: gfx94x: emulation/rocjitsu/configs/gfx942_cdna3_kmd.json diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index c064a633417d..4d94b72ca498 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -24,6 +24,8 @@ which amdhip64 and ROCr the suite loads. install-base On a GPU machine: install the run-time artifacts of the recorded base run and the capture-enabled amdhip64. + fetch-rocjitsu Install rocJitsu from the newest develop run that has the + rocJitsu artifacts, and report whether they hold the launcher. """ from __future__ import annotations @@ -78,6 +80,10 @@ "core-hip_lib", ) +# The rocJitsu launcher, its interposer and its KMD configs, for the simulator +# jobs. Only TheRock refs that package bin/rocjitsu ship the launcher. +ROCJITSU_ARTIFACTS = ("rocjitsu_run", "rocjitsu_lib") + # What CLR's install writes into lib/: amdhip64 plus hiprtc and its builtins. # All of it comes from this checkout, so all of it is carried to the test job. CAPTURE_LIB_GLOBS = ("libamdhip64.so*", "libhiprtc*.so*") @@ -134,7 +140,7 @@ def pick_base_run(finder_json: dict) -> dict: } -def find_base_run(therock: Path) -> dict: +def find_base_run(therock: Path, artifacts: Sequence[str] = BUILD_ARTIFACTS) -> dict: cmd = [ sys.executable, therock / "build_tools" / "find_latest_artifacts.py", @@ -148,7 +154,7 @@ def find_base_run(therock: Path) -> dict: BASE_ARTIFACT_GROUP, "--json", ] - for artifact in BUILD_ARTIFACTS: + for artifact in artifacts: cmd += ["--require-artifact", f"{artifact}_generic"] print("+ " + " ".join(str(c) for c in cmd), flush=True) # Only stdout is captured: it carries the JSON, stderr the finder's log. @@ -470,15 +476,55 @@ def cache_key_digest( return digest.hexdigest() -def cmd_cache_key(args: argparse.Namespace) -> int: - base = json.loads(args.base_json.read_text()) - digest = cache_key_digest(args.src, base, args.therock_pin, args.image, args.archs) - line = f"digest={digest}\n" +def write_output(line: str) -> None: output = os.environ.get("GITHUB_OUTPUT", "") if output: with open(output, "a", encoding="utf-8") as handle: handle.write(line) sys.stdout.write(line) + + +def cmd_cache_key(args: argparse.Namespace) -> int: + base = json.loads(args.base_json.read_text()) + digest = cache_key_digest(args.src, base, args.therock_pin, args.image, args.archs) + write_output(f"digest={digest}\n") + return 0 + + +def missing_rocjitsu(prefix: Path) -> list[str]: + """What hrr-gpu.yml packages from a prebuilt rocJitsu and prefix lacks.""" + missing = [] + if not (prefix / "bin" / "rocjitsu").is_file(): + missing.append("bin/rocjitsu") + if not any((prefix / "lib").glob("librocjitsu.so*")): + missing.append("lib/librocjitsu.so") + if not (prefix / "share" / "rocjitsu" / "configs").is_dir(): + missing.append("share/rocjitsu/configs") + return missing + + +def cmd_fetch_rocjitsu( + args: argparse.Namespace, + runner: Runner = run, + finder: Callable[[Path, Sequence[str]], dict] = find_base_run, +) -> int: + """Writes prebuilt=true when --rocjitsu-dir now holds the whole launcher + layout. Any other outcome writes prebuilt=false, and the caller builds + rocJitsu.""" + found = False + try: + base = finder(args.therock, ROCJITSU_ARTIFACTS) + print(f"rocJitsu: {base['repo']} run {base['run_id']} at {base['commit']}") + runner(fetch_command(args.therock, base, args.rocjitsu_dir, ROCJITSU_ARTIFACTS)) + missing = missing_rocjitsu(args.rocjitsu_dir) + if missing: + print( + f"{args.rocjitsu_dir} lacks {', '.join(missing)}; rocJitsu will be built" + ) + found = not missing + except (SystemExit, subprocess.CalledProcessError) as e: + print(f"::warning::no prebuilt rocJitsu, so it will be built: {e}") + write_output(f"prebuilt={'true' if found else 'false'}\n") return 0 @@ -506,7 +552,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "command", - choices=("fetch-base", "build", "stage", "verify", "install-base", "cache-key"), + choices=( + "fetch-base", + "build", + "stage", + "verify", + "install-base", + "cache-key", + "fetch-rocjitsu", + ), ) parser.add_argument("--therock", type=Path, default=Path("therock")) parser.add_argument("--src", type=Path, default=Path.cwd()) @@ -514,6 +568,9 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--build", type=Path, default=Path("build")) parser.add_argument("--capture-lib", type=Path, default=Path("build/capture-lib")) parser.add_argument("--base-json", type=Path, default=Path("build/base.json")) + parser.add_argument( + "--rocjitsu-dir", type=Path, default=Path("build/rocjitsu-prebuilt") + ) parser.add_argument( "--archs", default="", @@ -524,7 +581,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) parser.add_argument("--image", default="", help="Build container image digest") args = parser.parse_args(argv) - for name in ("therock", "src", "dist", "build", "capture_lib", "base_json"): + for name in ( + "therock", + "src", + "dist", + "build", + "capture_lib", + "base_json", + "rocjitsu_dir", + ): setattr(args, name, getattr(args, name).resolve()) if args.command in ("build", "verify", "cache-key") and not args.archs: config = hrr_test_config.load(args.src / INTEGRATION_CONFIG) @@ -541,6 +606,7 @@ def main(argv: list[str] | None = None) -> int: "verify": cmd_verify, "install-base": cmd_install_base, "cache-key": cmd_cache_key, + "fetch-rocjitsu": cmd_fetch_rocjitsu, }[args.command](args) diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index 38b2a8de6435..e27094815b8e 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -13,6 +13,8 @@ gpu_families The subset whose backend is gpu. skipped_families JSON list of {"family", "reason"}: the other ci.yaml families, when the suite runs. + build_rocjitsu "true" when the simulator jobs use rocJitsu built from this + tree rather than TheRock's (see builds_rocjitsu). reason One line explaining the decision, shown in the summary. rocm_sdk_version The ROCm SDK the unit job installs. @@ -83,6 +85,7 @@ class Simulator: runner: str default_families: tuple[str, ...] configs: dict[str, str] + source_paths: tuple[str, ...] = () @dataclass(frozen=True) @@ -105,6 +108,7 @@ class Decision: reason: str families: list[dict] = field(default_factory=list) skipped: list[dict] = field(default_factory=list) + build_rocjitsu: bool = False def _mapping( @@ -165,7 +169,9 @@ def _backend(value) -> Backend: def _simulator(value, families: dict) -> Simulator: - value = _mapping("simulator", value, ("runner", "default_families", "configs")) + value = _mapping( + "simulator", value, ("runner", "default_families", "configs"), ("source_paths",) + ) default_families = _strings("simulator.default_families", value["default_families"]) unknown = [key for key in default_families if key not in families] if unknown: @@ -189,6 +195,11 @@ def _simulator(value, families: dict) -> Simulator: runner=_string("simulator.runner", value["runner"]), default_families=default_families, configs=parsed, + source_paths=( + _strings("simulator.source_paths", value["source_paths"]) + if "source_paths" in value + else () + ), ) @@ -321,6 +332,15 @@ def touches_suite(paths: Iterable[str], config: Config) -> bool: return any(_matches(p, config.suite_paths) for p in relevant_paths(paths, config)) +def builds_rocjitsu(event: str, paths: list[str] | None, config: Config) -> bool: + """Whether the simulator jobs use rocJitsu built from this tree rather than + TheRock's. schedule and workflow_dispatch test this tree's rocJitsu, as + does a change set that touches simulator.source_paths or cannot be listed.""" + if event in ("schedule", "workflow_dispatch") or paths is None: + return True + return any(_matches(p, config.simulator.source_paths) for p in paths) + + def label_matters(label: str, config: Config) -> bool: """Whether adding this label can change what runs. hrr-ci.yml's concurrency group makes the same test, so that other labels do not cancel a run. @@ -531,7 +551,15 @@ def decide( return Decision( run_cpu, False, why + "; no GPU family with a runner was selected" ) - return Decision(run_cpu, True, why, families, skipped) + simulated = resolved == "simulator" + return Decision( + run_cpu, + True, + why, + families, + skipped, + build_rocjitsu=simulated and builds_rocjitsu(event, paths, config), + ) def is_fork(event: str, head_repo: str, repo: str) -> bool: @@ -601,6 +629,7 @@ def write_outputs( f"simulator_families={_dump(simulator)}", f"gpu_families={_dump(gpu)}", f"skipped_families={_dump(decision.skipped)}", + f"build_rocjitsu={'true' if decision.build_rocjitsu else 'false'}", f"reason={decision.reason}", f"rocm_sdk_version={rocm_sdk_version}", ] diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index 9a12a390fa1e..d5ada1b1af31 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -4,12 +4,16 @@ """Tests for hrr_capture_build.py.""" import argparse +import io import json import os +import subprocess import sys import tempfile import unittest +from contextlib import redirect_stdout from pathlib import Path +from unittest import mock sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -364,5 +368,94 @@ def test_reuses_recorded_base(self): self.assertIn("core-runtime_dev", cmd) +class FetchRocjitsuTest(unittest.TestCase): + # What fetch_artifacts.py --flatten writes for rocjitsu_run and rocjitsu_lib + # from a TheRock ref that does not package the launcher. + TODAY = ( + "bin/rj_pretranslate", + "lib/librocjitsu.so", + "share/rocjitsu/configs/a.json", + ) + LAUNCHER = "bin/rocjitsu" + + def fetch(self, finder, files=(), fail_after_writing=False): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "github_output" + rocjitsu_dir = Path(tmp) / "rocjitsu" + calls = [] + + def runner(cmd): + calls.append([str(c) for c in cmd]) + for rel in files: + (rocjitsu_dir / rel).parent.mkdir(parents=True, exist_ok=True) + (rocjitsu_dir / rel).write_text("") + if fail_after_writing: + raise subprocess.CalledProcessError(1, cmd) + + args = argparse.Namespace(therock=Path("/t"), rocjitsu_dir=rocjitsu_dir) + env = dict(os.environ, GITHUB_OUTPUT=str(out)) + with mock.patch.dict(os.environ, env), redirect_stdout(io.StringIO()): + self.assertEqual( + cb.cmd_fetch_rocjitsu(args, runner=runner, finder=finder), 0 + ) + return calls, out.read_text() + + @staticmethod + def found(*_): + return cb.pick_base_run(FINDER_FOUND) + + def test_launcher_in_the_artifacts(self): + requested = [] + + def finder(_therock, artifacts): + requested.append(artifacts) + return cb.pick_base_run(FINDER_FOUND) + + calls, output = self.fetch(finder, self.TODAY + (self.LAUNCHER,)) + self.assertEqual(output, "prebuilt=true\n") + self.assertEqual(requested, [cb.ROCJITSU_ARTIFACTS]) + self.assertEqual(calls[0][calls[0].index("--run-id") + 1], "123") + self.assertIn("rocjitsu_run", calls[0]) + + def test_todays_artifacts_lack_the_launcher(self): + _, output = self.fetch(self.found, self.TODAY) + self.assertEqual(output, "prebuilt=false\n") + + def test_launcher_without_its_interposer(self): + files = (self.LAUNCHER, "share/rocjitsu/configs/a.json") + _, output = self.fetch(self.found, files) + self.assertEqual(output, "prebuilt=false\n") + + def test_fetch_that_fails_after_writing_the_launcher(self): + _, output = self.fetch( + self.found, self.TODAY + (self.LAUNCHER,), fail_after_writing=True + ) + self.assertEqual(output, "prebuilt=false\n") + + def test_finder_finds_no_run(self): + # find_latest_artifacts.py exits 1 when no run matches. + def finder(*_): + raise subprocess.CalledProcessError(1, ["find_latest_artifacts.py"]) + + calls, output = self.fetch(finder, self.TODAY + (self.LAUNCHER,)) + self.assertEqual(calls, []) + self.assertEqual(output, "prebuilt=false\n") + + def test_finder_requires_the_rocjitsu_artifacts(self): + result = mock.Mock(stdout=json.dumps(FINDER_FOUND)) + with mock.patch.object(cb.subprocess, "run", return_value=result) as run: + with redirect_stdout(io.StringIO()): + cb.find_base_run(Path("/t"), cb.ROCJITSU_ARTIFACTS) + cmd = [str(c) for c in run.call_args.args[0]] + required = [cmd[i + 1] for i, c in enumerate(cmd) if c == "--require-artifact"] + self.assertEqual(required, ["rocjitsu_run_generic", "rocjitsu_lib_generic"]) + + def test_fetch_rocjitsu_command_and_its_directory(self): + # hrr-gpu.yml packages from build/rocjitsu-prebuilt. + args = cb.parse_args(["fetch-rocjitsu"]) + self.assertEqual(args.command, "fetch-rocjitsu") + self.assertEqual(args.rocjitsu_dir, Path("build/rocjitsu-prebuilt").resolve()) + + if __name__ == "__main__": unittest.main() diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 139af11fb2b2..e846cfc1b1ae 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -473,6 +473,98 @@ def test_unknown_event(self): self.assertFalse(d.run_gpu) +class RocjitsuSourceTest(unittest.TestCase): + HRR = "projects/hrr/playback/x.cpp" + + def test_pull_request_and_push_take_therocks_rocjitsu(self): + for event in ("pull_request", "push"): + decision = decide(event, [self.HRR], config=REPO) + self.assertTrue(decision.run_gpu, event) + self.assertFalse(decision.build_rocjitsu, event) + + def test_a_rocjitsu_change_builds_it(self): + for path in REPO.simulator.source_paths: + decision = decide("pull_request", [self.HRR, path + "x.cpp"], config=REPO) + self.assertTrue(decision.build_rocjitsu, path) + + def test_schedule_dispatch_and_unlisted_changes_build_it(self): + for event in ("schedule", "workflow_dispatch"): + self.assertTrue(decide(event, [], config=REPO).build_rocjitsu, event) + self.assertTrue(decide("pull_request", None, config=REPO).build_rocjitsu) + + def test_gpu_backend_needs_no_rocjitsu(self): + decision = decide("schedule", [], config=CONFIG) + self.assertTrue(decision.run_gpu) + self.assertFalse(decision.build_rocjitsu) + + def test_source_paths_are_optional(self): + data = config_data() + del data["simulator"]["source_paths"] + self.assertEqual(cfg.parse_config(data).simulator.source_paths, ()) + + def test_source_paths_must_be_a_list_of_paths(self): + for value in ("emulation/rocjitsu/", []): + data = config_data() + data["simulator"]["source_paths"] = value + with self.assertRaises(cfg.ConfigError, msg=repr(value)): + cfg.parse_config(data) + + def output(self, decision) -> str: + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "out" + with redirect_stdout(io.StringIO()): + cfg.write_outputs(decision, "1.2.3", str(out)) + return out.read_text() + + def test_build_rocjitsu_is_an_output(self): + self.assertIn( + "build_rocjitsu=true\n", self.output(decide("schedule", [], config=REPO)) + ) + self.assertIn( + "build_rocjitsu=false\n", + self.output(decide("pull_request", [self.HRR], config=REPO)), + ) + + +class WorkflowRocjitsuWiringTest(unittest.TestCase): + ROOT = Path(__file__).resolve().parents[4] + + def workflow(self, name: str) -> dict: + path = self.ROOT / ".github/workflows" / name + return yaml.safe_load(path.read_text(encoding="utf-8")) + + def test_build_rocjitsu_reaches_the_fetch_step(self): + ci = self.workflow("hrr-ci.yml")["jobs"] + self.assertEqual( + ci["configure"]["outputs"]["build_rocjitsu"], + "${{ steps.decide.outputs.build_rocjitsu }}", + ) + self.assertEqual( + ci["gpu"]["with"]["build_rocjitsu"], + "${{ needs.configure.outputs.build_rocjitsu == 'true' }}", + ) + gpu = self.workflow("hrr-gpu.yml") + triggers = gpu.get("on", gpu.get(True)) + self.assertEqual( + triggers["workflow_call"]["inputs"]["build_rocjitsu"]["type"], "boolean" + ) + steps = {s.get("name"): s for s in gpu["jobs"]["rocjitsu-build"]["steps"]} + fetch = steps["Fetch TheRock's rocJitsu"] + self.assertEqual(fetch["id"], "prebuilt") + self.assertEqual(fetch["if"], "${{ !inputs.build_rocjitsu }}") + self.assertIn("fetch-rocjitsu", fetch["run"]) + for name in ("Install rocJitsu build packages", "Build rocJitsu"): + self.assertEqual( + steps[name]["if"], "${{ steps.prebuilt.outputs.prebuilt != 'true' }}" + ) + + def test_rocjitsu_build_checks_out_its_source_paths(self): + steps = self.workflow("hrr-gpu.yml")["jobs"]["rocjitsu-build"]["steps"] + sparse = steps[0]["with"]["sparse-checkout"].split() + for path in REPO.simulator.source_paths: + self.assertIn(path.rstrip("/"), sparse) + + class SimulatorBackendTest(unittest.TestCase): def test_default_runs_no_real_gpu_job_on_any_event(self): """Pull requests, pushes and the nightly run only rocJitsu jobs, on every From 3948e668d97840847dfdc77f7c7b0f14ef676146 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 23 Oct 2026 08:40:32 +0200 Subject: [PATCH 62/89] HRR CI: clean up GPU processes around the real-GPU job The real-GPU job runs on TheRock's shared GPU runners. Run TheRock's cleanup_processes.sh as test_component.yml does: from its own checkout before the workspace is cleaned, and again after the suite. The second run cannot fail the job: the container's init does not reap the processes it kills, and the script can then report a failure. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-gpu.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 0a774e4a8176..4c6b8d200236 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -361,6 +361,23 @@ jobs: -e ROCR_VISIBLE_DEVICES -e KUBE_CPU_REQUEST steps: + # The pre-job cleanup in TheRock's test_component.yml: before the + # checkout below cleans the workspace, stop the GPU processes that an + # earlier job on this runner left running from its build directory. + - name: Fetch the GPU process cleanup script + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ROCm/TheRock + ref: 488e66e7c2220df926c2762e9b97a50becfb4239 # 2026-10-21 + path: prejob + sparse-checkout: build_tools/github_actions/cleanup_processes.sh + sparse-checkout-cone-mode: false + + - name: Pre-job cleanup GPU processes + timeout-minutes: 5 + run: bash prejob/build_tools/github_actions/cleanup_processes.sh + - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -434,3 +451,11 @@ jobs: path: integration-${{ matrix.family }}.xml if-no-files-found: ignore retention-days: 7 + + # The job container's init never reaps a killed process, so the script + # can report a failure after a passing suite; that must not fail the job. + - name: Post-job cleanup GPU processes + if: ${{ always() }} + continue-on-error: true + timeout-minutes: 5 + run: bash therock/build_tools/github_actions/cleanup_processes.sh From 5867e69c82d41a21e86bc33bc7b9cff83c0845e1 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 23 Oct 2026 08:41:08 +0200 Subject: [PATCH 63/89] HRR CI: document the paths filter and TheRock's rocJitsu The ci:skip note is out of date: HRR workflow, test, tool and skill changes now skip TheRock CI through skip-ci-config.toml. Describe the hrr-ci.yml paths filter and simulator.source_paths too. Signed-off-by: Daniel Olsson --- projects/hrr/tests/README.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index 897a0a2874c9..a3854a2eeb5c 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -153,10 +153,17 @@ applies (`scripts/hrr_ci_configure.py`): - `paths.cpu`: a change under one of these runs the generated-source, script and unit jobs. `paths.capture` also runs the capture-enabled build and the GPU suite. - `paths.excluded` counts for neither. + `paths.excluded` counts for neither. The `paths` filter of `hrr-ci.yml` is the same + lists, so a pull request that touches none of them starts no HRR CI run; + `test_hrr_ci_configure.py` fails when the filter and `ci.yaml` disagree. - `capture_label` (`ci:hrr-capture`) runs the GPU suite on a pull request that touches - no capture path. Adding it restarts the run; adding any other label, including - TheRock's `ci:gfx...` labels, leaves the run in progress alone. + an HRR path but no capture path. Adding it restarts the run; adding any other label, + including TheRock's `ci:gfx...` labels, leaves the run in progress alone. +- `simulator.source_paths`: the simulator jobs take rocJitsu, with that build's + configs, from the newest develop TheRock run that has the rocJitsu artifacts, if + those include the launcher. They build it from this tree instead on `schedule` and + `workflow_dispatch`, when a change touches one of these paths or cannot be listed, + and when that run's artifacts lack the launcher. - `families`: the GPU families the suite runs on, as TheRock family keys, on the runner TheRock's family matrix names. By default each runs on every event (`pull_request`, `push`, `schedule`, `workflow_dispatch`). Per family, `enabled: false` leaves it to @@ -179,13 +186,13 @@ GPU targets in TheRock's family matrix are not all in `targets.arch` of `config/integration.yaml`. To take load off the GPU pools, disable families or narrow their triggers here. -Until the TheRock commit pinned by `.github/workflows/therock-multi-arch-ci.yml` -includes [TheRock #8965](https://github.com/ROCm/TheRock/pull/8965), TheRock -treats `hrr-ci.yml`, `hrr-gpu.yml`, and `projects/hrr` as unclassified and -builds every stage. A pull request that changes only those workflow files or -`config/ci.yaml` should carry the `ci:skip` label. Remove that label when the -change should build HRR sources, and drop this note once the pin includes -#8965. +TheRock never reads the HRR workflows or `projects/hrr/tests`, `tools` and `skills`, +so `.github/skip-ci-config.toml` lists them: a pull request that changes only those +skips TheRock CI. CLR compiles against `projects/hrr/include`. Any other change to +`projects/hrr` builds every TheRock stage until the TheRock commit pinned by +`.github/workflows/therock-multi-arch-ci.yml` includes +[TheRock #8965](https://github.com/ROCm/TheRock/pull/8965), which adds +`hrr-playback` to TheRock's `compiler-runtime` stage on Linux. ## Configuring where a case runs From 2042d5ef912a35a5f391be9c0d7d7414938d686a Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 23 Oct 2026 09:35:03 +0200 Subject: [PATCH 64/89] HRR CI: build rocJitsu in this workflow again Revert taking rocJitsu from TheRock's artifacts for the simulator jobs. TheRock will package the rocJitsu launcher itself; until then this workflow keeps building rocJitsu from emulation/rocjitsu on every run. This reverts commit c7969417dd. Signed-off-by: Daniel Olsson --- .github/workflows/hrr-ci.yml | 2 - .github/workflows/hrr-gpu.yml | 69 ++++---------- projects/hrr/tests/README.md | 5 - projects/hrr/tests/config/ci.yaml | 10 -- .../hrr/tests/scripts/hrr_capture_build.py | 82 ++-------------- .../hrr/tests/scripts/hrr_ci_configure.py | 33 +------ .../tests/scripts/test_hrr_capture_build.py | 93 ------------------- .../tests/scripts/test_hrr_ci_configure.py | 92 ------------------ 8 files changed, 29 insertions(+), 357 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index d007326a84f7..f16616c6df83 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -100,7 +100,6 @@ jobs: simulator_families: ${{ steps.decide.outputs.simulator_families }} gpu_families: ${{ steps.decide.outputs.gpu_families }} skipped_families: ${{ steps.decide.outputs.skipped_families }} - build_rocjitsu: ${{ steps.decide.outputs.build_rocjitsu }} reason: ${{ steps.decide.outputs.reason }} rocm_sdk_version: ${{ steps.decide.outputs.rocm_sdk_version }} steps: @@ -373,7 +372,6 @@ jobs: families: ${{ needs.configure.outputs.families }} simulator_families: ${{ needs.configure.outputs.simulator_families }} gpu_families: ${{ needs.configure.outputs.gpu_families }} - build_rocjitsu: ${{ needs.configure.outputs.build_rocjitsu == 'true' }} ignore_test_config: ${{ inputs.ignore_test_config || false }} summary: diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 4c6b8d200236..a7b5fca00239 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -26,10 +26,6 @@ on: description: Real-GPU rows from that list. [] skips the GPU jobs. type: string default: "[]" - build_rocjitsu: - description: Build rocJitsu from this tree instead of taking TheRock's - type: boolean - default: true ignore_test_config: description: Run the cases the suite config skips type: boolean @@ -109,12 +105,11 @@ jobs: if-no-files-found: error retention-days: 7 - # rocJitsu has no family or architecture argument, so every simulator job - # uses the same launcher, prepared once here: TheRock's from a develop run - # unless build_rocjitsu is set or no run ships it, else built from this tree. - # The tarball has bin/rocjitsu and lib/librocjitsu.so*, which the launcher - # finds at ../lib from its own path, plus share/rocjitsu/configs when it is - # TheRock's. tar keeps the exec bit that upload-artifact would drop. + # rocJitsu has no family or architecture argument: every simulator job would + # build the same binary, so build it once here. The simulator jobs download + # it from the artifact. The launcher is tools/rocjitsu/rocjitsu and finds + # librocjitsu.so at ../../ from its own path, so the tarball keeps the build/rocjitsu layout, and tar keeps the + # exec bit that upload-artifact would drop. rocjitsu-build: name: Build rocJitsu if: inputs.simulator_families != '[]' @@ -127,7 +122,6 @@ jobs: persist-credentials: false sparse-checkout: | emulation/rocjitsu - projects/hrr/tests/scripts runtimes/api-headers shared/machine-readable-isa @@ -144,6 +138,14 @@ jobs: requirements.txt sparse-checkout-cone-mode: false + - name: Install rocJitsu build packages + shell: bash + run: | + set -euo pipefail + sudo -n apt-get update + sudo -n DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential ca-certificates clang libdrm-dev ninja-build + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" @@ -157,24 +159,7 @@ jobs: -r therock/requirements.txt 'cmake==3.31.10' 'PyYAML==6.0.2' echo "${RUNNER_TEMP}/hrr-venv/bin" >> "${GITHUB_PATH}" - - name: Fetch TheRock's rocJitsu - id: prebuilt - if: ${{ !inputs.build_rocjitsu }} - env: - GITHUB_TOKEN: ${{ github.token }} - run: python projects/hrr/tests/scripts/hrr_capture_build.py fetch-rocjitsu - - - name: Install rocJitsu build packages - if: ${{ steps.prebuilt.outputs.prebuilt != 'true' }} - shell: bash - run: | - set -euo pipefail - sudo -n apt-get update - sudo -n DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential ca-certificates clang libdrm-dev ninja-build - - name: Build rocJitsu - if: ${{ steps.prebuilt.outputs.prebuilt != 'true' }} shell: bash run: | set -euo pipefail @@ -183,23 +168,12 @@ jobs: - name: Package rocJitsu shell: bash - env: - PREBUILT: ${{ steps.prebuilt.outputs.prebuilt }} run: | set -euo pipefail - mkdir -p pkg/bin pkg/lib - # cp -a keeps the symlinks of a versioned librocjitsu.so chain. - if [[ "${PREBUILT}" == "true" ]]; then - src=build/rocjitsu-prebuilt - cp -a "${src}/bin/rocjitsu" pkg/bin/ - cp -a "${src}"/lib/librocjitsu.so* pkg/lib/ - mkdir -p pkg/share/rocjitsu - cp -a "${src}/share/rocjitsu/configs" pkg/share/rocjitsu/ - else - cp -a build/rocjitsu/tools/rocjitsu/rocjitsu pkg/bin/ - cp -a build/rocjitsu/librocjitsu.so* pkg/lib/ - fi - tar -C pkg -cf rocjitsu.tar . + # The glob must expand inside build/rocjitsu, so cd there. tar keeps + # symlinks, so a versioned librocjitsu.so chain survives. + ls -l build/rocjitsu/tools/rocjitsu/rocjitsu build/rocjitsu/librocjitsu.so* + (cd build/rocjitsu && tar -cf "${GITHUB_WORKSPACE}/rocjitsu.tar" tools/rocjitsu/rocjitsu librocjitsu.so*) tar -tvf rocjitsu.tar - name: Upload rocJitsu @@ -283,13 +257,8 @@ jobs: set -euo pipefail mkdir -p build/rocjitsu tar -C build/rocjitsu -xf rocjitsu-artifact/rocjitsu.tar - ls -l build/rocjitsu/bin build/rocjitsu/lib - # TheRock's launcher runs with the configs of the same build. - configs=build/rocjitsu/share/rocjitsu/configs - if [[ -d "${configs}" ]]; then - cp -a "${configs}/." emulation/rocjitsu/configs/ - fi - echo "${GITHUB_WORKSPACE}/build/rocjitsu/bin" >> "${GITHUB_PATH}" + ls -l build/rocjitsu/tools/rocjitsu build/rocjitsu/librocjitsu.so* + echo "${GITHUB_WORKSPACE}/build/rocjitsu/tools/rocjitsu" >> "${GITHUB_PATH}" - name: Download build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index a3854a2eeb5c..03b6ead3ddd4 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -159,11 +159,6 @@ applies (`scripts/hrr_ci_configure.py`): - `capture_label` (`ci:hrr-capture`) runs the GPU suite on a pull request that touches an HRR path but no capture path. Adding it restarts the run; adding any other label, including TheRock's `ci:gfx...` labels, leaves the run in progress alone. -- `simulator.source_paths`: the simulator jobs take rocJitsu, with that build's - configs, from the newest develop TheRock run that has the rocJitsu artifacts, if - those include the launcher. They build it from this tree instead on `schedule` and - `workflow_dispatch`, when a change touches one of these paths or cannot be listed, - and when that run's artifacts lack the launcher. - `families`: the GPU families the suite runs on, as TheRock family keys, on the runner TheRock's family matrix names. By default each runs on every event (`pull_request`, `push`, `schedule`, `workflow_dispatch`). Per family, `enabled: false` leaves it to diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 0398c4e6dd51..419c37cf655c 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -79,18 +79,8 @@ backend: # # Not listed: gfx103x, gfx1150 and gfx1153, which rocJitsu has no config for. # A workflow_dispatch families input that names one fails the configure job. -# -# The simulator jobs take rocJitsu, with that build's configs, from the newest -# develop TheRock run that has the rocJitsu artifacts, if those include the -# launcher (hrr_capture_build.py fetch-rocjitsu). They build it from this tree -# instead on schedule and workflow_dispatch, when a change touches -# source_paths or cannot be listed, and when that run lacks the launcher. simulator: runner: ubuntu-24.04 - source_paths: - - emulation/rocjitsu/ - - shared/machine-readable-isa/ - - runtimes/api-headers/ default_families: [gfx90a, gfx94x, gfx950, gfx110x, gfx1151, gfx120x, gfx125x] configs: gfx94x: emulation/rocjitsu/configs/gfx942_cdna3_kmd.json diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index 4d94b72ca498..c064a633417d 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -24,8 +24,6 @@ which amdhip64 and ROCr the suite loads. install-base On a GPU machine: install the run-time artifacts of the recorded base run and the capture-enabled amdhip64. - fetch-rocjitsu Install rocJitsu from the newest develop run that has the - rocJitsu artifacts, and report whether they hold the launcher. """ from __future__ import annotations @@ -80,10 +78,6 @@ "core-hip_lib", ) -# The rocJitsu launcher, its interposer and its KMD configs, for the simulator -# jobs. Only TheRock refs that package bin/rocjitsu ship the launcher. -ROCJITSU_ARTIFACTS = ("rocjitsu_run", "rocjitsu_lib") - # What CLR's install writes into lib/: amdhip64 plus hiprtc and its builtins. # All of it comes from this checkout, so all of it is carried to the test job. CAPTURE_LIB_GLOBS = ("libamdhip64.so*", "libhiprtc*.so*") @@ -140,7 +134,7 @@ def pick_base_run(finder_json: dict) -> dict: } -def find_base_run(therock: Path, artifacts: Sequence[str] = BUILD_ARTIFACTS) -> dict: +def find_base_run(therock: Path) -> dict: cmd = [ sys.executable, therock / "build_tools" / "find_latest_artifacts.py", @@ -154,7 +148,7 @@ def find_base_run(therock: Path, artifacts: Sequence[str] = BUILD_ARTIFACTS) -> BASE_ARTIFACT_GROUP, "--json", ] - for artifact in artifacts: + for artifact in BUILD_ARTIFACTS: cmd += ["--require-artifact", f"{artifact}_generic"] print("+ " + " ".join(str(c) for c in cmd), flush=True) # Only stdout is captured: it carries the JSON, stderr the finder's log. @@ -476,55 +470,15 @@ def cache_key_digest( return digest.hexdigest() -def write_output(line: str) -> None: +def cmd_cache_key(args: argparse.Namespace) -> int: + base = json.loads(args.base_json.read_text()) + digest = cache_key_digest(args.src, base, args.therock_pin, args.image, args.archs) + line = f"digest={digest}\n" output = os.environ.get("GITHUB_OUTPUT", "") if output: with open(output, "a", encoding="utf-8") as handle: handle.write(line) sys.stdout.write(line) - - -def cmd_cache_key(args: argparse.Namespace) -> int: - base = json.loads(args.base_json.read_text()) - digest = cache_key_digest(args.src, base, args.therock_pin, args.image, args.archs) - write_output(f"digest={digest}\n") - return 0 - - -def missing_rocjitsu(prefix: Path) -> list[str]: - """What hrr-gpu.yml packages from a prebuilt rocJitsu and prefix lacks.""" - missing = [] - if not (prefix / "bin" / "rocjitsu").is_file(): - missing.append("bin/rocjitsu") - if not any((prefix / "lib").glob("librocjitsu.so*")): - missing.append("lib/librocjitsu.so") - if not (prefix / "share" / "rocjitsu" / "configs").is_dir(): - missing.append("share/rocjitsu/configs") - return missing - - -def cmd_fetch_rocjitsu( - args: argparse.Namespace, - runner: Runner = run, - finder: Callable[[Path, Sequence[str]], dict] = find_base_run, -) -> int: - """Writes prebuilt=true when --rocjitsu-dir now holds the whole launcher - layout. Any other outcome writes prebuilt=false, and the caller builds - rocJitsu.""" - found = False - try: - base = finder(args.therock, ROCJITSU_ARTIFACTS) - print(f"rocJitsu: {base['repo']} run {base['run_id']} at {base['commit']}") - runner(fetch_command(args.therock, base, args.rocjitsu_dir, ROCJITSU_ARTIFACTS)) - missing = missing_rocjitsu(args.rocjitsu_dir) - if missing: - print( - f"{args.rocjitsu_dir} lacks {', '.join(missing)}; rocJitsu will be built" - ) - found = not missing - except (SystemExit, subprocess.CalledProcessError) as e: - print(f"::warning::no prebuilt rocJitsu, so it will be built: {e}") - write_output(f"prebuilt={'true' if found else 'false'}\n") return 0 @@ -552,15 +506,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "command", - choices=( - "fetch-base", - "build", - "stage", - "verify", - "install-base", - "cache-key", - "fetch-rocjitsu", - ), + choices=("fetch-base", "build", "stage", "verify", "install-base", "cache-key"), ) parser.add_argument("--therock", type=Path, default=Path("therock")) parser.add_argument("--src", type=Path, default=Path.cwd()) @@ -568,9 +514,6 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--build", type=Path, default=Path("build")) parser.add_argument("--capture-lib", type=Path, default=Path("build/capture-lib")) parser.add_argument("--base-json", type=Path, default=Path("build/base.json")) - parser.add_argument( - "--rocjitsu-dir", type=Path, default=Path("build/rocjitsu-prebuilt") - ) parser.add_argument( "--archs", default="", @@ -581,15 +524,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) parser.add_argument("--image", default="", help="Build container image digest") args = parser.parse_args(argv) - for name in ( - "therock", - "src", - "dist", - "build", - "capture_lib", - "base_json", - "rocjitsu_dir", - ): + for name in ("therock", "src", "dist", "build", "capture_lib", "base_json"): setattr(args, name, getattr(args, name).resolve()) if args.command in ("build", "verify", "cache-key") and not args.archs: config = hrr_test_config.load(args.src / INTEGRATION_CONFIG) @@ -606,7 +541,6 @@ def main(argv: list[str] | None = None) -> int: "verify": cmd_verify, "install-base": cmd_install_base, "cache-key": cmd_cache_key, - "fetch-rocjitsu": cmd_fetch_rocjitsu, }[args.command](args) diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index e27094815b8e..38b2a8de6435 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -13,8 +13,6 @@ gpu_families The subset whose backend is gpu. skipped_families JSON list of {"family", "reason"}: the other ci.yaml families, when the suite runs. - build_rocjitsu "true" when the simulator jobs use rocJitsu built from this - tree rather than TheRock's (see builds_rocjitsu). reason One line explaining the decision, shown in the summary. rocm_sdk_version The ROCm SDK the unit job installs. @@ -85,7 +83,6 @@ class Simulator: runner: str default_families: tuple[str, ...] configs: dict[str, str] - source_paths: tuple[str, ...] = () @dataclass(frozen=True) @@ -108,7 +105,6 @@ class Decision: reason: str families: list[dict] = field(default_factory=list) skipped: list[dict] = field(default_factory=list) - build_rocjitsu: bool = False def _mapping( @@ -169,9 +165,7 @@ def _backend(value) -> Backend: def _simulator(value, families: dict) -> Simulator: - value = _mapping( - "simulator", value, ("runner", "default_families", "configs"), ("source_paths",) - ) + value = _mapping("simulator", value, ("runner", "default_families", "configs")) default_families = _strings("simulator.default_families", value["default_families"]) unknown = [key for key in default_families if key not in families] if unknown: @@ -195,11 +189,6 @@ def _simulator(value, families: dict) -> Simulator: runner=_string("simulator.runner", value["runner"]), default_families=default_families, configs=parsed, - source_paths=( - _strings("simulator.source_paths", value["source_paths"]) - if "source_paths" in value - else () - ), ) @@ -332,15 +321,6 @@ def touches_suite(paths: Iterable[str], config: Config) -> bool: return any(_matches(p, config.suite_paths) for p in relevant_paths(paths, config)) -def builds_rocjitsu(event: str, paths: list[str] | None, config: Config) -> bool: - """Whether the simulator jobs use rocJitsu built from this tree rather than - TheRock's. schedule and workflow_dispatch test this tree's rocJitsu, as - does a change set that touches simulator.source_paths or cannot be listed.""" - if event in ("schedule", "workflow_dispatch") or paths is None: - return True - return any(_matches(p, config.simulator.source_paths) for p in paths) - - def label_matters(label: str, config: Config) -> bool: """Whether adding this label can change what runs. hrr-ci.yml's concurrency group makes the same test, so that other labels do not cancel a run. @@ -551,15 +531,7 @@ def decide( return Decision( run_cpu, False, why + "; no GPU family with a runner was selected" ) - simulated = resolved == "simulator" - return Decision( - run_cpu, - True, - why, - families, - skipped, - build_rocjitsu=simulated and builds_rocjitsu(event, paths, config), - ) + return Decision(run_cpu, True, why, families, skipped) def is_fork(event: str, head_repo: str, repo: str) -> bool: @@ -629,7 +601,6 @@ def write_outputs( f"simulator_families={_dump(simulator)}", f"gpu_families={_dump(gpu)}", f"skipped_families={_dump(decision.skipped)}", - f"build_rocjitsu={'true' if decision.build_rocjitsu else 'false'}", f"reason={decision.reason}", f"rocm_sdk_version={rocm_sdk_version}", ] diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index d5ada1b1af31..9a12a390fa1e 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -4,16 +4,12 @@ """Tests for hrr_capture_build.py.""" import argparse -import io import json import os -import subprocess import sys import tempfile import unittest -from contextlib import redirect_stdout from pathlib import Path -from unittest import mock sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -368,94 +364,5 @@ def test_reuses_recorded_base(self): self.assertIn("core-runtime_dev", cmd) -class FetchRocjitsuTest(unittest.TestCase): - # What fetch_artifacts.py --flatten writes for rocjitsu_run and rocjitsu_lib - # from a TheRock ref that does not package the launcher. - TODAY = ( - "bin/rj_pretranslate", - "lib/librocjitsu.so", - "share/rocjitsu/configs/a.json", - ) - LAUNCHER = "bin/rocjitsu" - - def fetch(self, finder, files=(), fail_after_writing=False): - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "github_output" - rocjitsu_dir = Path(tmp) / "rocjitsu" - calls = [] - - def runner(cmd): - calls.append([str(c) for c in cmd]) - for rel in files: - (rocjitsu_dir / rel).parent.mkdir(parents=True, exist_ok=True) - (rocjitsu_dir / rel).write_text("") - if fail_after_writing: - raise subprocess.CalledProcessError(1, cmd) - - args = argparse.Namespace(therock=Path("/t"), rocjitsu_dir=rocjitsu_dir) - env = dict(os.environ, GITHUB_OUTPUT=str(out)) - with mock.patch.dict(os.environ, env), redirect_stdout(io.StringIO()): - self.assertEqual( - cb.cmd_fetch_rocjitsu(args, runner=runner, finder=finder), 0 - ) - return calls, out.read_text() - - @staticmethod - def found(*_): - return cb.pick_base_run(FINDER_FOUND) - - def test_launcher_in_the_artifacts(self): - requested = [] - - def finder(_therock, artifacts): - requested.append(artifacts) - return cb.pick_base_run(FINDER_FOUND) - - calls, output = self.fetch(finder, self.TODAY + (self.LAUNCHER,)) - self.assertEqual(output, "prebuilt=true\n") - self.assertEqual(requested, [cb.ROCJITSU_ARTIFACTS]) - self.assertEqual(calls[0][calls[0].index("--run-id") + 1], "123") - self.assertIn("rocjitsu_run", calls[0]) - - def test_todays_artifacts_lack_the_launcher(self): - _, output = self.fetch(self.found, self.TODAY) - self.assertEqual(output, "prebuilt=false\n") - - def test_launcher_without_its_interposer(self): - files = (self.LAUNCHER, "share/rocjitsu/configs/a.json") - _, output = self.fetch(self.found, files) - self.assertEqual(output, "prebuilt=false\n") - - def test_fetch_that_fails_after_writing_the_launcher(self): - _, output = self.fetch( - self.found, self.TODAY + (self.LAUNCHER,), fail_after_writing=True - ) - self.assertEqual(output, "prebuilt=false\n") - - def test_finder_finds_no_run(self): - # find_latest_artifacts.py exits 1 when no run matches. - def finder(*_): - raise subprocess.CalledProcessError(1, ["find_latest_artifacts.py"]) - - calls, output = self.fetch(finder, self.TODAY + (self.LAUNCHER,)) - self.assertEqual(calls, []) - self.assertEqual(output, "prebuilt=false\n") - - def test_finder_requires_the_rocjitsu_artifacts(self): - result = mock.Mock(stdout=json.dumps(FINDER_FOUND)) - with mock.patch.object(cb.subprocess, "run", return_value=result) as run: - with redirect_stdout(io.StringIO()): - cb.find_base_run(Path("/t"), cb.ROCJITSU_ARTIFACTS) - cmd = [str(c) for c in run.call_args.args[0]] - required = [cmd[i + 1] for i, c in enumerate(cmd) if c == "--require-artifact"] - self.assertEqual(required, ["rocjitsu_run_generic", "rocjitsu_lib_generic"]) - - def test_fetch_rocjitsu_command_and_its_directory(self): - # hrr-gpu.yml packages from build/rocjitsu-prebuilt. - args = cb.parse_args(["fetch-rocjitsu"]) - self.assertEqual(args.command, "fetch-rocjitsu") - self.assertEqual(args.rocjitsu_dir, Path("build/rocjitsu-prebuilt").resolve()) - - if __name__ == "__main__": unittest.main() diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index e846cfc1b1ae..139af11fb2b2 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -473,98 +473,6 @@ def test_unknown_event(self): self.assertFalse(d.run_gpu) -class RocjitsuSourceTest(unittest.TestCase): - HRR = "projects/hrr/playback/x.cpp" - - def test_pull_request_and_push_take_therocks_rocjitsu(self): - for event in ("pull_request", "push"): - decision = decide(event, [self.HRR], config=REPO) - self.assertTrue(decision.run_gpu, event) - self.assertFalse(decision.build_rocjitsu, event) - - def test_a_rocjitsu_change_builds_it(self): - for path in REPO.simulator.source_paths: - decision = decide("pull_request", [self.HRR, path + "x.cpp"], config=REPO) - self.assertTrue(decision.build_rocjitsu, path) - - def test_schedule_dispatch_and_unlisted_changes_build_it(self): - for event in ("schedule", "workflow_dispatch"): - self.assertTrue(decide(event, [], config=REPO).build_rocjitsu, event) - self.assertTrue(decide("pull_request", None, config=REPO).build_rocjitsu) - - def test_gpu_backend_needs_no_rocjitsu(self): - decision = decide("schedule", [], config=CONFIG) - self.assertTrue(decision.run_gpu) - self.assertFalse(decision.build_rocjitsu) - - def test_source_paths_are_optional(self): - data = config_data() - del data["simulator"]["source_paths"] - self.assertEqual(cfg.parse_config(data).simulator.source_paths, ()) - - def test_source_paths_must_be_a_list_of_paths(self): - for value in ("emulation/rocjitsu/", []): - data = config_data() - data["simulator"]["source_paths"] = value - with self.assertRaises(cfg.ConfigError, msg=repr(value)): - cfg.parse_config(data) - - def output(self, decision) -> str: - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "out" - with redirect_stdout(io.StringIO()): - cfg.write_outputs(decision, "1.2.3", str(out)) - return out.read_text() - - def test_build_rocjitsu_is_an_output(self): - self.assertIn( - "build_rocjitsu=true\n", self.output(decide("schedule", [], config=REPO)) - ) - self.assertIn( - "build_rocjitsu=false\n", - self.output(decide("pull_request", [self.HRR], config=REPO)), - ) - - -class WorkflowRocjitsuWiringTest(unittest.TestCase): - ROOT = Path(__file__).resolve().parents[4] - - def workflow(self, name: str) -> dict: - path = self.ROOT / ".github/workflows" / name - return yaml.safe_load(path.read_text(encoding="utf-8")) - - def test_build_rocjitsu_reaches_the_fetch_step(self): - ci = self.workflow("hrr-ci.yml")["jobs"] - self.assertEqual( - ci["configure"]["outputs"]["build_rocjitsu"], - "${{ steps.decide.outputs.build_rocjitsu }}", - ) - self.assertEqual( - ci["gpu"]["with"]["build_rocjitsu"], - "${{ needs.configure.outputs.build_rocjitsu == 'true' }}", - ) - gpu = self.workflow("hrr-gpu.yml") - triggers = gpu.get("on", gpu.get(True)) - self.assertEqual( - triggers["workflow_call"]["inputs"]["build_rocjitsu"]["type"], "boolean" - ) - steps = {s.get("name"): s for s in gpu["jobs"]["rocjitsu-build"]["steps"]} - fetch = steps["Fetch TheRock's rocJitsu"] - self.assertEqual(fetch["id"], "prebuilt") - self.assertEqual(fetch["if"], "${{ !inputs.build_rocjitsu }}") - self.assertIn("fetch-rocjitsu", fetch["run"]) - for name in ("Install rocJitsu build packages", "Build rocJitsu"): - self.assertEqual( - steps[name]["if"], "${{ steps.prebuilt.outputs.prebuilt != 'true' }}" - ) - - def test_rocjitsu_build_checks_out_its_source_paths(self): - steps = self.workflow("hrr-gpu.yml")["jobs"]["rocjitsu-build"]["steps"] - sparse = steps[0]["with"]["sparse-checkout"].split() - for path in REPO.simulator.source_paths: - self.assertIn(path.rstrip("/"), sparse) - - class SimulatorBackendTest(unittest.TestCase): def test_default_runs_no_real_gpu_job_on_any_event(self): """Pull requests, pushes and the nightly run only rocJitsu jobs, on every From 093f5d477efef3f6f622ddb3a163cf0a9c785c6d Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 26 Oct 2026 14:12:57 +0100 Subject: [PATCH 65/89] HRR CI: list Unit_HRR_CaptureState_EndCaptureClearsFlag in handler.yaml The develop merge brought in the case (#13885); the case-list check needs an entry. --- projects/hrr/tests/config/handler.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml index cf1253ff7aca..418a1d038401 100644 --- a/projects/hrr/tests/config/handler.yaml +++ b/projects/hrr/tests/config/handler.yaml @@ -26,6 +26,7 @@ cases: Unit_HRR_AccessPolicyWindow_StreamAttributeTranslatesBasePtr: {} Unit_HRR_Blob_AbsentBlobDoesNotEndTheReplay: {} Unit_HRR_Blob_CapRefusalEndsTheReplay: {} + Unit_HRR_CaptureState_EndCaptureClearsFlag: {} Unit_HRR_GraphDeps_InlineCapacity: {} Unit_HRR_GraphDeps_MemAllocNodeRefusesOversizeCount: {} Unit_HRR_GraphDeps_SymbolCopyNodesRefuseOversizeCount: {} From 9d0a4a7454ea96b3059d1aa7a055e19d72f020ef Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 26 Oct 2026 14:12:57 +0100 Subject: [PATCH 66/89] HRR CI: remove unused and duplicated CI config, scripts and checks - Drop family triggers/runner/multi_gpu_triggers and backend.gpu.triggers: nothing set them, and real GPUs run only on workflow_dispatch. - Describe what ci.yaml, README and hrr-ci.yml's nightly actually run (the simulator on simulator.default_families). - Stop reporting a skipped 'Build rocJitsu' job as a failure on a backend gpu dispatch; a failed build already shows as missing simulator families. - hrr_check_sources.py keeps the API-ID compatibility and workload checks; the generated-file comparisons are test_generated_files_current.py. - Remove the unused stage and cache-key commands of hrr_capture_build.py. - Share the requested-family check; drop the tests that copied ci.yaml values and the TheRock pin test. - hrr-gpu.yml: drop the unused families input, the TheRock checkout and requirements from the rocJitsu build, and use matrix.runner for the simulator jobs. --- .github/workflows/hrr-ci.yml | 8 +- .github/workflows/hrr-gpu.yml | 29 +---- projects/hrr/tests/README.md | 29 ++--- projects/hrr/tests/config/ci.yaml | 29 +---- .../hrr/tests/scripts/hrr_capture_build.py | 110 +--------------- .../hrr/tests/scripts/hrr_ci_configure.py | 103 ++++----------- projects/hrr/tests/scripts/hrr_ci_report.py | 2 - .../tests/scripts/test_hrr_capture_build.py | 78 ----------- .../tests/scripts/test_hrr_check_sources.py | 86 ++---------- .../tests/scripts/test_hrr_ci_configure.py | 123 ++---------------- .../hrr/tests/scripts/test_hrr_ci_report.py | 16 +-- projects/hrr/tools/hrr_check_sources.py | 119 +++-------------- 12 files changed, 107 insertions(+), 625 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index f16616c6df83..9cb070d9254d 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -1,4 +1,4 @@ -# HRR CI: generated-source checks, the HRR helper-script tests and the host +# HRR CI: API-ID and workload checks, the HRR helper-script tests and the host # unit suite, plus the GPU capture/replay suite (hrr-gpu.yml) for changes that # touch capture or replay. projects/hrr/tests/scripts/hrr_ci_configure.py # decides what runs from projects/hrr/tests/config/ci.yaml; "HRR CI Summary" @@ -48,7 +48,8 @@ on: - "!projects/hrr/README.md" - "!projects/hrr/skills/**" schedule: - # Nightly: capture-enabled build and the GPU suite on every HRR family. + # Nightly: capture-enabled build and the rocJitsu simulator suite on + # simulator.default_families. Real GPUs run only on workflow_dispatch. - cron: "0 3 * * *" workflow_dispatch: inputs: @@ -197,7 +198,7 @@ jobs: - name: Install PyYAML run: python -m pip install --no-cache-dir PyYAML==6.0.2 - - name: Check generated sources, API IDs and the API matrix + - name: Check API IDs and the API matrix workloads env: BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} run: python projects/hrr/tools/hrr_check_sources.py all --base-sha "${BASE_SHA}" @@ -369,7 +370,6 @@ jobs: uses: ./.github/workflows/hrr-gpu.yml # Note: not using 'secrets: inherit' here; no secrets are needed. with: - families: ${{ needs.configure.outputs.families }} simulator_families: ${{ needs.configure.outputs.simulator_families }} gpu_families: ${{ needs.configure.outputs.gpu_families }} ignore_test_config: ${{ inputs.ignore_test_config || false }} diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index a7b5fca00239..575616f88f61 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -14,16 +14,12 @@ name: HRR GPU on: workflow_call: inputs: - families: - description: JSON list of {family, runner, backend, arch, config} from hrr_ci_configure.py - type: string - required: true simulator_families: - description: Simulator rows from that list. [] skips the rocJitsu jobs. + description: JSON list of {family, runner, backend, arch, config} rows from hrr_ci_configure.py for the rocJitsu jobs. [] skips them. type: string default: "[]" gpu_families: - description: Real-GPU rows from that list. [] skips the GPU jobs. + description: Real-GPU rows in the same form. [] skips the GPU jobs. type: string default: "[]" ignore_test_config: @@ -75,8 +71,7 @@ jobs: # No cache restore: a cache hit left librocm_kpack.so.0 out of the tree and # Verify failed. Off until that failure is understood, so the build always - # runs. The `cache-key` and `stage` commands of hrr_capture_build.py are - # the pieces to reuse. + # runs. - name: Build run: python3 projects/hrr/tests/scripts/hrr_capture_build.py build @@ -125,19 +120,6 @@ jobs: runtimes/api-headers shared/machine-readable-isa - - name: Checkout TheRock artifact tools - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - repository: ROCm/TheRock - ref: 488e66e7c2220df926c2762e9b97a50becfb4239 # 2026-10-21 - path: therock - sparse-checkout: | - build_tools - cmake/therock_amdgpu_targets.cmake - requirements.txt - sparse-checkout-cone-mode: false - - name: Install rocJitsu build packages shell: bash run: | @@ -155,8 +137,7 @@ jobs: run: | set -euo pipefail python -m venv "${RUNNER_TEMP}/hrr-venv" - "${RUNNER_TEMP}/hrr-venv/bin/python" -m pip install --no-cache-dir \ - -r therock/requirements.txt 'cmake==3.31.10' 'PyYAML==6.0.2' + "${RUNNER_TEMP}/hrr-venv/bin/python" -m pip install --no-cache-dir 'cmake==3.31.10' echo "${RUNNER_TEMP}/hrr-venv/bin" >> "${GITHUB_PATH}" - name: Build rocJitsu @@ -188,7 +169,7 @@ jobs: name: rocJitsu (${{ matrix.family }}) needs: [build, rocjitsu-build] if: inputs.simulator_families != '[]' - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.runner }} timeout-minutes: 45 strategy: fail-fast: false diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index 03b6ead3ddd4..1b6797ffd5f7 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -143,7 +143,7 @@ python3 -m unittest discover -s projects/hrr/tests/scripts -p 'test_*.py' -v They cover `run_catch2.py`, `summarize_junit.py`, `hrr_test_config.py`, and the HRR CI scripts (`hrr_ci_configure.py`, `hrr_capture_build.py`, `hrr_ci_report.py`), plus the tools in `../tools`: the HRR API ID compatibility check, the -generated-source checks (`hrr_check_sources.py`), the playback-handler default in +API-ID and workload checks (`hrr_check_sources.py`), the playback-handler default in the generator, and the [api-matrix](../tools/api-matrix/README.md) scripts. ## Choosing what CI runs @@ -159,27 +159,24 @@ applies (`scripts/hrr_ci_configure.py`): - `capture_label` (`ci:hrr-capture`) runs the GPU suite on a pull request that touches an HRR path but no capture path. Adding it restarts the run; adding any other label, including TheRock's `ci:gfx...` labels, leaves the run in progress alone. -- `families`: the GPU families the suite runs on, as TheRock family keys, on the runner - TheRock's family matrix names. By default each runs on every event (`pull_request`, - `push`, `schedule`, `workflow_dispatch`). Per family, `enabled: false` leaves it to - explicit requests, `triggers` narrows the events, `runner` overrides the runner label - and `multi_gpu_triggers` names the events on which it takes TheRock's multi-GPU - runner instead of the single-GPU one. TheRock's multi-GPU pools are 8-GPU nodes, so - only gfx950 uses one, nightly; elsewhere the cases that need two devices skip and - show in the summary's Skip column. The `workflow_dispatch` input `families` - replaces the selection and can name a disabled family. Only gfx94x is enabled - today; the others are listed with `enabled: false`, so a real-GPU run includes - them only when that input names them. `ci:gfx...` labels are TheRock's and do - not select an HRR family. +- `backend` and `simulator`: every event except a `workflow_dispatch` that asks for + `backend: gpu` runs on the rocJitsu simulator, on the families in + `simulator.default_families`, each with its rocJitsu config from `simulator.configs`. + No pull request, push or nightly run queues on a real GPU runner. +- `families`: the GPU families as TheRock family keys. They matter only for a + `workflow_dispatch` with `backend: gpu`, which runs the `enabled` ones on the runner + TheRock's family matrix names, or the ones the `families` input names, disabled or + not. Only gfx94x is enabled today; the others are listed with `enabled: false`. + The input also replaces the default families of a simulator run. `ci:gfx...` labels + are TheRock's and do not select an HRR family. `targets.arch` in `config/integration.yaml` stays wide on purpose: configure fails unless it covers every family listed here, and a requested family needs its architecture compiled into the binary. - `rocm_sdk_version`: the ROCm SDK the unit job installs. -An unknown key, trigger or family fails the configure job, and so does a family whose +An unknown key or family fails the configure job, and so does a family whose GPU targets in TheRock's family matrix are not all in `targets.arch` of -`config/integration.yaml`. To take load off the GPU -pools, disable families or narrow their triggers here. +`config/integration.yaml`. To take a pool out of a real-GPU run, set `enabled: false`. TheRock never reads the HRR workflows or `projects/hrr/tests`, `tools` and `skills`, so `.github/skip-ci-config.toml` lists them: a pull request that changes only those diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 419c37cf655c..4b303281bc45 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -47,8 +47,7 @@ paths: - projects/hrr/tests/scripts/hrr_ci_report.py - .github/workflows/hrr-ci.yml - .github/workflows/hrr-gpu.yml - # Runs the suite without changing the capture binary. The capture cache key - # does not include these paths. + # Runs the suite without changing the capture binary. suite: - projects/hrr/tests/config/ci.yaml # Ignored by both lists; hrr-skills.yml covers these. @@ -62,14 +61,12 @@ paths: capture_label: ci:hrr-capture # simulator runs rocJitsu on ubuntu-24.04. gpu runs TheRock's real runners. -# workflow_dispatch passes its backend input. Any other event uses default, -# unless the event is listed under backend.gpu.triggers. No event is, so a -# pull request, a push and the nightly schedule never queue on a real GPU -# runner; a person asks for one with workflow_dispatch (backend gpu). +# workflow_dispatch passes its backend input. Any other event uses default, so +# a pull request, a push and the nightly schedule run on the simulator and +# never queue on a real GPU runner; a person asks for one with +# workflow_dispatch (backend gpu). backend: default: simulator - gpu: - triggers: [] # Default simulator jobs. These do not consult families.*.enabled. Each row is # a rocJitsu KMD config that exposes the family's GPU to HIP, and the whole @@ -100,19 +97,10 @@ simulator: # enabled false: run only when the workflow_dispatch families input names # it. Default true. ci:gfx labels belong to TheRock and are not # read here. -# triggers Events that run this family: pull_request, push, schedule, -# workflow_dispatch. Default all four. -# runner Runner label to use instead of TheRock's. -# multi_gpu_triggers -# Events on which to take TheRock's multi-GPU runner instead of -# its single-GPU one. Default none. The cases that need two -# devices (the T3 tier) skip on a single GPU. TheRock's only -# multi-GPU pools are 8-GPU nodes, so name only events that must -# cover those cases. # # A family without a runner label is skipped. Every job queues on its own -# runner pool and the summary waits for all of them, so to reduce GPU load or -# take a pool out, set enabled: false or narrow triggers here. +# runner pool and the summary waits for all of them, so to take a pool out, +# set enabled: false. # # Real GPUs run only when a person starts workflow_dispatch with backend gpu. # Then only gfx94x is enabled; the other families stay listed and disabled, so @@ -126,10 +114,7 @@ families: gfx90a: enabled: false gfx94x: {} - # Off by default with the other non-gfx94x families. Nightly on an 8-GPU - # node, so once re-enabled the multi-device cases run once a day. gfx950: - multi_gpu_triggers: [schedule] enabled: false gfx103x: enabled: false diff --git a/projects/hrr/tests/scripts/hrr_capture_build.py b/projects/hrr/tests/scripts/hrr_capture_build.py index c064a633417d..d9fe4c5a251d 100644 --- a/projects/hrr/tests/scripts/hrr_capture_build.py +++ b/projects/hrr/tests/scripts/hrr_capture_build.py @@ -29,7 +29,6 @@ from __future__ import annotations import argparse -import hashlib import json import os import re @@ -92,20 +91,6 @@ # What the GPU suite runs. The host unit suite is built and run by hrr-ci.yml. HRR_TARGETS = ("hrr-playback", "hrr-integration-tests", "hrr-test-dispatch-tool") -# What the capture cache key hashes. ci.yaml is a suite-selection file and is -# deliberately absent: a family or backend edit must restore this build. -CACHE_SOURCE_PATHS = ( - "projects/clr", - "projects/hip", - "projects/hrr/CMakeLists.txt", - "projects/hrr/include", - "projects/hrr/playback", - "projects/hrr/tests/CMakeLists.txt", - "projects/hrr/tests/include", - "projects/hrr/tests/integration", - "projects/hrr/tests/scripts/hrr_capture_build.py", -) - # Uploaded by hrr-gpu.yml's build job, relative to --build. EXECUTABLES = ( "hrr/playback/hrr-playback", @@ -291,9 +276,8 @@ def staged_library_sources(ldd_output: str, lib_dir: Path) -> list[Path]: """Libraries ldd resolved outside lib_dir and outside the system directories. The integration binary's RUNPATH searches dist/lib, which is where it loads - amdhip64. A cache restore skips the build, so a dependency that the build - tree had and the base artifacts do not (librocm_kpack.so.0) has to be - copied here for the restored binary to start. + amdhip64. A dependency that the build tree has and the base artifacts do + not (librocm_kpack.so.0) has to be copied here for the binary to start. """ system_roots = (Path("/lib"), Path("/lib64"), Path("/usr/lib"), Path("/usr/lib64")) sources: list[Path] = [] @@ -310,14 +294,12 @@ def staged_library_sources(ldd_output: str, lib_dir: Path) -> list[Path]: return sources -def stage_runtime_libs( - binary: Path, lib_dir: Path, env: dict | None = None -) -> list[str]: +def stage_runtime_libs(binary: Path, lib_dir: Path) -> list[str]: """Copy non-system libraries the binary loads into lib_dir. Returns names copied.""" if not binary.is_file(): return [] out = subprocess.run( - ["ldd", str(binary)], check=True, capture_output=True, text=True, env=env + ["ldd", str(binary)], check=True, capture_output=True, text=True ).stdout lib_dir.mkdir(parents=True, exist_ok=True) copied: list[str] = [] @@ -380,28 +362,6 @@ def cmd_build(args: argparse.Namespace, runner: Runner = run) -> int: return 0 -def cmd_stage(args: argparse.Namespace) -> int: - """Stage the runtime libraries again after a cache restore. - - The cache keeps the binary but not the copies build staged into dist/lib. - Without the build tree, ldd finds the libraries only in the installed - subtrees (dist/lib/llvm/lib, dist/lib/rocm_sysdeps/lib), so put those on - its search path. - """ - lib_dir = args.dist / "lib" - search = [str(p) for p in sorted(lib_dir.glob("*/lib")) if p.is_dir()] - env = dict(os.environ) - env["LD_LIBRARY_PATH"] = os.pathsep.join( - search + ([env["LD_LIBRARY_PATH"]] if env.get("LD_LIBRARY_PATH") else []) - ) - stage_runtime_libs( - args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests", - lib_dir, - env, - ) - return 0 - - def cmd_verify(args: argparse.Namespace) -> int: exe = args.build / "hrr" / "tests" / "integration" / "hrr-integration-tests" run( @@ -430,58 +390,6 @@ def cmd_verify(args: argparse.Namespace) -> int: return check_linkage(exe, args.dist / "lib") -def _tree_digest(path: Path) -> str: - digest = hashlib.sha256() - if path.is_file(): - digest.update(path.read_bytes()) - return digest.hexdigest() - if path.is_dir(): - files = sorted(item for item in path.rglob("*") if item.is_file()) - for item in files: - digest.update(item.relative_to(path).as_posix().encode()) - digest.update(b"\0") - digest.update(item.read_bytes()) - digest.update(b"\0") - return digest.hexdigest() - - -def cache_key_digest( - src: Path, base: dict, therock_pin: str, image: str, archs: str -) -> str: - """Content key for the capture build. Not the merge SHA.""" - digest = hashlib.sha256() - header = "\n".join( - [ - f"repo={base['repo']}", - f"run_id={base['run_id']}", - f"commit={base['commit']}", - f"artifact_group={base['artifact_group']}", - f"therock={therock_pin}", - f"image={image}", - f"archs={archs}", - ] - ) - digest.update(header.encode()) - for rel in CACHE_SOURCE_PATHS: - digest.update(b"\n") - digest.update(rel.encode()) - digest.update(b"\n") - digest.update(_tree_digest(src / rel).encode()) - return digest.hexdigest() - - -def cmd_cache_key(args: argparse.Namespace) -> int: - base = json.loads(args.base_json.read_text()) - digest = cache_key_digest(args.src, base, args.therock_pin, args.image, args.archs) - line = f"digest={digest}\n" - output = os.environ.get("GITHUB_OUTPUT", "") - if output: - with open(output, "a", encoding="utf-8") as handle: - handle.write(line) - sys.stdout.write(line) - return 0 - - def restore_executable_bits(build: Path) -> None: """Artifact downloads drop the executable bit.""" for rel in EXECUTABLES: @@ -506,7 +414,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "command", - choices=("fetch-base", "build", "stage", "verify", "install-base", "cache-key"), + choices=("fetch-base", "build", "verify", "install-base"), ) parser.add_argument("--therock", type=Path, default=Path("therock")) parser.add_argument("--src", type=Path, default=Path.cwd()) @@ -519,14 +427,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: default="", help=f"Semicolon-separated GPU targets; default targets.arch of {INTEGRATION_CONFIG}", ) - parser.add_argument( - "--therock-pin", default="", help="TheRock commit the build tools came from" - ) - parser.add_argument("--image", default="", help="Build container image digest") args = parser.parse_args(argv) for name in ("therock", "src", "dist", "build", "capture_lib", "base_json"): setattr(args, name, getattr(args, name).resolve()) - if args.command in ("build", "verify", "cache-key") and not args.archs: + if args.command in ("build", "verify") and not args.archs: config = hrr_test_config.load(args.src / INTEGRATION_CONFIG) args.archs = ";".join(config["targets"]["arch"]) return args @@ -537,10 +441,8 @@ def main(argv: list[str] | None = None) -> int: return { "fetch-base": cmd_fetch_base, "build": cmd_build, - "stage": cmd_stage, "verify": cmd_verify, "install-base": cmd_install_base, - "cache-key": cmd_cache_key, }[args.command](args) diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index 38b2a8de6435..ce9669bac628 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -27,10 +27,8 @@ workflow_dispatch families input replaces the selection. A simulator family with no config is an error. ci:gfx labels are TheRock's, so this script does not read them. The GPU backend -runs every enabled family whose triggers include the event, on TheRock's -runner unless ci.yaml overrides it, and takes the multi-GPU runner only on -its multi_gpu_triggers. workflow_dispatch uses its backend input; any other -event uses backend.default unless the event is in backend.gpu.triggers. +runs every enabled family on TheRock's runner. workflow_dispatch uses its +backend input; any other event uses backend.default. """ from __future__ import annotations @@ -67,15 +65,11 @@ class ConfigError(ValueError): @dataclass(frozen=True) class Family: enabled: bool = True - triggers: tuple[str, ...] = EVENTS - runner: str = "" - multi_gpu_triggers: tuple[str, ...] = () @dataclass(frozen=True) class Backend: default: str - gpu_triggers: tuple[str, ...] @dataclass(frozen=True) @@ -139,29 +133,14 @@ def _bool(where: str, value) -> bool: return value -def _triggers(where: str, value, allow_empty: bool = False) -> tuple[str, ...]: - if allow_empty and isinstance(value, list) and not value: - return () - triggers = _strings(where, value) - for trigger in triggers: - if trigger not in EVENTS: - raise ConfigError( - f"{where}: unknown trigger {trigger!r} (expected {', '.join(EVENTS)})" - ) - return triggers - - def _backend(value) -> Backend: - value = _mapping("backend", value, ("default", "gpu")) + value = _mapping("backend", value, ("default",)) default = _string("backend.default", value["default"]) if default not in ("simulator", "gpu"): raise ConfigError( f"backend.default: expected simulator or gpu, got {default!r}" ) - gpu = _mapping("backend.gpu", value["gpu"], ("triggers",)) - return Backend( - default, _triggers("backend.gpu.triggers", gpu["triggers"], allow_empty=True) - ) + return Backend(default) def _simulator(value, families: dict) -> Simulator: @@ -193,19 +172,8 @@ def _simulator(value, families: dict) -> Simulator: def _family(where: str, value) -> Family: - value = _mapping( - where, value or {}, (), ("enabled", "triggers", "runner", "multi_gpu_triggers") - ) - return Family( - enabled=_bool(f"{where}.enabled", value.get("enabled", True)), - triggers=_triggers(f"{where}.triggers", value.get("triggers", list(EVENTS))), - runner=_string(f"{where}.runner", value["runner"]) if "runner" in value else "", - multi_gpu_triggers=( - _triggers(f"{where}.multi_gpu_triggers", value["multi_gpu_triggers"]) - if "multi_gpu_triggers" in value - else () - ), - ) + value = _mapping(where, value or {}, (), ("enabled",)) + return Family(enabled=_bool(f"{where}.enabled", value.get("enabled", True))) def parse_config(data) -> Config: @@ -329,15 +297,7 @@ def label_matters(label: str, config: Config) -> bool: return label == config.capture_label.lower() -def runner_for( - linux: dict, pick_weighted: Callable[[list, str], str], multi_gpu: bool = False -) -> str: - if multi_gpu: - labels = linux.get("test-runs-on-multi-gpu-labels") - if labels: - return pick_weighted(labels, "hrr-multi-gpu") - if linux.get("test-runs-on-multi-gpu"): - return linux["test-runs-on-multi-gpu"] +def runner_for(linux: dict, pick_weighted: Callable[[list, str], str]) -> str: labels = linux.get("test-runs-on-labels") if labels: return pick_weighted(labels, "hrr") @@ -345,14 +305,11 @@ def runner_for( def resolve_backend(event: str, config: Config, backend_input: str) -> str: - """workflow_dispatch uses its input. Any other event uses the default, - unless that event is listed under backend.gpu.triggers.""" + """workflow_dispatch uses its input. Any other event uses the default.""" if event == "workflow_dispatch" and backend_input: if backend_input not in ("simulator", "gpu"): raise ConfigError(f"backend {backend_input!r} is not simulator or gpu") return backend_input - if event in config.backend.gpu_triggers: - return "gpu" return config.backend.default @@ -370,8 +327,17 @@ def _job(family: str, runner: str, backend: str, arch: str, config: str = "") -> } +def _requested(config: Config, requested: list[str]) -> set[str]: + unknown = [key for key in requested if key not in config.families] + if unknown: + raise ConfigError( + f"requested families {', '.join(unknown)} are not in ci.yaml" + f" (known: {', '.join(config.families)})" + ) + return set(requested) + + def select_families( - event: str, config: Config, requested: list[str], matrix: dict, @@ -380,34 +346,21 @@ def select_families( """GPU families to test and the other ci.yaml families with the reason, in ci.yaml order.""" if requested: - unknown = [key for key in requested if key not in config.families] - if unknown: - raise ConfigError( - f"requested families {', '.join(unknown)} are not in ci.yaml" - f" (known: {', '.join(config.families)})" - ) - chosen = set(requested) + chosen = _requested(config, requested) else: - chosen = { - key - for key, family in config.families.items() - if family.enabled and event in family.triggers - } + chosen = {key for key, family in config.families.items() if family.enabled} families, skipped = [], [] for key, family in config.families.items(): if key not in chosen: if requested: reason = "not requested" - elif not family.enabled: - reason = "disabled in ci.yaml" else: - reason = f"not run on {event} (ci.yaml triggers)" + reason = "disabled in ci.yaml" skipped.append({"family": key, "reason": reason}) continue linux = matrix.get(key, {}).get("linux", {}) - multi_gpu = event in family.multi_gpu_triggers - runner = family.runner or runner_for(linux, pick_weighted, multi_gpu) + runner = runner_for(linux, pick_weighted) if not runner: skipped.append({"family": key, "reason": NO_RUNNER}) continue @@ -423,13 +376,7 @@ def select_simulator( """rocJitsu families. default_families, unless the dispatch input replaces them. A family with no config is an error.""" if requested: - unknown = [key for key in requested if key not in config.families] - if unknown: - raise ConfigError( - f"requested families {', '.join(unknown)} are not in ci.yaml" - f" (known: {', '.join(config.families)})" - ) - chosen = set(requested) + chosen = _requested(config, requested) else: chosen = set(config.simulator.default_families) missing = sorted(key for key in chosen if key not in config.simulator.configs) @@ -521,9 +468,7 @@ def decide( if resolved == "simulator": families, skipped = select_simulator(config, requested, matrix) else: - families, skipped = select_families( - event, config, requested, matrix, pick_weighted - ) + families, skipped = select_families(config, requested, matrix, pick_weighted) no_runner = [s["family"] for s in skipped if s["reason"] == NO_RUNNER] if no_runner: why += f"; no runner for {', '.join(no_runner)}" diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index 70ca7866c98e..aaf4e5ef064f 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -40,8 +40,6 @@ ("Script tests", re.compile(r"^HRR script tests$")), ("Unit", re.compile(r"(?:^|/ )Unit \((?P[^)]+)\)$")), ("Build", re.compile(r"(?:^|/ )Build capture-enabled HIP and HRR$")), - # The simulator jobs download this one build of rocJitsu; its failure fails them too. - ("Build", re.compile(r"(?:^|/ )Build rocJitsu$")), # The real-GPU job is "Integration ()", the rocJitsu one "rocJitsu ()". ("Integration", re.compile(r"(?:^|/ )(?:Integration|rocJitsu) \((?P[^)]+)\)$")), ) diff --git a/projects/hrr/tests/scripts/test_hrr_capture_build.py b/projects/hrr/tests/scripts/test_hrr_capture_build.py index 9a12a390fa1e..304e82be4c52 100644 --- a/projects/hrr/tests/scripts/test_hrr_capture_build.py +++ b/projects/hrr/tests/scripts/test_hrr_capture_build.py @@ -243,84 +243,6 @@ def test_kpack_outside_the_install_is_staged(self): self.assertEqual(sources, [Path("/work/build/clr/lib/librocm_kpack.so.0")]) -class StageCommandTest(unittest.TestCase): - def test_stage_is_a_command(self): - args = cb.parse_args(["stage"]) - self.assertEqual(args.command, "stage") - - def test_stage_without_a_binary_copies_nothing(self): - with tempfile.TemporaryDirectory() as tmp: - args = cb.parse_args( - ["stage", "--dist", f"{tmp}/dist", "--build", f"{tmp}/build"] - ) - self.assertEqual(cb.cmd_stage(args), 0) - - -class CacheKeyTest(unittest.TestCase): - def test_inputs_are_the_build_and_not_ci_yaml(self): - paths = cb.CACHE_SOURCE_PATHS - for required in ( - "projects/clr", - "projects/hip", - "projects/hrr/CMakeLists.txt", - "projects/hrr/include", - "projects/hrr/playback", - "projects/hrr/tests/CMakeLists.txt", - "projects/hrr/tests/include", - "projects/hrr/tests/integration", - "projects/hrr/tests/scripts/hrr_capture_build.py", - ): - self.assertIn(required, paths) - self.assertNotIn("projects/hrr/tests/config/ci.yaml", paths) - self.assertFalse(any("ci.yaml" in path for path in paths)) - - def test_digest_tracks_base_pin_image_and_archs(self): - with tempfile.TemporaryDirectory() as tmp: - src = Path(tmp) - for rel in cb.CACHE_SOURCE_PATHS: - path = src / rel - path.parent.mkdir(parents=True, exist_ok=True) - if rel.endswith(".py") or rel.endswith(".txt"): - path.write_text("source") - else: - path.mkdir(exist_ok=True) - (path / "file.cpp").write_text("int x;") - (src / "projects/hrr/tests/config").mkdir(parents=True) - (src / "projects/hrr/tests/config/ci.yaml").write_text("families: {}\n") - base = { - "repo": "ROCm/rocm-systems", - "run_id": "1", - "commit": "abc", - "artifact_group": "gfx94X-dcgpu", - } - first = cb.cache_key_digest(src, base, "pin", "image@sha", "gfx950;gfx1250") - self.assertEqual( - first, - cb.cache_key_digest(src, base, "pin", "image@sha", "gfx950;gfx1250"), - ) - self.assertNotEqual( - first, - cb.cache_key_digest( - src, base, "other-pin", "image@sha", "gfx950;gfx1250" - ), - ) - self.assertNotEqual( - first, cb.cache_key_digest(src, base, "pin", "image@sha", "gfx942") - ) - (src / "projects/hrr/tests/config/ci.yaml").write_text( - "backend: simulator\n" - ) - self.assertEqual( - first, - cb.cache_key_digest(src, base, "pin", "image@sha", "gfx950;gfx1250"), - ) - (src / "projects/hrr/playback/file.cpp").write_text("int y;") - self.assertNotEqual( - first, - cb.cache_key_digest(src, base, "pin", "image@sha", "gfx950;gfx1250"), - ) - - class ArchsTest(unittest.TestCase): SRC = Path(__file__).resolve().parents[4] diff --git a/projects/hrr/tests/scripts/test_hrr_check_sources.py b/projects/hrr/tests/scripts/test_hrr_check_sources.py index 11c9608d536a..9f1164b658cd 100644 --- a/projects/hrr/tests/scripts/test_hrr_check_sources.py +++ b/projects/hrr/tests/scripts/test_hrr_check_sources.py @@ -22,7 +22,7 @@ def write(path: Path, text: str) -> None: class Tree(unittest.TestCase): - """A fake checkout whose generator writes whatever self.generated holds.""" + """A fake checkout whose runner records the commands it is given.""" def setUp(self): self._tmp = tempfile.TemporaryDirectory() @@ -30,11 +30,7 @@ def setUp(self): self.root = base / "src" self.out = base / "out" self.out.mkdir() - for tracked in chk.GENERATED_FILES: - write(self.root / tracked, f"// {tracked}\n") - write(self.root / chk.MATRIX_EXPECTATIONS, "expect\n") - self.generated = {t: f"// {t}\n" for t in chk.GENERATED_FILES} - self.emitted = "expect\n" + write(self.root / chk.API_ARGS_HEADER, "// current header\n") self.calls = [] self.rc = {} @@ -43,13 +39,7 @@ def tearDown(self): def runner(self, cmd): self.calls.append(list(cmd)) - script = Path(cmd[1]).name - if script == "gen_hrr_api_args.py": - for tracked, name in chk.GENERATED_FILES.items(): - write(self.out / name, self.generated[tracked]) - if "--emit-cxx" in cmd: - write(Path(cmd[cmd.index("--emit-cxx") + 1]), self.emitted) - return self.rc.get(script, 0) + return self.rc.get(Path(cmd[1]).name, 0) def quiet(self, fn, *args, **kwargs): buf = io.StringIO() @@ -58,41 +48,8 @@ def quiet(self, fn, *args, **kwargs): return result, buf.getvalue() -class GeneratedTest(Tree): - def test_up_to_date(self): - ok, _ = self.quiet(chk.check_generated, self.root, self.out, self.runner) - self.assertTrue(ok) - self.assertIn("--check-hrr-coverage", self.calls[0]) - - def test_stale_file_is_reported(self): - stale = "projects/hrr/playback/hip_playback_generated.cpp" - self.generated[stale] = "// changed\n" - ok, output = self.quiet(chk.check_generated, self.root, self.out, self.runner) - self.assertFalse(ok) - self.assertIn("::error::checked-in HRR generated sources are stale", output) - self.assertIn(stale, output) - self.assertIn("+// changed", output) - - def test_line_endings_are_ignored(self): - tracked = chk.API_ARGS_HEADER - self.generated[tracked] = f"// {tracked}\r\n" - ok, _ = self.quiet(chk.check_generated, self.root, self.out, self.runner) - self.assertTrue(ok) - - def test_generator_failure(self): - self.rc["gen_hrr_api_args.py"] = 2 - ok, output = self.quiet(chk.check_generated, self.root, self.out, self.runner) - self.assertFalse(ok) - self.assertIn("failed (exit 2)", output) - - class ApiCompatTest(Tree): - def regenerate(self): - self.quiet(chk.check_generated, self.root, self.out, self.runner) - self.calls.clear() - def test_compares_against_base(self): - self.regenerate() ok, _ = self.quiet( chk.check_api_compat, self.root, self.out, "abc", self.runner, lambda root, sha: ("base header\n", None), @@ -100,11 +57,12 @@ def test_compares_against_base(self): self.assertTrue(ok) cmd = self.calls[0] self.assertEqual(Path(cmd[1]).name, "check_hrr_api_compat.py") + current = Path(cmd[cmd.index("--current-header") + 1]) + self.assertEqual(current, self.root / chk.API_ARGS_HEADER) base = Path(cmd[cmd.index("--base-header") + 1]) self.assertEqual(base.read_text(), "base header\n") def test_incompatible_ids_fail(self): - self.regenerate() self.rc["check_hrr_api_compat.py"] = 1 ok, output = self.quiet( chk.check_api_compat, self.root, self.out, "abc", @@ -114,7 +72,6 @@ def test_incompatible_ids_fail(self): self.assertIn("not compatible with abc", output) def test_missing_base_header_passes(self): - self.regenerate() ok, output = self.quiet( chk.check_api_compat, self.root, self.out, "abc", self.runner, lambda root, sha: (None, None), @@ -126,7 +83,6 @@ def test_missing_base_header_passes(self): def test_git_failure_fails_instead_of_passing(self): # A base blob the blob:none clone cannot fetch, or a base commit it # does not have: not "the file is absent", so no silent pass. - self.regenerate() failure = (None, "cannot read projects/hrr/include/hrr/hrr_api_args.h " "at abc: fatal: could not read from remote repository") ok, output = self.quiet( @@ -138,14 +94,6 @@ def test_git_failure_fails_instead_of_passing(self): self.assertIn("::error::cannot read", output) self.assertIn("do not pass it silently", output) - def test_needs_regenerated_header(self): - ok, output = self.quiet( - chk.check_api_compat, self.root, self.out, "abc", - self.runner, lambda root, sha: ("x", None), - ) - self.assertFalse(ok) - self.assertIn("run the generated check first", output) - class BaseHeaderTest(unittest.TestCase): """base_header's telling "the path is absent" from "git broke".""" @@ -190,24 +138,18 @@ def test_base_commit_the_clone_does_not_have_fails(self): self.assertIn("at " + "0" * 40, failure) -class ApiMatrixTest(Tree): - def test_up_to_date(self): - ok, _ = self.quiet(chk.check_api_matrix, self.root, self.out, self.runner) +class WorkloadsTest(Tree): + def test_runs_the_workload_check(self): + ok, _ = self.quiet(chk.check_workloads, self.root, self.runner) self.assertTrue(ok) - self.assertEqual(len(self.calls), 3) - - def test_stale_expectations(self): - self.emitted = "different\n" - ok, output = self.quiet(chk.check_api_matrix, self.root, self.out, self.runner) - self.assertFalse(ok) - self.assertIn("hrr_api_matrix_expectations.h is stale", output) + self.assertEqual(Path(self.calls[0][1]).name, "check_matrix.py") + self.assertIn("--check-workloads", self.calls[0]) - def test_step_failure_stops(self): - self.rc["derive_manifest.py"] = 3 - ok, output = self.quiet(chk.check_api_matrix, self.root, self.out, self.runner) + def test_failure_is_reported(self): + self.rc["check_matrix.py"] = 2 + ok, output = self.quiet(chk.check_workloads, self.root, self.runner) self.assertFalse(ok) - self.assertEqual(len(self.calls), 1) - self.assertIn("derive_manifest.py failed (exit 3)", output) + self.assertIn("--check-workloads failed (exit 2)", output) class ResolveBaseTest(unittest.TestCase): diff --git a/projects/hrr/tests/scripts/test_hrr_ci_configure.py b/projects/hrr/tests/scripts/test_hrr_ci_configure.py index 139af11fb2b2..4efd60fd464e 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_configure.py @@ -80,8 +80,8 @@ FAMILIES = { "gfx90a": {}, "gfx94x": None, - "gfx950": {"multi_gpu_triggers": ["schedule"]}, - "gfx110x": {"triggers": ["schedule", "workflow_dispatch"]}, + "gfx950": {}, + "gfx110x": {"enabled": False}, "gfx1150": {}, "gfx125x": {"enabled": False}, } @@ -112,7 +112,7 @@ def config_data(**changes): def gpu_data(**changes): """The family-selection tests predate the simulator default.""" data = config_data(**changes) - data["backend"] = {"default": "gpu", "gpu": {"triggers": []}} + data["backend"] = {"default": "gpu"} return data @@ -135,20 +135,6 @@ def family_keys(decision): class RepoConfigTest(unittest.TestCase): - def test_only_gfx94x_runs_by_default(self): - """Every family stays listed, but only gfx94x runs unless asked for. - - The others stay in ci.yaml so the workflow_dispatch families input can - still request them; disabling is what keeps them off the default run. - ci:gfx labels are TheRock's and do not select an HRR family. - """ - self.assertEqual( - sorted(key for key, family in REPO.families.items() if family.enabled), - ["gfx94x"], - ) - for key, family in REPO.families.items(): - self.assertEqual(family.triggers, cfg.EVENTS, key) - def test_a_gfx_label_does_not_add_a_simulator_family(self): """ci:gfx is a TheRock label. It must not widen the rocJITsu matrix.""" data = copy.deepcopy(REPO_CONFIG) @@ -162,12 +148,6 @@ def test_a_gfx_label_does_not_add_a_simulator_family(self): self.assertEqual(family_keys(d), ["gfx950", "gfx125x"]) self.assertTrue(all(row["backend"] == "simulator" for row in d.families)) - def test_multi_gpu_runner_only_nightly(self): - for key, family in REPO.families.items(): - self.assertIn(family.multi_gpu_triggers, ((), ("schedule",)), key) - self.assertEqual(REPO.families["gfx950"].multi_gpu_triggers, ("schedule",)) - - class PathClassificationTest(unittest.TestCase): def test_playback_change_is_capture(self): self.assertTrue( @@ -231,17 +211,6 @@ def test_unknown_family_option(self): config_data(families=families), "families.gfx94x: unknown key runs_on" ) - def test_unknown_trigger(self): - families = {"gfx94x": {"triggers": ["nightly"]}} - self.assertRejected(config_data(families=families), "unknown trigger 'nightly'") - - def test_unknown_multi_gpu_trigger(self): - families = {"gfx950": {"multi_gpu_triggers": ["nightly"]}} - self.assertRejected( - config_data(families=families), - "families.gfx950.multi_gpu_triggers: unknown trigger", - ) - def test_enabled_must_be_a_bool(self): families = {"gfx94x": {"enabled": "no"}} self.assertRejected(config_data(families=families), "families.gfx94x.enabled") @@ -346,8 +315,7 @@ def test_cpu_only_pull_request(self): def test_capture_pull_request_runs_enabled_families(self): d = decide("pull_request", ["projects/clr/hipamd/src/hrr/hip_capture.cpp"]) self.assertTrue(d.run_gpu) - # gfx110x is not triggered by pull requests, gfx125x is disabled and - # gfx1150 has no runner. + # gfx110x and gfx125x are disabled and gfx1150 has no runner. self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950"]) self.assertEqual( d.families[1], @@ -363,10 +331,7 @@ def test_capture_pull_request_runs_enabled_families(self): self.assertEqual( d.skipped, [ - { - "family": "gfx110x", - "reason": "not run on pull_request (ci.yaml triggers)", - }, + {"family": "gfx110x", "reason": "disabled in ci.yaml"}, {"family": "gfx1150", "reason": cfg.NO_RUNNER}, {"family": "gfx125x", "reason": "disabled in ci.yaml"}, ], @@ -439,22 +404,9 @@ def test_unlisted_change_set_runs_everything(self): self.assertTrue(d.run_cpu) self.assertTrue(d.run_gpu) - def test_schedule_follows_triggers(self): + def test_schedule_runs_the_enabled_families(self): d = decide("schedule", []) - self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950", "gfx110x"]) - - def test_multi_gpu_runner_on_its_triggers_only(self): - nightly = {f["family"]: f["runner"] for f in decide("schedule", []).families} - self.assertEqual(nightly["gfx950"], "gpu-950-8") - pr = decide("pull_request", ["projects/hrr/playback/x.cpp"]) - self.assertEqual( - {f["family"]: f["runner"] for f in pr.families}["gfx950"], "gpu-950-1" - ) - - def test_runner_override(self): - config = cfg.parse_config(gpu_data(families={"gfx94x": {"runner": "my-gpu"}})) - d = decide("schedule", [], config=config) - self.assertEqual(d.families[0]["runner"], "my-gpu") + self.assertEqual(family_keys(d), ["gfx90a", "gfx94x", "gfx950"]) def test_dispatch_families_replace_the_selection(self): d = decide("workflow_dispatch", [], requested=["gfx125x", "gfx1150"]) @@ -475,30 +427,14 @@ def test_unknown_event(self): class SimulatorBackendTest(unittest.TestCase): def test_default_runs_no_real_gpu_job_on_any_event(self): - """Pull requests, pushes and the nightly run only rocJitsu jobs, on every - family that has a rocJitsu config; none queues on a GPU runner.""" + """Pull requests, pushes and the nightly run only rocJitsu jobs; none + queues on a GPU runner.""" for event in ("pull_request", "push", "schedule", "workflow_dispatch"): decision = decide(event, ["projects/hrr/playback/x.cpp"], config=REPO) self.assertTrue(decision.run_gpu, event) - self.assertEqual( - family_keys(decision), - [ - "gfx90a", - "gfx94x", - "gfx950", - "gfx110x", - "gfx1151", - "gfx120x", - "gfx125x", - ], - event, - ) self.assertEqual( {row["backend"] for row in decision.families}, {"simulator"}, event ) - self.assertEqual( - {row["runner"] for row in decision.families}, {"ubuntu-24.04"}, event - ) def test_every_default_simulator_family_has_a_config_file(self): root = Path(__file__).resolve().parents[4] @@ -553,17 +489,6 @@ def test_gpu_dispatch_selects_the_therock_runner(self): self.assertEqual(decision.families[0]["runner"], "gpu-94x-a") self.assertNotIn("gfx125x", family_keys(decision)) - def test_pull_request_gpu_triggers_select_real_runners(self): - data = config_data() - data["backend"]["gpu"]["triggers"] = ["pull_request"] - config = cfg.parse_config(data) - decision = decide( - "pull_request", ["projects/hrr/playback/x.cpp"], config=config - ) - self.assertEqual(family_keys(decision), ["gfx90a", "gfx94x", "gfx950"]) - self.assertTrue(all(row["backend"] == "gpu" for row in decision.families)) - self.assertNotIn("gfx125x", family_keys(decision)) - def test_simulator_rejects_a_requested_family_without_a_config(self): with self.assertRaises(cfg.ConfigError) as caught: decide("workflow_dispatch", [], requested=["gfx103x"], config=REPO) @@ -751,36 +676,6 @@ def test_write_outputs(self): self.assertIn("rocm_sdk_version=1.2.3\n", text) -class WorkflowPinTest(unittest.TestCase): - ROOT = Path(__file__).resolve().parents[4] - SHA = re.compile(r"(?:@|ref:\s*)([0-9a-f]{40})") - - def therock_shas(self, path: Path) -> list[str]: - lines = path.read_text(encoding="utf-8").splitlines() - found = [] - for index, line in enumerate(lines): - if "ROCm/TheRock" not in line: - continue - window = "\n".join(lines[index : index + 6]) - found.extend(self.SHA.findall(window)) - return found - - def test_hrr_workflows_use_the_multi_arch_therock_pin(self): - pinned = self.therock_shas( - self.ROOT / ".github/workflows/therock-multi-arch-ci.yml" - ) - self.assertTrue(pinned) - self.assertEqual(set(pinned), {pinned[0]}) - for name in ("hrr-ci.yml", "hrr-gpu.yml"): - path = self.ROOT / ".github/workflows" / name - text = path.read_text(encoding="utf-8") - self.assertNotIn("THEROCK_PIN", text, name) - self.assertNotIn("BUILD_IMAGE", text, name) - found = self.therock_shas(path) - self.assertTrue(found, name) - self.assertEqual(set(found), {pinned[0]}, name) - - class WorkflowPathsFilterTest(unittest.TestCase): ROOT = Path(__file__).resolve().parents[4] diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index a3c1711d5df5..c3bc285ecf3f 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -81,19 +81,18 @@ def test_rocjitsu_job_counts_as_the_family_integration_job(self): self.assertEqual(rpt.missing_families(families, rows), []) - def test_a_failed_rocjitsu_build_fails_the_summary_and_is_no_missing_family(self): + def test_a_failed_rocjitsu_build_leaves_its_families_missing(self): + # The simulator jobs skip, and a skipped matrix job keeps its + # unexpanded name, so the build's failure shows as missing families. rows = rpt.classify_jobs( [ {"name": "GPU / Build rocJitsu", "conclusion": "failure", "html_url": "u0"}, - {"name": "GPU / rocJitsu (gfx90a)", "conclusion": "skipped", "html_url": "u1"}, + {"name": "GPU / rocJitsu (${{ matrix.family }})", "conclusion": "skipped"}, ] ) - self.assertEqual( - [(r["kind"], r["name"], r["conclusion"]) for r in rows], - [("Build", "", "failure"), ("Integration", "gfx90a", "skipped")], - ) - self.assertEqual(rpt.missing_families([{"family": "gfx90a"}], rows), []) - self.assertFalse(rpt.overall_ok(rows, [])) + missing = rpt.missing_families([{"family": "gfx90a"}], rows) + self.assertEqual(missing, ["gfx90a"]) + self.assertFalse(rpt.overall_ok(rows, missing)) class OverallTest(unittest.TestCase): @@ -181,7 +180,6 @@ def test_run_39793856571_does_not_pass(self): {"name": "Unit (ubuntu-24.04)", "conclusion": "success"}, {"name": "Unit (windows-2022)", "conclusion": "success"}, {"name": "GPU / Build capture-enabled HIP and HRR", "conclusion": "success"}, - {"name": "GPU / Build rocJitsu", "status": "in_progress", "html_url": "u1"}, {"name": "GPU / rocJitsu (gfx950)", "status": "queued", "html_url": "u2"}, ] needs = { diff --git a/projects/hrr/tools/hrr_check_sources.py b/projects/hrr/tools/hrr_check_sources.py index 54d9a3dc9980..58a73b1762d0 100644 --- a/projects/hrr/tools/hrr_check_sources.py +++ b/projects/hrr/tools/hrr_check_sources.py @@ -1,23 +1,22 @@ # Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -"""Check the HRR sources that are generated from the HIP API headers. +"""Check the HRR sources against their base and the Catch2 sources. - generated Regenerate hrr_api_args.h and the capture and playback dispatch - sources and compare them with the checked-in copies. - api-compat Check that the regenerated API IDs stay compatible with the + api-compat Check that the checked-in API IDs stay compatible with the header on the base commit (IDs are part of the archive format). - api-matrix Re-derive the API class manifest and compare the generated - test expectations with the checked-in header. - all All three, reporting every failure. + workloads Read the API matrix tier workload lists against the Catch2 + sources. + all Both, reporting every failure. +That the checked-in generated files are current is tools/tests/ +test_generated_files_current.py, which the script tests job runs. Everything is written under --out; checked-in files are never modified. """ from __future__ import annotations import argparse -import difflib import re import subprocess import sys @@ -26,21 +25,9 @@ Runner = Callable[[Sequence[str]], int] -API_TRACE = "projects/clr/hipamd/include/hip/amd_detail/hip_api_trace.hpp" -PUBLIC_HEADER = "projects/hip/include/hip/hip_runtime_api.h" -MANUAL_CAPTURE = "projects/clr/hipamd/src/hrr/hip_capture.cpp" -GENERATOR = "projects/hrr/tools/gen_hrr_api_args.py" COMPAT_CHECKER = "projects/hrr/tools/check_hrr_api_compat.py" MATRIX_TOOLS = "projects/hrr/tools/api-matrix" API_ARGS_HEADER = "projects/hrr/include/hrr/hrr_api_args.h" -MATRIX_EXPECTATIONS = "projects/hrr/tests/integration/hrr_api_matrix_expectations.h" - -# Checked-in file -> name of the regenerated copy under --out. -GENERATED_FILES = { - API_ARGS_HEADER: "include/hrr/hrr_api_args.h", - "projects/clr/hipamd/src/hrr/hip_capture_generated.cpp": "hip_capture_generated.cpp", - "projects/hrr/playback/hip_playback_generated.cpp": "hip_playback_generated.cpp", -} def run(cmd: Sequence[str]) -> int: @@ -52,50 +39,6 @@ def error(message: str) -> None: print(f"::error::{message}", flush=True) -def diff_files(expected: Path, actual: Path) -> list[str]: - """Unified diff of two text files, ignoring line-ending differences.""" - a = expected.read_text(encoding="utf-8").splitlines() - b = actual.read_text(encoding="utf-8").splitlines() - return list( - difflib.unified_diff(a, b, fromfile=str(expected), tofile=str(actual), lineterm="") - ) - - -def check_generated(root: Path, out: Path, runner: Runner = run) -> bool: - (out / "include" / "hrr").mkdir(parents=True, exist_ok=True) - rc = runner( - [ - sys.executable, str(root / GENERATOR), - "--input", str(root / API_TRACE), - "--public-header", str(root / PUBLIC_HEADER), - "--output-header", str(out / GENERATED_FILES[API_ARGS_HEADER]), - "--output-capture", str(out / "hip_capture_generated.cpp"), - "--output-playback", str(out / "hip_playback_generated.cpp"), - "--check-hrr-coverage", - "--manual-capture-source", str(root / MANUAL_CAPTURE), - "--silent", - ] - ) - if rc != 0: - error(f"{GENERATOR} failed (exit {rc})") - return False - - stale = [] - for tracked, generated in GENERATED_FILES.items(): - lines = diff_files(root / tracked, out / generated) - if lines: - print("\n".join(lines)) - stale.append(tracked) - if stale: - error( - "checked-in HRR generated sources are stale: " - + ", ".join(stale) - + f". Regenerate them with {GENERATOR} --check-hrr-coverage." - ) - return False - return True - - def base_header(root: Path, base_sha: str) -> tuple[str | None, str | None]: """hrr_api_args.h at base_sha. @@ -158,10 +101,7 @@ def check_api_compat( runner: Runner = run, read_base: Callable[[Path, str], tuple[str | None, str | None]] = base_header, ) -> bool: - current = out / GENERATED_FILES[API_ARGS_HEADER] - if not current.is_file(): - error("api-compat needs the regenerated header; run the generated check first") - return False + current = root / API_ARGS_HEADER text, failure = read_base(root, base_sha) if failure: error(failure) @@ -185,39 +125,18 @@ def check_api_compat( return True -def check_api_matrix(root: Path, out: Path, runner: Runner = run) -> bool: - tools = root / MATRIX_TOOLS - matrix_out = out / "api-matrix" - matrix_out.mkdir(parents=True, exist_ok=True) - manifest = matrix_out / "api_classes.json" - emitted = matrix_out / "hrr_api_matrix_expectations.h" - steps = [ - [sys.executable, str(tools / "derive_manifest.py"), "--quiet", "--output", str(manifest)], - [sys.executable, str(tools / "check_matrix.py"), "--check-workloads"], - [ - sys.executable, str(tools / "check_matrix.py"), - "--manifest", str(manifest), "--emit-cxx", str(emitted), - ], - ] - for step in steps: - rc = runner(step) - if rc != 0: - error(f"{Path(step[1]).name} failed (exit {rc})") - return False - lines = diff_files(root / MATRIX_EXPECTATIONS, emitted) - if lines: - print("\n".join(lines)) - error( - f"{MATRIX_EXPECTATIONS} is stale; run derive_manifest.py, then " - f"check_matrix.py --emit-cxx, in {MATRIX_TOOLS}" - ) - return False - return True +def check_workloads(root: Path, runner: Runner = run) -> bool: + rc = runner( + [sys.executable, str(root / MATRIX_TOOLS / "check_matrix.py"), "--check-workloads"] + ) + if rc != 0: + error(f"check_matrix.py --check-workloads failed (exit {rc})") + return rc == 0 def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("check", choices=("generated", "api-compat", "api-matrix", "all")) + parser.add_argument("check", choices=("api-compat", "workloads", "all")) parser.add_argument("--root", type=Path, default=Path.cwd()) parser.add_argument("--out", type=Path, default=Path("build/hrr-codegen")) parser.add_argument("--base-sha", default="") @@ -228,12 +147,10 @@ def main(argv: list[str] | None = None) -> int: out.mkdir(parents=True, exist_ok=True) ok = True - if args.check in ("generated", "all"): - ok &= check_generated(root, out) if args.check in ("api-compat", "all"): ok &= check_api_compat(root, out, resolve_base(root, args.base_sha)) - if args.check in ("api-matrix", "all"): - ok &= check_api_matrix(root, out) + if args.check in ("workloads", "all"): + ok &= check_workloads(root) return 0 if ok else 1 From 17e659c2a487b5b07274fda4507a44762bbe95ff Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 26 Oct 2026 23:17:29 +0100 Subject: [PATCH 67/89] HRR CI: pin Python requirements in projects/hrr, condense the workflow header, date the image pin Move the PyYAML and pytest pins into projects/hrr/requirements.txt and requirements-test.txt and install from them in every job. State at the top of hrr-ci.yml how to run the capture suite (ci:hrr-capture label) and what is expected to happen once TheRock CI builds and tests HRR. --- .github/workflows/hrr-ci.yml | 40 ++++++++++++++++-------------- .github/workflows/hrr-gpu.yml | 3 ++- projects/hrr/requirements-test.txt | 3 +++ projects/hrr/requirements.txt | 3 +++ projects/hrr/tests/README.md | 3 ++- 5 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 projects/hrr/requirements-test.txt create mode 100644 projects/hrr/requirements.txt diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index 9cb070d9254d..d69f62aa947b 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -1,21 +1,22 @@ -# HRR CI: API-ID and workload checks, the HRR helper-script tests and the host -# unit suite, plus the GPU capture/replay suite (hrr-gpu.yml) for changes that -# touch capture or replay. projects/hrr/tests/scripts/hrr_ci_configure.py -# decides what runs from projects/hrr/tests/config/ci.yaml; "HRR CI Summary" -# reports every platform and GPU family in one table. +# HRR CI: API-ID and workload checks, HRR script tests and the host unit suite, +# plus the GPU capture/replay suite (hrr-gpu.yml). +# projects/hrr/tests/config/ci.yaml and hrr_ci_configure.py decide what runs. +# +# To run the capture/GPU suite on a pull request, add the `ci:hrr-capture` label. +# +# Interim: once TheRock CI builds and tests HRR (TheRock PR #8965 adds the +# core-hrr artifact), the build and test jobs here are expected to go. This +# workflow would keep only the checks TheRock does not run (API-ID and workload +# checks, helper-script tests) and the rocJitsu simulator jobs until TheRock has +# a capture variant. Not a committed plan. name: HRR CI -# labeled stays in the types so that adding ci:hrr-capture can still start a -# run; hrr_ci_configure.py decides from the label itself -# whether anything runs, and the summary job reports the verdict of the run -# that actually tested the commit (see `pass_through_ok`). A label that -# changes nothing therefore cannot mask a failing run on the same commit. -# -# The paths filters are the union of the paths.cpu, paths.capture and -# paths.suite lists of projects/hrr/tests/config/ci.yaml, less paths.excluded, -# so a pull request that touches no HRR path starts no run. -# test_hrr_ci_configure.py fails when the two disagree. +# The paths filters are the union of paths.cpu, paths.capture and paths.suite in +# ci.yaml, less paths.excluded; test_hrr_ci_configure.py fails if they differ. +# `labeled` stays in the types so the label can start a run; the summary job +# reports the run that tested the commit, so an irrelevant label cannot mask a +# failure (see `pass_through_ok`). on: pull_request: types: [opened, synchronize, reopened, labeled] @@ -112,6 +113,7 @@ jobs: projects/hrr/tests/scripts/hrr_ci_configure.py projects/hrr/tests/config/ci.yaml projects/hrr/tests/config/integration.yaml + projects/hrr/requirements.txt sparse-checkout-cone-mode: false - name: Checkout TheRock family matrix @@ -144,7 +146,7 @@ jobs: python-version: "3.12" - name: Install PyYAML - run: python -m pip install --no-cache-dir PyYAML==6.0.2 + run: python -m pip install --no-cache-dir -r projects/hrr/requirements.txt - name: Decide id: decide @@ -196,7 +198,7 @@ jobs: python-version: "3.12" - name: Install PyYAML - run: python -m pip install --no-cache-dir PyYAML==6.0.2 + run: python -m pip install --no-cache-dir -r projects/hrr/requirements.txt - name: Check API IDs and the API matrix workloads env: @@ -231,7 +233,7 @@ jobs: python-version: "3.12" - name: Install test requirements - run: python -m pip install --no-cache-dir PyYAML==6.0.2 pytest==8.4.2 + run: python -m pip install --no-cache-dir -r projects/hrr/requirements-test.txt - name: Lint the suite configs run: >- @@ -281,7 +283,7 @@ jobs: env: ROCM_SDK_VERSION: ${{ needs.configure.outputs.rocm_sdk_version }} run: | - python -m pip install --no-cache-dir PyYAML==6.0.2 + python -m pip install --no-cache-dir -r projects/hrr/requirements.txt python -m pip install --no-cache-dir --index-url "${ROCM_INDEX_URL}" "rocm[devel]==${ROCM_SDK_VERSION}" rocm-sdk init echo "ROCM_SDK=$(rocm-sdk path --root)" >> "${GITHUB_ENV}" diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 575616f88f61..37812fc0960d 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -179,6 +179,7 @@ jobs: # workspace path is this container's workspace, so the paths compiled into # the test binaries resolve without a symlink on the runner. container: + # Digest pinned 2026-10-13. image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98 options: --user 0:0 steps: @@ -223,7 +224,7 @@ jobs: set -euo pipefail python -m venv "${RUNNER_TEMP}/hrr-venv" "${RUNNER_TEMP}/hrr-venv/bin/python" -m pip install --no-cache-dir \ - -r therock/requirements.txt 'PyYAML==6.0.2' + -r therock/requirements.txt -r projects/hrr/requirements.txt echo "${RUNNER_TEMP}/hrr-venv/bin" >> "${GITHUB_PATH}" - name: Download rocJitsu diff --git a/projects/hrr/requirements-test.txt b/projects/hrr/requirements-test.txt new file mode 100644 index 000000000000..c704a979f4d7 --- /dev/null +++ b/projects/hrr/requirements-test.txt @@ -0,0 +1,3 @@ +# Adds what the HRR script tests (projects/hrr/tests/scripts, projects/hrr/tools/tests) need. +-r requirements.txt +pytest==8.4.2 diff --git a/projects/hrr/requirements.txt b/projects/hrr/requirements.txt new file mode 100644 index 000000000000..ec5cb15e28a6 --- /dev/null +++ b/projects/hrr/requirements.txt @@ -0,0 +1,3 @@ +# Python requirements of the HRR helper scripts, the api-matrix tools and the +# CI workflows (.github/workflows/hrr-ci.yml, hrr-gpu.yml). Change a pin here. +PyYAML==6.0.2 diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index 1b6797ffd5f7..358161f8adf1 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -134,7 +134,8 @@ The host unit binary runs once in either command. ## Testing the Python helpers `scripts/` holds both the helpers and their tests. The tests need only Python and -PyYAML (used by the api-matrix tools), not ROCm: +PyYAML (used by the api-matrix tools), not ROCm. Install them with `python3 -m pip install -r projects/hrr/requirements-test.txt` +(`requirements.txt` has only the runtime pins): ```bash python3 -m unittest discover -s projects/hrr/tests/scripts -p 'test_*.py' -v From b8f13188d79603f3e17a3ab11a84116c5b621375 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 27 Oct 2026 13:48:10 +0100 Subject: [PATCH 68/89] fix(hrr): address review of the CI workflow (AIRUNTIME-3071) Name the CPU job for what it checks (API IDs and workloads), drop the duplicate PyYAML requirements file from the rocJitsu jobs, and test that a gpu dispatch with a skipped Build rocJitsu passes. --- .github/workflows/hrr-ci.yml | 6 +-- .github/workflows/hrr-gpu.yml | 3 +- projects/hrr/requirements.txt | 3 +- projects/hrr/tests/README.md | 2 +- projects/hrr/tests/config/ci.yaml | 4 +- .../hrr/tests/scripts/hrr_ci_configure.py | 2 +- projects/hrr/tests/scripts/hrr_ci_report.py | 4 +- .../hrr/tests/scripts/test_hrr_ci_report.py | 41 +++++++++++++------ 8 files changed, 41 insertions(+), 24 deletions(-) diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml index d69f62aa947b..4099414f4242 100644 --- a/.github/workflows/hrr-ci.yml +++ b/.github/workflows/hrr-ci.yml @@ -173,8 +173,8 @@ jobs: --backend "${BACKEND}" --therock-dir therock - codegen: - name: Generated sources + api-checks: + name: API IDs and workloads needs: configure if: ${{ needs.configure.outputs.run_cpu == 'true' }} runs-on: ubuntu-24.04 @@ -378,7 +378,7 @@ jobs: summary: name: HRR CI Summary - needs: [configure, codegen, scripts, unit, gpu] + needs: [configure, api-checks, scripts, unit, gpu] # Also when configure failed: then nothing else ran, and this job fails. if: >- ${{ always() && (needs.configure.result != 'success' diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 37812fc0960d..70f4b3d4f4d8 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -223,8 +223,7 @@ jobs: run: | set -euo pipefail python -m venv "${RUNNER_TEMP}/hrr-venv" - "${RUNNER_TEMP}/hrr-venv/bin/python" -m pip install --no-cache-dir \ - -r therock/requirements.txt -r projects/hrr/requirements.txt + "${RUNNER_TEMP}/hrr-venv/bin/python" -m pip install --no-cache-dir -r therock/requirements.txt echo "${RUNNER_TEMP}/hrr-venv/bin" >> "${GITHUB_PATH}" - name: Download rocJitsu diff --git a/projects/hrr/requirements.txt b/projects/hrr/requirements.txt index ec5cb15e28a6..6f1a3b3dfab6 100644 --- a/projects/hrr/requirements.txt +++ b/projects/hrr/requirements.txt @@ -1,3 +1,4 @@ # Python requirements of the HRR helper scripts, the api-matrix tools and the -# CI workflows (.github/workflows/hrr-ci.yml, hrr-gpu.yml). Change a pin here. +# CPU jobs of .github/workflows/hrr-ci.yml. hrr-gpu.yml takes PyYAML from +# TheRock's requirements.txt instead; keep both pins equal. PyYAML==6.0.2 diff --git a/projects/hrr/tests/README.md b/projects/hrr/tests/README.md index 358161f8adf1..36ec1c6dfdd1 100644 --- a/projects/hrr/tests/README.md +++ b/projects/hrr/tests/README.md @@ -152,7 +152,7 @@ the generator, and the [api-matrix](../tools/api-matrix/README.md) scripts. [`config/ci.yaml`](config/ci.yaml) is the policy the configure job of `hrr-ci.yml` applies (`scripts/hrr_ci_configure.py`): -- `paths.cpu`: a change under one of these runs the generated-source, script and unit +- `paths.cpu`: a change under one of these runs the API check, script and unit jobs. `paths.capture` also runs the capture-enabled build and the GPU suite. `paths.excluded` counts for neither. The `paths` filter of `hrr-ci.yml` is the same lists, so a pull request that touches none of them starts no HRR CI run; diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 4b303281bc45..6b2d53df547c 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -10,7 +10,7 @@ rocm_sdk_version: "10.1.0a20260822" # Path prefixes, relative to the repository root. A path ending in "/" matches # everything under it; any other path matches only itself. paths: - # A change under any of these runs the generated-source, script and unit jobs. + # A change under any of these runs the API check, script and unit jobs. # Changes under emulation/rocjitsu, runtimes/api-headers and # shared/machine-readable-isa are not path triggers; the nightly schedule and # workflow_dispatch cover them. @@ -24,7 +24,7 @@ paths: - .github/workflows/hrr-gpu.yml # A change under any of these also runs the capture-enabled build and the GPU # suite. hip_runtime_api.h is deliberately absent: almost every HIP API change - # touches it, and the generated-source job already checks HRR against it. + # touches it, and the API check job already checks HRR against it. # amdhip.def.in is absent for the same reason (cpu only). capture: - projects/clr/hipamd/src/hrr/ diff --git a/projects/hrr/tests/scripts/hrr_ci_configure.py b/projects/hrr/tests/scripts/hrr_ci_configure.py index ce9669bac628..898b6ea409d0 100644 --- a/projects/hrr/tests/scripts/hrr_ci_configure.py +++ b/projects/hrr/tests/scripts/hrr_ci_configure.py @@ -5,7 +5,7 @@ Writes these GitHub Actions outputs: - run_cpu "true" when the generated-source, script and unit jobs run. + run_cpu "true" when the API check, script and unit jobs run. run_gpu "true" when the capture-enabled build and the GPU suite run. families JSON list of {family, runner, backend, arch, config}. simulator_families diff --git a/projects/hrr/tests/scripts/hrr_ci_report.py b/projects/hrr/tests/scripts/hrr_ci_report.py index aaf4e5ef064f..a92f5558a18a 100644 --- a/projects/hrr/tests/scripts/hrr_ci_report.py +++ b/projects/hrr/tests/scripts/hrr_ci_report.py @@ -36,7 +36,7 @@ import summarize_junit JOB_PATTERNS = ( - ("Generated sources", re.compile(r"^Generated sources$")), + ("API checks", re.compile(r"^API IDs and workloads$")), ("Script tests", re.compile(r"^HRR script tests$")), ("Unit", re.compile(r"(?:^|/ )Unit \((?P[^)]+)\)$")), ("Build", re.compile(r"(?:^|/ )Build capture-enabled HIP and HRR$")), @@ -46,7 +46,7 @@ # Anything else (queued, in_progress, waiting, unknown) has not finished. TERMINAL_OK = {"success", "neutral"} BAD_NEEDS = {"failure", "cancelled"} -CPU_NEEDS = ("codegen", "scripts", "unit") +CPU_NEEDS = ("api-checks", "scripts", "unit") def classify_jobs(jobs: list[dict]) -> list[dict]: diff --git a/projects/hrr/tests/scripts/test_hrr_ci_report.py b/projects/hrr/tests/scripts/test_hrr_ci_report.py index c3bc285ecf3f..e974700de2d9 100644 --- a/projects/hrr/tests/scripts/test_hrr_ci_report.py +++ b/projects/hrr/tests/scripts/test_hrr_ci_report.py @@ -20,7 +20,7 @@ JOBS = [ {"name": "Configure", "conclusion": "success", "html_url": "u0"}, - {"name": "Generated sources", "conclusion": "success", "html_url": "c0"}, + {"name": "API IDs and workloads", "conclusion": "success", "html_url": "c0"}, {"name": "Unit (ubuntu-24.04)", "conclusion": "success", "html_url": "u1"}, {"name": "Unit (windows-2022)", "conclusion": "failure", "html_url": "u2"}, {"name": "gpu / Build capture-enabled HIP and HRR", "conclusion": "success", "html_url": "u3"}, @@ -37,7 +37,7 @@ def test_picks_hrr_jobs(self): self.assertEqual( [(r["kind"], r["name"]) for r in rows], [ - ("Generated sources", ""), + ("API checks", ""), ("Unit", "ubuntu-24.04"), ("Unit", "windows-2022"), ("Build", ""), @@ -95,6 +95,23 @@ def test_a_failed_rocjitsu_build_leaves_its_families_missing(self): self.assertFalse(rpt.overall_ok(rows, missing)) +class GpuDispatchTest(unittest.TestCase): + def test_skipped_rocjitsu_build_does_not_fail_a_gpu_dispatch(self): + # A gpu dispatch runs the real-GPU jobs; "Build rocJitsu" is skipped. + # It is no HRR job row, so the green Integration rows must pass. + rows = rpt.classify_jobs( + [ + {"name": "GPU / Build rocJitsu", "conclusion": "skipped"}, + {"name": "GPU / Integration (gfx94x)", "conclusion": "success", "html_url": "u1"}, + {"name": "GPU / Integration (gfx950)", "conclusion": "success", "html_url": "u2"}, + ] + ) + families = [{"family": "gfx94x"}, {"family": "gfx950"}] + missing = rpt.missing_families(families, rows) + self.assertEqual(missing, []) + self.assertTrue(rpt.overall_ok(rows, missing)) + + class OverallTest(unittest.TestCase): def test_failure_or_cancel_fails(self): self.assertFalse(rpt.overall_ok(rpt.classify_jobs(JOBS), [])) @@ -175,7 +192,7 @@ def test_run_39793856571_does_not_pass(self): JUnit file already contained Unit_HRR_PinnedHostFork. All three must fail the summary; any one of them is enough.""" jobs = [ - {"name": "Generated sources", "conclusion": "success", "html_url": "u0"}, + {"name": "API IDs and workloads", "conclusion": "success", "html_url": "u0"}, {"name": "HRR script tests", "conclusion": "success"}, {"name": "Unit (ubuntu-24.04)", "conclusion": "success"}, {"name": "Unit (windows-2022)", "conclusion": "success"}, @@ -184,7 +201,7 @@ def test_run_39793856571_does_not_pass(self): ] needs = { "configure": {"result": "success"}, - "codegen": {"result": "success"}, + "api-checks": {"result": "success"}, "scripts": {"result": "success"}, "unit": {"result": "success"}, "gpu": {"result": "failure"}, @@ -212,13 +229,13 @@ def test_run_39793856571_does_not_pass(self): def test_junit_failure_fails_when_jobs_and_needs_succeeded(self): jobs = [ - {"name": "Generated sources", "conclusion": "success"}, + {"name": "API IDs and workloads", "conclusion": "success"}, {"name": "HRR script tests", "conclusion": "success"}, {"name": "Unit (ubuntu-24.04)", "conclusion": "success"}, {"name": "GPU / rocJitsu (gfx94x)", "conclusion": "success"}, ] needs = {name: {"result": "success"} for name in ( - "configure", "codegen", "scripts", "unit", "gpu")} + "configure", "api-checks", "scripts", "unit", "gpu")} failed = ( '' '' @@ -239,7 +256,7 @@ def test_junit_failure_fails_when_jobs_and_needs_succeeded(self): def test_a_selected_cpu_job_that_was_skipped_fails(self): needs = { "configure": {"result": "success"}, - "codegen": {"result": "skipped"}, + "api-checks": {"result": "skipped"}, "scripts": {"result": "success"}, "unit": {"result": "success"}, "gpu": {"result": "skipped"}, @@ -250,18 +267,18 @@ def test_a_selected_cpu_job_that_was_skipped_fails(self): contextlib.redirect_stdout(io.StringIO()) as out: rc = rpt.main(["--results-dir", tmp, "--needs-json", json.dumps(needs)]) self.assertEqual(rc, 1) - self.assertIn("codegen is skipped", out.getvalue()) + self.assertIn("api-checks is skipped", out.getvalue()) def test_green_needs_jobs_and_junit_pass(self): jobs = [ - {"name": "Generated sources", "conclusion": "success"}, + {"name": "API IDs and workloads", "conclusion": "success"}, {"name": "HRR script tests", "conclusion": "success"}, {"name": "Unit (ubuntu-24.04)", "conclusion": "success"}, {"name": "Unit (windows-2022)", "conclusion": "success"}, {"name": "GPU / rocJitsu (gfx94x)", "conclusion": "success"}, ] needs = {name: {"result": "success"} for name in ( - "configure", "codegen", "scripts", "unit", "gpu")} + "configure", "api-checks", "scripts", "unit", "gpu")} passed = ( '' '' @@ -312,7 +329,7 @@ class PassThroughTest(unittest.TestCase): # What a run whose configure selected nothing lists: its jobs, skipped. LABEL_ONLY = [ {"name": "Configure", "conclusion": "success", "html_url": "u0"}, - {"name": "Generated sources", "conclusion": "skipped", "html_url": "k1"}, + {"name": "API IDs and workloads", "conclusion": "skipped", "html_url": "k1"}, {"name": "HRR script tests", "conclusion": "skipped", "html_url": "k2"}, {"name": "Unit (${{ matrix.os }})", "conclusion": "skipped", "html_url": "k3"}, {"name": "GPU", "conclusion": "skipped", "html_url": "k4"}, @@ -348,7 +365,7 @@ def test_delivers_the_earlier_run_jobs(self): rc, text = self.run_main([(99, 1, "2026-10-15T10:00:00Z", self.GREEN)]) self.assertEqual(rc, 0, text) self.assertIn("[run 99 attempt 1](r99)", text) - self.assertIn("| Generated sources | — | [success](c0) |", text) + self.assertIn("| API checks | — | [success](c0) |", text) def test_fails_when_the_earlier_run_failed(self): rc, text = self.run_main([(99, 1, "2026-10-15T10:00:00Z", JOBS + [self.SUMMARY_RED])]) From 79a6f28ce0aeefc266c57fb4bec98ebf1e3060e7 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 27 Oct 2026 15:26:56 +0100 Subject: [PATCH 69/89] test(hrr): run the generator's --check-hrr-coverage in the generated-files test (AIRUNTIME-2871) CLR_HRR_CODEGEN_CHECK is off by default, so no CI job ran the coverage check once hrr_check_sources.py stopped (9d0a4a7454). Add the flag to the regeneration in test_generated_files_current.py, and a test that the check fails when the manual capture source lacks its shims. --- .../tests/test_generated_files_current.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/projects/hrr/tools/tests/test_generated_files_current.py b/projects/hrr/tools/tests/test_generated_files_current.py index d386874a0737..202aa873838a 100644 --- a/projects/hrr/tools/tests/test_generated_files_current.py +++ b/projects/hrr/tools/tests/test_generated_files_current.py @@ -11,7 +11,10 @@ API matrix and the dispatch tables disagree about what an API does at replay. This regenerates each into a scratch directory and compares it with the -checked-in copy. Nothing is written into the tree. The matrix half needs PyYAML, +checked-in copy, with the generator's --check-hrr-coverage on: every API outside +MANUAL_CAPTURE_APIS must have a parseable dispatch slot and every manual one its +hand-written shim in hip_capture.cpp. The CLR build runs that check only with +CLR_HRR_CODEGEN_CHECK, which is off by default. Nothing is written into the tree. The matrix half needs PyYAML, as the matrix tools do, and is skipped without it. Run: python3 -m unittest discover -s projects/hrr/tools/tests @@ -58,7 +61,7 @@ class GeneratedFilesCurrent(unittest.TestCase): def test_generator_outputs_match_the_checked_in_files(self): with tempfile.TemporaryDirectory() as tmp: out = Path(tmp) - _run([TOOLS / "gen_hrr_api_args.py", "--silent", + _run([TOOLS / "gen_hrr_api_args.py", "--silent", "--check-hrr-coverage", "--output-header", out / "hrr_api_args.h", "--output-capture", out / "capture.cpp", "--output-playback", out / "playback.cpp"]) @@ -73,6 +76,19 @@ def test_generator_outputs_match_the_checked_in_files(self): ) if d] self.assertEqual(stale, [], "\n\n".join(stale)) + def test_coverage_check_rejects_a_missing_manual_capture_shim(self): + # The check above only means something if --check-hrr-coverage can fail. + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + empty = out / "hip_capture.cpp" + empty.write_text("") + with self.assertRaises(RuntimeError): + _run([TOOLS / "gen_hrr_api_args.py", "--silent", "--check-hrr-coverage", + "--manual-capture-source", empty, + "--output-header", out / "hrr_api_args.h", + "--output-capture", out / "capture.cpp", + "--output-playback", out / "playback.cpp"]) + @unittest.skipUnless(HAVE_YAML, "PyYAML is required for the matrix tools") def test_api_matrix_header_matches_the_derived_manifest(self): with tempfile.TemporaryDirectory() as tmp: From ff8c18c5838b31acaf25470ea0f23a3061f5deef Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 28 Oct 2026 13:24:14 +0100 Subject: [PATCH 70/89] docs(hrr): say which CI job checks hip_runtime_api.h (AIRUNTIME-2871) The comment in ci.yaml credited the API check job, which compares API IDs and workloads and never reads the header. The script tests job regenerates HRR from it (test_generated_files_current.py). --- projects/hrr/tests/config/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/config/ci.yaml b/projects/hrr/tests/config/ci.yaml index 6b2d53df547c..53403328af0f 100644 --- a/projects/hrr/tests/config/ci.yaml +++ b/projects/hrr/tests/config/ci.yaml @@ -24,7 +24,8 @@ paths: - .github/workflows/hrr-gpu.yml # A change under any of these also runs the capture-enabled build and the GPU # suite. hip_runtime_api.h is deliberately absent: almost every HIP API change - # touches it, and the API check job already checks HRR against it. + # touches it, and the script tests job (test_generated_files_current.py) + # already regenerates HRR from it and compares. # amdhip.def.in is absent for the same reason (cpu only). capture: - projects/clr/hipamd/src/hrr/ From 3d5d204a6108303517a57c145a3f9a55cf42d0e2 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 28 Oct 2026 14:29:48 +0100 Subject: [PATCH 71/89] test(hrr): list new develop cases in handler and integration configs (AIRUNTIME-2871) Merging origin/develop added Unit_HRR_ExtMemcpyBatchRoundtrip and Unit_HRR_ExtMemcpyBatch_Direct (integration), and replaced Unit_HRR_HostBudget_LandingBufferChargesGrowthOnly with Unit_HRR_HostBudget_LandingBufferChargesWholeBlock and ..._PointerSurvivesGrowth (handler). Regenerated with hrr_test_config.py check --update against binaries built from the merge; listing needs no GPU. unit.yaml already matched (82 cases). --- projects/hrr/tests/config/handler.yaml | 3 ++- projects/hrr/tests/config/integration.yaml | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml index 418a1d038401..f8616c3556b8 100644 --- a/projects/hrr/tests/config/handler.yaml +++ b/projects/hrr/tests/config/handler.yaml @@ -39,7 +39,8 @@ cases: Unit_HRR_HostBudget_DispatchFreesReplacedHostAllocation: {} Unit_HRR_HostBudget_DispatchTableUsesTheChargingHandlers: {} Unit_HRR_HostBudget_HostAllocationHandlersRefuseOverBudget: {} - Unit_HRR_HostBudget_LandingBufferChargesGrowthOnly: {} + Unit_HRR_HostBudget_LandingBufferChargesWholeBlock: {} + Unit_HRR_HostBudget_LandingBufferPointerSurvivesGrowth: {} Unit_HRR_Query_CompletingQueryReturnsSuccess: {} Unit_HRR_Query_DefaultHasNoAttemptBound: {} Unit_HRR_Query_NeverCompletingQueryEndsTheReplay: {} diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 7c3b30dc77c7..beb0b47b897d 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -159,6 +159,8 @@ cases: Unit_HRR_EmbeddedPtrStruct_Direct: {} Unit_HRR_ExtMallocRoundtrip: {} Unit_HRR_ExtMalloc_Direct: {} + Unit_HRR_ExtMemcpyBatchRoundtrip: {} + Unit_HRR_ExtMemcpyBatch_Direct: {} Unit_HRR_ExtModuleLaunchKernelRoundtrip: {} Unit_HRR_ExtModuleLaunchKernel_Direct: {} Unit_HRR_FailedBlobClose_Direct: {} From 9dc27a6cdba82dc19dd768a2fbea8f81e5f48781 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Wed, 28 Oct 2026 16:18:25 +0100 Subject: [PATCH 72/89] test(hrr): skip ExtMemcpyBatchRoundtrip under rocJitsu on gfx94x/gfx95x/gfx125x (AIRUNTIME-2871) hipExtMemcpyBatchAsync with hipMemcpyFlagExtOpSwap makes ROCr emit SDMA COPY sub-opcode 9; rocJitsu's decoder rejects it and the stream never syncs, so the case runs into run_catch2's 1200 s timeout. Disabled for emulator runs only, tracked in #14464. Unit_HRR_ExtMemcpyBatch_Direct is a direct workload (not selected by CI, cannot carry skip rules). --- projects/hrr/tests/config/integration.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index beb0b47b897d..7bf6a75e5ce0 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -159,7 +159,12 @@ cases: Unit_HRR_EmbeddedPtrStruct_Direct: {} Unit_HRR_ExtMallocRoundtrip: {} Unit_HRR_ExtMalloc_Direct: {} - Unit_HRR_ExtMemcpyBatchRoundtrip: {} + Unit_HRR_ExtMemcpyBatchRoundtrip: + skip: + - when: {arch: ["gfx94*", "gfx95*", "gfx125*"], emulator: true} + kind: disabled + reason: "hipExtMemcpyBatchAsync with hipMemcpyFlagExtOpSwap hangs under rocJitsu: the SDMA decoder rejects COPY_LINEAR_SWAP (sub-opcode 9), the ring stops before the fence and the stream never syncs" + issue: "https://github.com/ROCm/rocm-systems/issues/14464" Unit_HRR_ExtMemcpyBatch_Direct: {} Unit_HRR_ExtModuleLaunchKernelRoundtrip: {} Unit_HRR_ExtModuleLaunchKernel_Direct: {} From bc06d127504b11846ba6fd3f136c0a0b9b7b0388 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 29 Oct 2026 19:05:51 +0100 Subject: [PATCH 73/89] test(hrr): list new develop cases in unit and handler configs (AIRUNTIME-2871) Develop gained cases from #14159 and #14157, so the CI step "Check the compiled case list" failed on both Unit jobs. unit.yaml: +9 (Unit_HRR_Limits_* x8, Unit_HRR_Recovery_RepairRootHonoursMaxHostBytes), 82 -> 91. handler.yaml: +5 (Unit_HRR_Memcpy3DArray_* x4, Unit_HRR_Memcpy3DBatchArray_BadOperandIsRefused), 32 -> 37. integration.yaml already matched (294). Names added by hand-merging the output of hrr_test_config.py check --update, so existing comments and formatting are kept. The case set changed; listing needs no GPU. --- projects/hrr/tests/config/handler.yaml | 5 +++++ projects/hrr/tests/config/unit.yaml | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml index f8616c3556b8..aceb199d3c6b 100644 --- a/projects/hrr/tests/config/handler.yaml +++ b/projects/hrr/tests/config/handler.yaml @@ -41,6 +41,11 @@ cases: Unit_HRR_HostBudget_HostAllocationHandlersRefuseOverBudget: {} Unit_HRR_HostBudget_LandingBufferChargesWholeBlock: {} Unit_HRR_HostBudget_LandingBufferPointerSurvivesGrowth: {} + Unit_HRR_Memcpy3DArray_FreedHandleIsForgotten: {} + Unit_HRR_Memcpy3DArray_HostCopiesAreNotIssued: {} + Unit_HRR_Memcpy3DArray_TranslatesRecordedHandles: {} + Unit_HRR_Memcpy3DArray_UnknownHandleIsRefused: {} + Unit_HRR_Memcpy3DBatchArray_BadOperandIsRefused: {} Unit_HRR_Query_CompletingQueryReturnsSuccess: {} Unit_HRR_Query_DefaultHasNoAttemptBound: {} Unit_HRR_Query_NeverCompletingQueryEndsTheReplay: {} diff --git a/projects/hrr/tests/config/unit.yaml b/projects/hrr/tests/config/unit.yaml index 56acfe32827b..cf6d79ef6c9e 100644 --- a/projects/hrr/tests/config/unit.yaml +++ b/projects/hrr/tests/config/unit.yaml @@ -40,18 +40,26 @@ cases: Unit_HRR_Limits_FatalFlagEndsWait: {} Unit_HRR_Limits_FindDuplicateSeq: {} Unit_HRR_Limits_FindSeqGap: {} + Unit_HRR_Limits_LoadByteBudget: {} + Unit_HRR_Limits_LoadChargesBlobAndCodeObjectMaps: {} + Unit_HRR_Limits_LoadChargesEventVectorAndSort: {} + Unit_HRR_Limits_LoadChargesParsedKernelExpansion: {} + Unit_HRR_Limits_LoadChargesPerAllocationOverhead: {} Unit_HRR_Limits_NoStallWhileACallIsInFlight: {} Unit_HRR_Limits_ParseU64: {} Unit_HRR_Limits_PlaybackContextHostBudget: {} + Unit_HRR_Limits_PlaybackInfoRefusesOverByteBudget: {} Unit_HRR_Limits_PlaybackRefusesTooManyEvents: {} Unit_HRR_Limits_PlaybackRefusesTooManyThreads: {} Unit_HRR_Limits_PlaybackRefusesWallSecondsAboveMaximum: {} Unit_HRR_Limits_QueryRetryIsBounded: {} Unit_HRR_Limits_ReadFileMissing: {} Unit_HRR_Limits_ReadFileRefusesOversize: {} + Unit_HRR_Limits_RecordChargedBeforeAllocation: {} Unit_HRR_Limits_ScopedCharge: {} Unit_HRR_Limits_SequenceGapEndsReplay: {} Unit_HRR_Limits_ThreadCap: {} + Unit_HRR_Limits_WallClockEndsBlockedLoad: {} Unit_HRR_Limits_WallClockWatchdog: {} Unit_HRR_Playback_PlacementHeldRanges: {} Unit_HRR_Playback_PlacementRangeChecked: {} @@ -65,6 +73,7 @@ cases: Unit_HRR_Recovery_NoTrailer: {} Unit_HRR_Recovery_OversizeRecordBeforeTrailer: {} Unit_HRR_Recovery_RecordAfterTrailer: {} + Unit_HRR_Recovery_RepairRootHonoursMaxHostBytes: {} Unit_HRR_Recovery_RepairRootRewritesMiscountedTrailer: {} Unit_HRR_Recovery_RepairWritesProcessManifest: {} Unit_HRR_Recovery_RootInfoChecksTrailerCount: {} From 46ffe850710d267e471fb73dcbe29ad8e356b1e8 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 29 Oct 2026 19:54:00 +0100 Subject: [PATCH 74/89] AIRUNTIME-2871: skip the FIFO wall-clock load test on Windows in unit.yaml Unit_HRR_Limits_WallClockEndsBlockedLoad is compiled out under _WIN32, so the Windows case-list check reported it as missing. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/unit.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/config/unit.yaml b/projects/hrr/tests/config/unit.yaml index cf6d79ef6c9e..95365a712dd4 100644 --- a/projects/hrr/tests/config/unit.yaml +++ b/projects/hrr/tests/config/unit.yaml @@ -59,7 +59,11 @@ cases: Unit_HRR_Limits_ScopedCharge: {} Unit_HRR_Limits_SequenceGapEndsReplay: {} Unit_HRR_Limits_ThreadCap: {} - Unit_HRR_Limits_WallClockEndsBlockedLoad: {} + Unit_HRR_Limits_WallClockEndsBlockedLoad: + skip: + - when: {os: windows} + kind: unsupported + reason: hrr_limits_test.cc compiles this case out under _WIN32 (it blocks the load on a FIFO) Unit_HRR_Limits_WallClockWatchdog: {} Unit_HRR_Playback_PlacementHeldRanges: {} Unit_HRR_Playback_PlacementRangeChecked: {} From c5a97e6c429c59538faed1d37e6bf65bdd53adff Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 29 Oct 2026 22:33:32 +0100 Subject: [PATCH 75/89] AIRUNTIME-2871: skip the image-copy workload tests under rocJitsu until it executes image_store #14157 added hipMemcpy3D copies to and from hipArrays to Unit_HRR_AllApis_Direct. CLR runs them as image blit kernels and rocJitsu cannot execute image_store, so the queue aborts on the emulated gfx90a, gfx11 and gfx12 configs. Skip AllApisRoundtrip and AllApis_Direct there, and T4 on gfx90a (gfx11/gfx120 are already skipped for #13724). Tracked in ROCm/rocm-systems#14612. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/integration.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 7bf6a75e5ce0..caefa4897930 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -19,8 +19,18 @@ targets: os: [linux, windows] arch: [gfx90a, gfx942, gfx950, gfx1030, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1153, gfx1200, gfx1201, gfx1250] cases: - Unit_HRR_AllApisRoundtrip: {} - Unit_HRR_AllApis_Direct: {} + Unit_HRR_AllApisRoundtrip: + skip: + - when: {arch: ["gfx90a", "gfx11*", "gfx120*"], emulator: true} + kind: disabled + reason: "rocJitsu does not execute image_store, which hipMemcpy3D to and from a hipArray needs on image-capable archs" + issue: "https://github.com/ROCm/rocm-systems/issues/14612" + Unit_HRR_AllApis_Direct: + skip: + - when: {arch: ["gfx90a", "gfx11*", "gfx120*"], emulator: true} + kind: disabled + reason: "rocJitsu does not execute image_store, which hipMemcpy3D to and from a hipArray needs on image-capable archs" + issue: "https://github.com/ROCm/rocm-systems/issues/14612" Unit_HRR_ApiMatrix_Breadth2_Direct: {} Unit_HRR_ApiMatrix_Breadth_Direct: {} Unit_HRR_ApiMatrix_Deprioritised_Direct: {} @@ -55,6 +65,10 @@ cases: kind: disabled reason: "hipMemcpyToArray and hipMemcpyFromArray copy no data under the RDNA3 and RDNA4 rocJitsu configs" issue: "https://github.com/ROCm/rocm-systems/issues/13724" + - when: {arch: gfx90a, emulator: true} + kind: disabled + reason: "rocJitsu does not execute image_store, which hipMemcpy3D to and from a hipArray needs on image-capable archs" + issue: "https://github.com/ROCm/rocm-systems/issues/14612" Unit_HRR_ApiMatrix_T5_Roundtrip: {} Unit_HRR_ApiMatrix_UC1Dense_Direct: {} Unit_HRR_BlankCaptureOutputCapturesNothing: {} From 8586302ae336f13be8e2aea32cdeb9ba213ca181 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 29 Oct 2026 22:35:58 +0100 Subject: [PATCH 76/89] test(hrr): list the KernelArgsSignature handler cases from develop (AIRUNTIME-2871) #14154 added six Unit_HRR_KernelArgsSignature_* cases to the handler test sources; the handler config must list every compiled case. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/handler.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml index aceb199d3c6b..172c75924f37 100644 --- a/projects/hrr/tests/config/handler.yaml +++ b/projects/hrr/tests/config/handler.yaml @@ -41,6 +41,12 @@ cases: Unit_HRR_HostBudget_HostAllocationHandlersRefuseOverBudget: {} Unit_HRR_HostBudget_LandingBufferChargesWholeBlock: {} Unit_HRR_HostBudget_LandingBufferPointerSurvivesGrowth: {} + Unit_HRR_KernelArgsSignature_ArgumentsCarryReadPadding: {} + Unit_HRR_KernelArgsSignature_GraphKernelNode: {} + Unit_HRR_KernelArgsSignature_GraphSetParams: {} + Unit_HRR_KernelArgsSignature_NarrowArgument: {} + Unit_HRR_KernelArgsSignature_SetParamsWithoutParameters: {} + Unit_HRR_KernelArgsSignature_TooFewArguments: {} Unit_HRR_Memcpy3DArray_FreedHandleIsForgotten: {} Unit_HRR_Memcpy3DArray_HostCopiesAreNotIssued: {} Unit_HRR_Memcpy3DArray_TranslatesRecordedHandles: {} From 0ec29be97eb965a8a3a3f47fc201e39e8bff261f Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Thu, 29 Oct 2026 23:11:46 +0100 Subject: [PATCH 77/89] test(hrr): drop the skip on the AllApis direct workload (AIRUNTIME-2871) hrr_test_config.py check rejects skip rules on a direct workload. The AllApisRoundtrip skip already covers it, since the roundtrip runs the workload as its capture subprocess. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/integration.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 01183b7042e9..d038ea94098a 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -25,12 +25,7 @@ cases: kind: disabled reason: "rocJitsu does not execute image_store, which hipMemcpy3D to and from a hipArray needs on image-capable archs" issue: "https://github.com/ROCm/rocm-systems/issues/14612" - Unit_HRR_AllApis_Direct: - skip: - - when: {arch: ["gfx90a", "gfx11*", "gfx120*"], emulator: true} - kind: disabled - reason: "rocJitsu does not execute image_store, which hipMemcpy3D to and from a hipArray needs on image-capable archs" - issue: "https://github.com/ROCm/rocm-systems/issues/14612" + Unit_HRR_AllApis_Direct: {} Unit_HRR_ApiMatrix_Breadth2_Direct: {} Unit_HRR_ApiMatrix_Breadth_Direct: {} Unit_HRR_ApiMatrix_Deprioritised_Direct: {} From 9a61c4d2c77493524ae2f88958109c27fdb899dc Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 2 Nov 2026 09:20:06 +0000 Subject: [PATCH 78/89] ci(hrr): build and ship the rocjitsu-memory keeper for the simulator jobs librocjitsu.so now execs a private rocjitsu-memory helper. The workflow built only rocjitsu_bin and rocjitsu_shared, so the helper was missing and every simulator job saw no ROCm-capable device. Build it, and package it where the installed layout looks for it. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/hrr-gpu.yml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 70f4b3d4f4d8..28ba58c53007 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -145,7 +145,7 @@ jobs: run: | set -euo pipefail cmake -S emulation/rocjitsu -B build/rocjitsu -G Ninja -DCMAKE_BUILD_TYPE=Release - cmake --build build/rocjitsu --target rocjitsu_bin rocjitsu_shared -j + cmake --build build/rocjitsu --target rocjitsu_bin rocjitsu_shared rocjitsu_memory -j - name: Package rocJitsu shell: bash @@ -153,8 +153,19 @@ jobs: set -euo pipefail # The glob must expand inside build/rocjitsu, so cd there. tar keeps # symlinks, so a versioned librocjitsu.so chain survives. - ls -l build/rocjitsu/tools/rocjitsu/rocjitsu build/rocjitsu/librocjitsu.so* - (cd build/rocjitsu && tar -cf "${GITHUB_WORKSPACE}/rocjitsu.tar" tools/rocjitsu/rocjitsu librocjitsu.so*) + # + # librocjitsu.so execs the private rocjitsu-memory keeper. The path + # baked in for a build tree is this runner's, which the simulator + # containers do not have, so the keeper goes where the installed + # layout looks for it: ../libexec/rocjitsu/ beside the library's + # directory (build/libexec/rocjitsu/ next to build/rocjitsu/). + ls -l build/rocjitsu/tools/rocjitsu/rocjitsu build/rocjitsu/librocjitsu.so* \ + build/rocjitsu/tools/rocjitsu-memory + mkdir -p build/libexec/rocjitsu + cp build/rocjitsu/tools/rocjitsu-memory build/libexec/rocjitsu/rocjitsu-memory + (cd build && tar -cf "${GITHUB_WORKSPACE}/rocjitsu.tar" \ + rocjitsu/tools/rocjitsu/rocjitsu rocjitsu/librocjitsu.so* \ + libexec/rocjitsu/rocjitsu-memory) tar -tvf rocjitsu.tar - name: Upload rocJitsu @@ -236,9 +247,10 @@ jobs: shell: bash run: | set -euo pipefail - mkdir -p build/rocjitsu - tar -C build/rocjitsu -xf rocjitsu-artifact/rocjitsu.tar - ls -l build/rocjitsu/tools/rocjitsu build/rocjitsu/librocjitsu.so* + mkdir -p build + tar -C build -xf rocjitsu-artifact/rocjitsu.tar + ls -l build/rocjitsu/tools/rocjitsu build/rocjitsu/librocjitsu.so* \ + build/libexec/rocjitsu/rocjitsu-memory echo "${GITHUB_WORKSPACE}/build/rocjitsu/tools/rocjitsu" >> "${GITHUB_PATH}" - name: Download build From 2e9cd5dc815afd41bf51d0bf7495cd8a2759f0ce Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 2 Nov 2026 09:39:52 +0000 Subject: [PATCH 79/89] test(hrr): split the hipArray block out of AllApis so a rocJitsu skip covers only it Unit_HRR_AllApisRoundtrip was skipped on gfx90a, gfx11* and gfx120* under rocJitsu for the one hipArray block, which takes the other ~55 APIs with it. The block is now Unit_HRR_AllApisArrays_Direct with its own roundtrip, and only that is skipped. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/integration.yaml | 4 +- .../integration/hrr_api_matrix_expectations.h | 1 + .../tests/integration/hrr_roundtrip_test.cc | 41 ++- .../tests/integration/hrr_workload_test.cc | 291 +++++++++--------- 4 files changed, 187 insertions(+), 150 deletions(-) diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index d038ea94098a..d7d2b7d7c728 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -19,12 +19,14 @@ targets: os: [linux, windows] arch: [gfx90a, gfx942, gfx950, gfx1030, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1153, gfx1200, gfx1201, gfx1250] cases: - Unit_HRR_AllApisRoundtrip: + Unit_HRR_AllApisRoundtrip: {} + Unit_HRR_AllApisArraysRoundtrip: skip: - when: {arch: ["gfx90a", "gfx11*", "gfx120*"], emulator: true} kind: disabled reason: "rocJitsu does not execute image_store, which hipMemcpy3D to and from a hipArray needs on image-capable archs" issue: "https://github.com/ROCm/rocm-systems/issues/14612" + Unit_HRR_AllApisArrays_Direct: {} Unit_HRR_AllApis_Direct: {} Unit_HRR_ApiMatrix_Breadth2_Direct: {} Unit_HRR_ApiMatrix_Breadth_Direct: {} diff --git a/projects/hrr/tests/integration/hrr_api_matrix_expectations.h b/projects/hrr/tests/integration/hrr_api_matrix_expectations.h index 4014428d5378..ee254beb91f8 100644 --- a/projects/hrr/tests/integration/hrr_api_matrix_expectations.h +++ b/projects/hrr/tests/integration/hrr_api_matrix_expectations.h @@ -675,6 +675,7 @@ inline constexpr const char* const kHrrWorkloadsT4[] = { "Unit_HRR_StreamCaptureQuerySpt_Direct", "Unit_HRR_ApiMatrix_MemPoolShare_Direct", "Unit_HRR_AllApis_Direct", + "Unit_HRR_AllApisArrays_Direct", "Unit_HRR_StressApis_Direct", "Unit_HRR_DeviceInfo_Direct", "Unit_HRR_DeviceExtra_Direct", diff --git a/projects/hrr/tests/integration/hrr_roundtrip_test.cc b/projects/hrr/tests/integration/hrr_roundtrip_test.cc index f1aee06d2ded..62d7eae59781 100644 --- a/projects/hrr/tests/integration/hrr_roundtrip_test.cc +++ b/projects/hrr/tests/integration/hrr_roundtrip_test.cc @@ -371,21 +371,44 @@ HRR_TEST_CASE(Unit_HRR_AllApisRoundtrip) { REQUIRE(arc.events.size() >= 40); // ~55 distinct APIs exercised } - // The workload's array copies (block 19) need image support, which gfx950 - // (MI350X) reports as absent. Where it has them, a capture without the copies - // would replay perfectly well, so ask for them. - int images = 0; - HRR_HIP_CHECK(hipDeviceGetAttribute(&images, hipDeviceAttributeImageSupport, 0)); - if (images) - hrr_require_recorded_apis(cap.path, {HRR_API_HIPARRAY3DCREATE, HRR_API_HIPARRAYCREATE, - HRR_API_HIPMEMCPY3D, HRR_API_HIPMEMCPY3DASYNC}); - // ------------------------------------------------------------------------- // Step 3: playback + D2H validation (d2[i] == 94) // ------------------------------------------------------------------------- hrr_run_playback(cap.path); } +/** + * Test Description + * ---------------- + * - Spawns HrrTest Unit_HRR_AllApisArrays_Direct as a subprocess with + * HIP_HRR_CAPTURE_OUTPUT set to a temp directory: hipArrayCreate, + * hipArray3DCreate and hipMemcpy3D/hipMemcpy3DAsync with an array on either + * end. + * - Requires the array creation and copy calls in the archive, then replays it + * and validates the D2H blobs of the copied-out buffers. + * - Skips where the device reports no image support (gfx950 / MI350X), since + * the workload then records no array calls. + */ +HRR_TEST_CASE(Unit_HRR_AllApisArraysRoundtrip) { + int images = 0; + HRR_HIP_CHECK(hipDeviceGetAttribute(&images, hipDeviceAttributeImageSupport, 0)); + if (!images) HRR_SKIP("device reports no image support"); + + ScopedDir cap{fs::temp_directory_path() / "hrr_roundtrip_allapis_arrays"}; + { + hrr::test::SpawnProc proc(hrr_self_exe()); + proc.setEnv("HIP_HRR_CAPTURE_OUTPUT", cap.path.string()); + set_proc_search_path(proc); + int ret = proc.run("\"Unit_HRR_AllApisArrays_Direct\""); + INFO("AllApisArrays capture subprocess exit code: " << ret); + REQUIRE(ret == 0); + } + + hrr_require_recorded_apis(cap.path, {HRR_API_HIPARRAY3DCREATE, HRR_API_HIPARRAYCREATE, + HRR_API_HIPMEMCPY3D, HRR_API_HIPMEMCPY3DASYNC}); + hrr_run_playback(cap.path, "", /*require_d2h=*/true); +} + /** * Test Description * ---------------- diff --git a/projects/hrr/tests/integration/hrr_workload_test.cc b/projects/hrr/tests/integration/hrr_workload_test.cc index 75d2f559817d..5a8752780159 100644 --- a/projects/hrr/tests/integration/hrr_workload_test.cc +++ b/projects/hrr/tests/integration/hrr_workload_test.cc @@ -331,6 +331,157 @@ TEST_CASE("Unit_HRR_HostMemWorkload_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipFree(buf)); } +// --------------------------------------------------------------------------- +// Array handle-map and hipMemcpy3D coverage — hidden ([.]). +// +// Split from Unit_HRR_AllApis_Direct so that a GPU emulator which cannot execute +// the array copies skips only these, not the rest of the API coverage. +// The D2H blobs captured are the output buffers of the array copies. +// --------------------------------------------------------------------------- +TEST_CASE("Unit_HRR_AllApisArrays_Direct", "[.][hrr-direct]") { + HRR_HIP_CHECK(hipSetDevice(0)); + + // ========================================================================= + // 19. hipArrayCreate + hipArray3DCreate — handle-map coverage + // + // Array creation requires image support. Query device capability first and + // skip gracefully on hardware that reports no texture support. + // ========================================================================= + { + int supportsImages = 0; + HRR_HIP_CHECK(hipDeviceGetAttribute(&supportsImages, hipDeviceAttributeImageSupport, 0)); + if (supportsImages) { + HIP_ARRAY_DESCRIPTOR desc1d{}; + desc1d.Width = static_cast(N); + desc1d.Height = 0; // 1-D + desc1d.Format = HIP_AD_FORMAT_SIGNED_INT32; + desc1d.NumChannels = 1; + hipArray_t arr1d = nullptr; + HRR_HIP_CHECK(hipArrayCreate(&arr1d, &desc1d)); + HRR_HIP_CHECK(hipFreeArray(arr1d)); + + HIP_ARRAY3D_DESCRIPTOR desc3d{}; + desc3d.Width = 4; + desc3d.Height = 4; + desc3d.Depth = 4; + desc3d.Format = HIP_AD_FORMAT_SIGNED_INT32; + desc3d.NumChannels = 1; + desc3d.Flags = 0; + hipArray_t arr3d = nullptr; + HRR_HIP_CHECK(hipArray3DCreate(&arr3d, &desc3d)); + HRR_HIP_CHECK(hipFreeArray(arr3d)); + + // hipMemcpy3D and hipMemcpy3DAsync with an array on either end: device + // buffers into a 3-D and a 2-D array, array to array, and back into device + // buffers, whose hipMemcpy to the host is the D2H blob replay validates. A + // replay that handed the recorded array address to HIP instead of the array + // it created for it would crash or move other bytes. No copy has a host end, + // which replay does not issue. + constexpr size_t W = 16, H = 4, D = 2; // elements; one int each + constexpr size_t ROW = W * sizeof(int); + constexpr size_t BYTES3 = ROW * H * D; + constexpr size_t BYTES2 = ROW * H; + + std::vector h3(W * H * D), h2(W * H); + for (size_t i = 0; i < h3.size(); ++i) h3[i] = static_cast(i * 2654435761u) ^ 0x3d3d3d3d; + for (size_t i = 0; i < h2.size(); ++i) h2[i] = static_cast(i * 2654435761u) ^ 0x2d2d2d2d; + + HIP_ARRAY3D_DESCRIPTOR d3{}; + d3.Width = W; + d3.Height = H; + d3.Depth = D; + d3.Format = HIP_AD_FORMAT_SIGNED_INT32; + d3.NumChannels = 1; + hipArray_t a3 = nullptr, a3_copy = nullptr; + HRR_HIP_CHECK(hipArray3DCreate(&a3, &d3)); + HRR_HIP_CHECK(hipArray3DCreate(&a3_copy, &d3)); + + HIP_ARRAY_DESCRIPTOR d2{}; + d2.Width = W; + d2.Height = H; + d2.Format = HIP_AD_FORMAT_SIGNED_INT32; + d2.NumChannels = 1; + hipArray_t a2 = nullptr; + HRR_HIP_CHECK(hipArrayCreate(&a2, &d2)); + + // Device buffers the arrays are filled from (in*) and read back into (out*). + void *in3 = nullptr, *in2 = nullptr, *out3 = nullptr, *out3_copy = nullptr, *out2 = nullptr; + HRR_HIP_CHECK(hipMalloc(&in3, BYTES3)); + HRR_HIP_CHECK(hipMalloc(&in2, BYTES2)); + HRR_HIP_CHECK(hipMalloc(&out3, BYTES3)); + HRR_HIP_CHECK(hipMalloc(&out3_copy, BYTES3)); + HRR_HIP_CHECK(hipMalloc(&out2, BYTES2)); + HRR_HIP_CHECK(hipMemcpy(in3, h3.data(), BYTES3, hipMemcpyHostToDevice)); + HRR_HIP_CHECK(hipMemcpy(in2, h2.data(), BYTES2, hipMemcpyHostToDevice)); + HRR_HIP_CHECK(hipMemset(out3, 0, BYTES3)); + HRR_HIP_CHECK(hipMemset(out3_copy, 0, BYTES3)); + HRR_HIP_CHECK(hipMemset(out2, 0, BYTES2)); + + // device -> array, plain and async + { + hipMemcpy3DParms p{}; + p.srcPtr = make_hipPitchedPtr(in3, ROW, W, H); + p.dstArray = a3; + p.extent = make_hipExtent(W, H, D); + p.kind = hipMemcpyDeviceToDevice; + HRR_HIP_CHECK(hipMemcpy3D(&p)); + } + { + hipMemcpy3DParms p{}; + p.srcPtr = make_hipPitchedPtr(in2, ROW, W, H); + p.dstArray = a2; + p.extent = make_hipExtent(W, H, 1); + p.kind = hipMemcpyDeviceToDevice; + HRR_HIP_CHECK(hipMemcpy3DAsync(&p, nullptr)); + HRR_HIP_CHECK(hipDeviceSynchronize()); + } + // array -> array + { + hipMemcpy3DParms p{}; + p.srcArray = a3; + p.dstArray = a3_copy; + p.extent = make_hipExtent(W, H, D); + p.kind = hipMemcpyDeviceToDevice; + HRR_HIP_CHECK(hipMemcpy3D(&p)); + } + // array -> device + auto array_to_device = [&](hipArray_t src, void* dst, size_t depth) { + hipMemcpy3DParms p{}; + p.srcArray = src; + p.dstPtr = make_hipPitchedPtr(dst, ROW, W, H); + p.extent = make_hipExtent(W, H, depth); + p.kind = hipMemcpyDeviceToDevice; + HRR_HIP_CHECK(hipMemcpy3D(&p)); + }; + array_to_device(a3, out3, D); + array_to_device(a3_copy, out3_copy, D); + array_to_device(a2, out2, 1); + HRR_HIP_CHECK(hipDeviceSynchronize()); + + // The D2H blobs of the output buffers are what replay validates. + std::vector r3(W * H * D), r3c(W * H * D), r2(W * H); + HRR_HIP_CHECK(hipMemcpy(r3.data(), out3, BYTES3, hipMemcpyDeviceToHost)); + HRR_HIP_CHECK(hipMemcpy(r3c.data(), out3_copy, BYTES3, hipMemcpyDeviceToHost)); + HRR_HIP_CHECK(hipMemcpy(r2.data(), out2, BYTES2, hipMemcpyDeviceToHost)); + + // Each array copy must have moved the bytes: if the capture's copies moved + // nothing, the blobs would be zeros and the roundtrip would still agree. + REQUIRE(r3 == h3); + REQUIRE(r3c == h3); + REQUIRE(r2 == h2); + + HRR_HIP_CHECK(hipFree(in3)); + HRR_HIP_CHECK(hipFree(in2)); + HRR_HIP_CHECK(hipFree(out3)); + HRR_HIP_CHECK(hipFree(out3_copy)); + HRR_HIP_CHECK(hipFree(out2)); + HRR_HIP_CHECK(hipFreeArray(a3)); + HRR_HIP_CHECK(hipFreeArray(a3_copy)); + HRR_HIP_CHECK(hipFreeArray(a2)); + } + } +} + // --------------------------------------------------------------------------- // Comprehensive HIP API coverage workload — hidden ([.]). // @@ -640,146 +791,6 @@ TEST_CASE("Unit_HRR_AllApis_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipMemPoolSetAccess(pool, &access_desc, 1)); } - // ========================================================================= - // 19. hipArrayCreate + hipArray3DCreate — handle-map coverage - // - // Array creation requires image support. Query device capability first and - // skip gracefully on hardware that reports no texture support. - // ========================================================================= - { - int supportsImages = 0; - HRR_HIP_CHECK(hipDeviceGetAttribute(&supportsImages, hipDeviceAttributeImageSupport, 0)); - if (supportsImages) { - HIP_ARRAY_DESCRIPTOR desc1d{}; - desc1d.Width = static_cast(N); - desc1d.Height = 0; // 1-D - desc1d.Format = HIP_AD_FORMAT_SIGNED_INT32; - desc1d.NumChannels = 1; - hipArray_t arr1d = nullptr; - HRR_HIP_CHECK(hipArrayCreate(&arr1d, &desc1d)); - HRR_HIP_CHECK(hipFreeArray(arr1d)); - - HIP_ARRAY3D_DESCRIPTOR desc3d{}; - desc3d.Width = 4; - desc3d.Height = 4; - desc3d.Depth = 4; - desc3d.Format = HIP_AD_FORMAT_SIGNED_INT32; - desc3d.NumChannels = 1; - desc3d.Flags = 0; - hipArray_t arr3d = nullptr; - HRR_HIP_CHECK(hipArray3DCreate(&arr3d, &desc3d)); - HRR_HIP_CHECK(hipFreeArray(arr3d)); - - // hipMemcpy3D and hipMemcpy3DAsync with an array on either end: device - // buffers into a 3-D and a 2-D array, array to array, and back into device - // buffers, whose hipMemcpy to the host is the D2H blob replay validates. A - // replay that handed the recorded array address to HIP instead of the array - // it created for it would crash or move other bytes. No copy has a host end, - // which replay does not issue. - constexpr size_t W = 16, H = 4, D = 2; // elements; one int each - constexpr size_t ROW = W * sizeof(int); - constexpr size_t BYTES3 = ROW * H * D; - constexpr size_t BYTES2 = ROW * H; - - std::vector h3(W * H * D), h2(W * H); - for (size_t i = 0; i < h3.size(); ++i) h3[i] = static_cast(i * 2654435761u) ^ 0x3d3d3d3d; - for (size_t i = 0; i < h2.size(); ++i) h2[i] = static_cast(i * 2654435761u) ^ 0x2d2d2d2d; - - HIP_ARRAY3D_DESCRIPTOR d3{}; - d3.Width = W; - d3.Height = H; - d3.Depth = D; - d3.Format = HIP_AD_FORMAT_SIGNED_INT32; - d3.NumChannels = 1; - hipArray_t a3 = nullptr, a3_copy = nullptr; - HRR_HIP_CHECK(hipArray3DCreate(&a3, &d3)); - HRR_HIP_CHECK(hipArray3DCreate(&a3_copy, &d3)); - - HIP_ARRAY_DESCRIPTOR d2{}; - d2.Width = W; - d2.Height = H; - d2.Format = HIP_AD_FORMAT_SIGNED_INT32; - d2.NumChannels = 1; - hipArray_t a2 = nullptr; - HRR_HIP_CHECK(hipArrayCreate(&a2, &d2)); - - // Device buffers the arrays are filled from (in*) and read back into (out*). - void *in3 = nullptr, *in2 = nullptr, *out3 = nullptr, *out3_copy = nullptr, *out2 = nullptr; - HRR_HIP_CHECK(hipMalloc(&in3, BYTES3)); - HRR_HIP_CHECK(hipMalloc(&in2, BYTES2)); - HRR_HIP_CHECK(hipMalloc(&out3, BYTES3)); - HRR_HIP_CHECK(hipMalloc(&out3_copy, BYTES3)); - HRR_HIP_CHECK(hipMalloc(&out2, BYTES2)); - HRR_HIP_CHECK(hipMemcpy(in3, h3.data(), BYTES3, hipMemcpyHostToDevice)); - HRR_HIP_CHECK(hipMemcpy(in2, h2.data(), BYTES2, hipMemcpyHostToDevice)); - HRR_HIP_CHECK(hipMemset(out3, 0, BYTES3)); - HRR_HIP_CHECK(hipMemset(out3_copy, 0, BYTES3)); - HRR_HIP_CHECK(hipMemset(out2, 0, BYTES2)); - - // device -> array, plain and async - { - hipMemcpy3DParms p{}; - p.srcPtr = make_hipPitchedPtr(in3, ROW, W, H); - p.dstArray = a3; - p.extent = make_hipExtent(W, H, D); - p.kind = hipMemcpyDeviceToDevice; - HRR_HIP_CHECK(hipMemcpy3D(&p)); - } - { - hipMemcpy3DParms p{}; - p.srcPtr = make_hipPitchedPtr(in2, ROW, W, H); - p.dstArray = a2; - p.extent = make_hipExtent(W, H, 1); - p.kind = hipMemcpyDeviceToDevice; - HRR_HIP_CHECK(hipMemcpy3DAsync(&p, nullptr)); - HRR_HIP_CHECK(hipDeviceSynchronize()); - } - // array -> array - { - hipMemcpy3DParms p{}; - p.srcArray = a3; - p.dstArray = a3_copy; - p.extent = make_hipExtent(W, H, D); - p.kind = hipMemcpyDeviceToDevice; - HRR_HIP_CHECK(hipMemcpy3D(&p)); - } - // array -> device - auto array_to_device = [&](hipArray_t src, void* dst, size_t depth) { - hipMemcpy3DParms p{}; - p.srcArray = src; - p.dstPtr = make_hipPitchedPtr(dst, ROW, W, H); - p.extent = make_hipExtent(W, H, depth); - p.kind = hipMemcpyDeviceToDevice; - HRR_HIP_CHECK(hipMemcpy3D(&p)); - }; - array_to_device(a3, out3, D); - array_to_device(a3_copy, out3_copy, D); - array_to_device(a2, out2, 1); - HRR_HIP_CHECK(hipDeviceSynchronize()); - - // The D2H blobs of the output buffers are what replay validates. - std::vector r3(W * H * D), r3c(W * H * D), r2(W * H); - HRR_HIP_CHECK(hipMemcpy(r3.data(), out3, BYTES3, hipMemcpyDeviceToHost)); - HRR_HIP_CHECK(hipMemcpy(r3c.data(), out3_copy, BYTES3, hipMemcpyDeviceToHost)); - HRR_HIP_CHECK(hipMemcpy(r2.data(), out2, BYTES2, hipMemcpyDeviceToHost)); - - // Each array copy must have moved the bytes: if the capture's copies moved - // nothing, the blobs would be zeros and the roundtrip would still agree. - REQUIRE(r3 == h3); - REQUIRE(r3c == h3); - REQUIRE(r2 == h2); - - HRR_HIP_CHECK(hipFree(in3)); - HRR_HIP_CHECK(hipFree(in2)); - HRR_HIP_CHECK(hipFree(out3)); - HRR_HIP_CHECK(hipFree(out3_copy)); - HRR_HIP_CHECK(hipFree(out2)); - HRR_HIP_CHECK(hipFreeArray(a3)); - HRR_HIP_CHECK(hipFreeArray(a3_copy)); - HRR_HIP_CHECK(hipFreeArray(a2)); - } - } - // ========================================================================= // Cleanup (reverse allocation order) // ========================================================================= From bf6d8ced89c4c5137cfef586187eb3b44ac28626 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Mon, 2 Nov 2026 10:13:04 +0000 Subject: [PATCH 80/89] [hrr] List Unit_HRR_AllApisArrays_Direct in the API matrix workloads The generated expectations header was stale after the hipArray block moved into its own workload. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tools/api-matrix/api_matrix.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/hrr/tools/api-matrix/api_matrix.yaml b/projects/hrr/tools/api-matrix/api_matrix.yaml index 832c3da85888..8daab1ef19c3 100644 --- a/projects/hrr/tools/api-matrix/api_matrix.yaml +++ b/projects/hrr/tools/api-matrix/api_matrix.yaml @@ -354,6 +354,7 @@ tiers: - Unit_HRR_StreamCaptureQuerySpt_Direct - Unit_HRR_ApiMatrix_MemPoolShare_Direct - Unit_HRR_AllApis_Direct + - Unit_HRR_AllApisArrays_Direct - Unit_HRR_StressApis_Direct - Unit_HRR_DeviceInfo_Direct - Unit_HRR_DeviceExtra_Direct From 3bd0ff2491b34f176f2550e9706266bddde4d339 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 3 Nov 2026 14:32:20 +0100 Subject: [PATCH 81/89] [HRR] Enable integration cases that now pass under rocJitsu (AIRUNTIME-2871) Drop the emulator skips for ExtMemcpyBatchRoundtrip, FailedBlobWrite_LeavesCaptureIncomplete, Fork_ChildArchiveCompleteAfterParentFailure, Fork_ChildListsOnlyItsOwnUnreplayableApis, MemcpyExtraRoundtrip and PinnedHostFork. Each passes under rocJitsu (origin/develop b96bbff01ec) on every arch its skip named, and natively on gfx950. ApiMatrix_T4 (gfx11*/gfx120*, gfx90a), AllApisArraysRoundtrip, ApiMatrix_T1, CaptureCapabilities and VaPlacement_StoredPointer still fail and keep their skips. --- projects/hrr/tests/config/integration.yaml | 42 ++++------------------ 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index 07a2dc8cdf16..b3e775593154 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -177,24 +177,14 @@ cases: Unit_HRR_ExtLaunchEventMarkers: {} Unit_HRR_ExtMallocRoundtrip: {} Unit_HRR_ExtMalloc_Direct: {} - Unit_HRR_ExtMemcpyBatchRoundtrip: - skip: - - when: {arch: ["gfx94*", "gfx95*", "gfx125*"], emulator: true} - kind: disabled - reason: "hipExtMemcpyBatchAsync with hipMemcpyFlagExtOpSwap hangs under rocJitsu: the SDMA decoder rejects COPY_LINEAR_SWAP (sub-opcode 9), the ring stops before the fence and the stream never syncs" - issue: "https://github.com/ROCm/rocm-systems/issues/14464" + Unit_HRR_ExtMemcpyBatchRoundtrip: {} Unit_HRR_ExtMemcpyBatch_Direct: {} Unit_HRR_ExtModuleLaunchKernelRoundtrip: {} Unit_HRR_ExtModuleLaunchKernel_Direct: {} Unit_HRR_FailedBlobClose_Direct: {} Unit_HRR_FailedBlobClose_LeavesCaptureIncomplete: {} Unit_HRR_FailedBlobWrite_Direct: {} - Unit_HRR_FailedBlobWrite_LeavesCaptureIncomplete: - skip: - - when: {arch: ["gfx94*", "gfx95*", "gfx125*"], emulator: true} - kind: disabled - reason: "Local-mode rocJitsu runs inside the process, and the case lowers RLIMIT_FSIZE, which also limits the simulator" - issue: "https://github.com/ROCm/rocm-systems/issues/13726" + Unit_HRR_FailedBlobWrite_LeavesCaptureIncomplete: {} Unit_HRR_FailedEndCaptureInvalidatedRoundtrip: {} Unit_HRR_FailedEndCaptureInvalidated_Direct: {} Unit_HRR_FailedEndCaptureRoundtrip: {} @@ -219,18 +209,8 @@ cases: Unit_HRR_ForkWhileRecording_Direct: {} Unit_HRR_ForkWhileWriterHoldsLock: {} Unit_HRR_ForkWhileWriterHoldsLock_Direct: {} - Unit_HRR_Fork_ChildArchiveCompleteAfterParentFailure: - skip: - - when: {arch: ["gfx94*", "gfx95*", "gfx125*"], emulator: true} - kind: disabled - reason: "Local-mode rocJitsu runs inside the process, and the case lowers RLIMIT_FSIZE, which also limits the simulator" - issue: "https://github.com/ROCm/rocm-systems/issues/13726" - Unit_HRR_Fork_ChildListsOnlyItsOwnUnreplayableApis: - skip: - - when: {arch: ["gfx94*", "gfx95*", "gfx125*"], emulator: true} - kind: disabled - reason: "Local-mode rocJitsu runs inside the process, and the case lowers RLIMIT_FSIZE, which also limits the simulator" - issue: "https://github.com/ROCm/rocm-systems/issues/13726" + Unit_HRR_Fork_ChildArchiveCompleteAfterParentFailure: {} + Unit_HRR_Fork_ChildListsOnlyItsOwnUnreplayableApis: {} Unit_HRR_Fork_Direct: {} Unit_HRR_ForkedChildRecordsAfterShutdown: {} Unit_HRR_ForkedChildRecordsAfterShutdown_Direct: {} @@ -264,12 +244,7 @@ cases: Unit_HRR_MemPoolExtended_Direct: {} Unit_HRR_Memcpy3DSptRoundtrip: {} Unit_HRR_Memcpy3DSpt_Direct: {} - Unit_HRR_MemcpyExtraRoundtrip: - skip: - - when: {arch: ["gfx11*", "gfx120*"], emulator: true} - kind: disabled - reason: "hipMemcpyToArray and hipMemcpyFromArray copy no data under the RDNA3 and RDNA4 rocJitsu configs" - issue: "https://github.com/ROCm/rocm-systems/issues/13724" + Unit_HRR_MemcpyExtraRoundtrip: {} Unit_HRR_MemcpyExtra_Direct: {} Unit_HRR_MemcpyPeerRoundtrip: {} Unit_HRR_MemcpyPeer_Direct: {} @@ -304,12 +279,7 @@ cases: Unit_HRR_OverflowingMemcpy3DNotRecorded: {} Unit_HRR_OverflowingMemcpy3D_Direct: {} Unit_HRR_PinnedHostBlobRefused: {} - Unit_HRR_PinnedHostFork: - skip: - - when: {arch: ["gfx95*"], emulator: true} - kind: disabled - reason: "The capture subprocess exited 42 under the two-GPU gfx950 rocJITsu configuration; the same case passed on the other rocJITsu families" - issue: "https://github.com/ROCm/rocm-systems/issues/14033" + Unit_HRR_PinnedHostFork: {} Unit_HRR_PinnedHostFork_Direct: {} Unit_HRR_PinnedHostFreedRoundtrip: {} Unit_HRR_PinnedHostFreed_Direct: {} From 3eedc359f50069efb35f02f56e00d6e3b36a3358 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Tue, 3 Nov 2026 15:16:25 +0100 Subject: [PATCH 82/89] [HRR] Refresh the API baseline and handler case list after develop drift (AIRUNTIME-2871) develop's #14341 moved hipEventElapsedTime from the generated to the manual replay class (GENERATED 205->204, MANUAL 102->103). Its observable class stays REAL, so api_matrix.yaml and hrr_api_matrix_expectations.h do not change; only the recorded baseline is refreshed. develop's #14343 and #14346 added six hrr-handler-tests cases (HostUnregister FailedCallKeepsBuffer and five SnapshotOrder cases); handler.yaml lists them (43 -> 49, matching the binary). The case names come from the TEST_CASE registrations, not a built binary; listing needs no GPU. --- projects/hrr/tests/config/handler.yaml | 6 ++++++ projects/hrr/tools/api-matrix/api_classes_baseline.json | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml index 172c75924f37..f653d3bec145 100644 --- a/projects/hrr/tests/config/handler.yaml +++ b/projects/hrr/tests/config/handler.yaml @@ -41,6 +41,7 @@ cases: Unit_HRR_HostBudget_HostAllocationHandlersRefuseOverBudget: {} Unit_HRR_HostBudget_LandingBufferChargesWholeBlock: {} Unit_HRR_HostBudget_LandingBufferPointerSurvivesGrowth: {} + Unit_HRR_HostUnregister_FailedCallKeepsBuffer: {} Unit_HRR_KernelArgsSignature_ArgumentsCarryReadPadding: {} Unit_HRR_KernelArgsSignature_GraphKernelNode: {} Unit_HRR_KernelArgsSignature_GraphSetParams: {} @@ -63,5 +64,10 @@ cases: Unit_HRR_Refusals_LinkCreateRefusesPointerCarryingOptions: {} Unit_HRR_Refusals_Memcpy2DNoOpCopiesAreNotRefused: {} Unit_HRR_Refusals_Memcpy2DWithUnmappedEndIsRefused: {} + Unit_HRR_SnapshotOrder_FailedUnregisterKeepsBuffer: {} + Unit_HRR_SnapshotOrder_GuardReleasesTheHold: {} + Unit_HRR_SnapshotOrder_HostFreeWaitsForLaunch: {} + Unit_HRR_SnapshotOrder_LaunchHandlersTakeTheHold: {} + Unit_HRR_SnapshotOrder_UnregisterWaitsForLaunch: {} Unit_HRR_UnmappedNote_IpcCloseOfSkippedImport: {} Unit_HRR_UnmappedNote_MemMap: {} diff --git a/projects/hrr/tools/api-matrix/api_classes_baseline.json b/projects/hrr/tools/api-matrix/api_classes_baseline.json index 641bcd983830..6a1ca3cc7712 100644 --- a/projects/hrr/tools/api-matrix/api_classes_baseline.json +++ b/projects/hrr/tools/api-matrix/api_classes_baseline.json @@ -13,8 +13,8 @@ "replay_class.COMPILER_NOOP": 7, "replay_class.CUSTOM": 1, "replay_class.ERROR_STUB": 4, - "replay_class.GENERATED": 205, - "replay_class.MANUAL": 102, + "replay_class.GENERATED": 204, + "replay_class.MANUAL": 103, "replay_class.NOOP": 222, "replay_class.SPECIAL": 6, "replay_class.UNREPLAYABLE": 12, From c635a48493a50bee2bb07bd431e92385e263f162 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 6 Nov 2026 08:50:12 +0100 Subject: [PATCH 83/89] test(hrr): list the SymbolCopyBounds cases in handler.yaml develop's #14350 added three hrr-handler-tests cases (52 compiled vs 49 listed), so the compiled-case-list check failed. The case set was checked against the TEST_CASE names in the handler sources, not a built binary. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/handler.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml index f653d3bec145..494c92ea4301 100644 --- a/projects/hrr/tests/config/handler.yaml +++ b/projects/hrr/tests/config/handler.yaml @@ -69,5 +69,8 @@ cases: Unit_HRR_SnapshotOrder_HostFreeWaitsForLaunch: {} Unit_HRR_SnapshotOrder_LaunchHandlersTakeTheHold: {} Unit_HRR_SnapshotOrder_UnregisterWaitsForLaunch: {} + Unit_HRR_SymbolCopyBounds_InRangeCopyReachesHip: {} + Unit_HRR_SymbolCopyBounds_OutOfRangeIsRefusedBeforeHip: {} + Unit_HRR_SymbolCopyBounds_WrappingSumIsRefused: {} Unit_HRR_UnmappedNote_IpcCloseOfSkippedImport: {} Unit_HRR_UnmappedNote_MemMap: {} From 0d6e095f9ba24164e4631c46bcedfe20df502cad Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 6 Nov 2026 13:03:16 +0100 Subject: [PATCH 84/89] test(hrr): list develop's four new handler cases in handler.yaml hrr_test_config.py check on the merged tree reported 52 expected, 56 found. The cases come from hrr_host_budget_test.cc and hrr_unregister_fail_test.cc. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/handler.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml index 494c92ea4301..da61a607cf71 100644 --- a/projects/hrr/tests/config/handler.yaml +++ b/projects/hrr/tests/config/handler.yaml @@ -38,10 +38,14 @@ cases: Unit_HRR_HostBudget_ConcurrentLoadsOfOneBlobChargeOnce: {} Unit_HRR_HostBudget_DispatchFreesReplacedHostAllocation: {} Unit_HRR_HostBudget_DispatchTableUsesTheChargingHandlers: {} + Unit_HRR_HostBudget_HostBlocksStartOnCacheLine: {} Unit_HRR_HostBudget_HostAllocationHandlersRefuseOverBudget: {} + Unit_HRR_HostBudget_HostRegisterAllocationFailureRefuses: {} Unit_HRR_HostBudget_LandingBufferChargesWholeBlock: {} Unit_HRR_HostBudget_LandingBufferPointerSurvivesGrowth: {} Unit_HRR_HostUnregister_FailedCallKeepsBuffer: {} + Unit_HRR_HostUnregister_FailedReplaceTakesNewBlock: {} + Unit_HRR_HostUnregister_GraphNodeKeepsBuffer: {} Unit_HRR_KernelArgsSignature_ArgumentsCarryReadPadding: {} Unit_HRR_KernelArgsSignature_GraphKernelNode: {} Unit_HRR_KernelArgsSignature_GraphSetParams: {} From 39affab70b4938ef455d1cf5aca4388d5745f6ff Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 6 Nov 2026 14:20:44 +0100 Subject: [PATCH 85/89] HRR CI: skip Unit_HRR_CaptureCrashDuringFork on gfx125x rocJitsu Under the rocJitsu emulator on gfx125x, the parent process hangs in the crash path. The watchdog kills it, so the test sees exit 137 instead of 139. Disable the case there until https://github.com/ROCm/rocm-systems/issues/15217 is fixed. The case list is unchanged, and the _Direct variant is not touched. Co-Authored-By: Claude Code --- projects/hrr/tests/config/integration.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index b3e775593154..8c05aaf85590 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -83,7 +83,12 @@ cases: Unit_HRR_CaptureCapabilities_Direct: {} Unit_HRR_CaptureClosesArchiveDirs: {} Unit_HRR_CaptureClosesArchiveDirs_Direct: {} - Unit_HRR_CaptureCrashDuringFork: {} + Unit_HRR_CaptureCrashDuringFork: + skip: + - when: {arch: ["gfx125*"], emulator: true} + kind: disabled + reason: "The parent hangs in the crash path under rocJitsu, the watchdog kills it, and the test sees exit 137 instead of 139" + issue: "https://github.com/ROCm/rocm-systems/issues/15217" Unit_HRR_CaptureCrashDuringFork_Direct: {} Unit_HRR_CaptureCrashOnSmallStack: {} Unit_HRR_CaptureCrashSmallStack_Direct: {} From 1a9131a060dabf5a72dffd69a46e4cfc91ef7564 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 6 Nov 2026 14:27:07 +0100 Subject: [PATCH 86/89] test(hrr): list develop's six HostBudget handler cases in handler.yaml The merge with develop brings the six new Unit_HRR_HostBudget_* cases from hrr_host_budget_test.cc, so hrr-handler-tests registers 62 cases against 56 in handler.yaml. List the six, in sorted order. The case set was derived from the TEST_CASE sources, not from a built binary. Co-Authored-By: Claude Code --- projects/hrr/tests/config/handler.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/projects/hrr/tests/config/handler.yaml b/projects/hrr/tests/config/handler.yaml index da61a607cf71..b33c54443abe 100644 --- a/projects/hrr/tests/config/handler.yaml +++ b/projects/hrr/tests/config/handler.yaml @@ -36,13 +36,19 @@ cases: Unit_HRR_HostBudget_BlobCacheRefusesOverBudgetAndMissing: {} Unit_HRR_HostBudget_CodeObjectCacheIsChargedSeparately: {} Unit_HRR_HostBudget_ConcurrentLoadsOfOneBlobChargeOnce: {} + Unit_HRR_HostBudget_DeviceMemCreateNotCharged: {} Unit_HRR_HostBudget_DispatchFreesReplacedHostAllocation: {} Unit_HRR_HostBudget_DispatchTableUsesTheChargingHandlers: {} - Unit_HRR_HostBudget_HostBlocksStartOnCacheLine: {} Unit_HRR_HostBudget_HostAllocationHandlersRefuseOverBudget: {} + Unit_HRR_HostBudget_HostBlocksStartOnCacheLine: {} + Unit_HRR_HostBudget_HostMemCreateRefusedOverBudget: {} Unit_HRR_HostBudget_HostRegisterAllocationFailureRefuses: {} Unit_HRR_HostBudget_LandingBufferChargesWholeBlock: {} Unit_HRR_HostBudget_LandingBufferPointerSurvivesGrowth: {} + Unit_HRR_HostBudget_ManagedAllocationRefusedOverBudget: {} + Unit_HRR_HostBudget_ManagedAllocationReturnsChargeOnFree: {} + Unit_HRR_HostBudget_MemCreateChargeHeldWhileMapped: {} + Unit_HRR_HostBudget_RegionReleaseKeepsReplacingAllocation: {} Unit_HRR_HostUnregister_FailedCallKeepsBuffer: {} Unit_HRR_HostUnregister_FailedReplaceTakesNewBlock: {} Unit_HRR_HostUnregister_GraphNodeKeepsBuffer: {} From 3ddec442d9677caa4629dc60bf1d92e324493ac4 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 6 Nov 2026 14:54:40 +0100 Subject: [PATCH 87/89] HRR CI: cache rocJitsu compiles with ccache The Build rocJitsu job compiled everything cold on every run, about 16 minutes. Install ccache, set it as the C and C++ compiler launcher, and restore and save its directory with actions/cache (the SHA already used in amdsmi-dme-ci.yml). The cache holds only rocJitsu compile objects, in a directory under RUNNER_TEMP. The ROCm install in the build job stays uncached because of the librocm_kpack.so.0 failure. The key has the runner OS and arch, a hash of the c++ and cc versions, and a hash of emulation/rocjitsu, runtimes/api-headers and shared/machine-readable-isa; restore-keys fall back to the compiler prefix. ccache hashes preprocessed source, so a partial restore cannot give a stale object. The size limit is 500M and the job prints ccache statistics at the end. Co-Authored-By: Claude Code --- .github/workflows/hrr-gpu.yml | 39 +++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/hrr-gpu.yml b/.github/workflows/hrr-gpu.yml index 28ba58c53007..116f3d047a02 100644 --- a/.github/workflows/hrr-gpu.yml +++ b/.github/workflows/hrr-gpu.yml @@ -126,7 +126,7 @@ jobs: set -euo pipefail sudo -n apt-get update sudo -n DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential ca-certificates clang libdrm-dev ninja-build + build-essential ca-certificates ccache clang libdrm-dev ninja-build - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -140,13 +140,48 @@ jobs: "${RUNNER_TEMP}/hrr-venv/bin/python" -m pip install --no-cache-dir 'cmake==3.31.10' echo "${RUNNER_TEMP}/hrr-venv/bin" >> "${GITHUB_PATH}" + # ccache holds only the rocJitsu compile objects, in its own directory + # under RUNNER_TEMP. It never covers the ROCm install in the build job, + # which stays uncached (see the note above that job's Build step). + # ccache keys each object on the preprocessed source and the compiler, + # so a stale hit is not possible; the key below only decides when a new + # cache is saved. Pull requests restore the cache that develop pushes and + # the schedule saved, and save their own, which only that PR reads. + - name: Configure ccache + id: ccache + shell: bash + run: | + set -euo pipefail + { + echo "CCACHE_DIR=${RUNNER_TEMP}/ccache-rocjitsu" + echo "CCACHE_BASEDIR=${GITHUB_WORKSPACE}" + echo "CCACHE_COMPILERCHECK=content" + echo "CCACHE_MAXSIZE=500M" + } >> "${GITHUB_ENV}" + echo "compiler=$( (c++ --version | head -n1; cc --version | head -n1) | sha256sum | cut -c1-12)" >> "${GITHUB_OUTPUT}" + + - name: Restore ccache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ runner.temp }}/ccache-rocjitsu + key: ccache-rocjitsu-${{ runner.os }}-${{ runner.arch }}-${{ steps.ccache.outputs.compiler }}-${{ hashFiles('emulation/rocjitsu/**', 'runtimes/api-headers/**', 'shared/machine-readable-isa/**') }} + restore-keys: | + ccache-rocjitsu-${{ runner.os }}-${{ runner.arch }}-${{ steps.ccache.outputs.compiler }}- + - name: Build rocJitsu shell: bash run: | set -euo pipefail - cmake -S emulation/rocjitsu -B build/rocjitsu -G Ninja -DCMAKE_BUILD_TYPE=Release + ccache --zero-stats + cmake -S emulation/rocjitsu -B build/rocjitsu -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache cmake --build build/rocjitsu --target rocjitsu_bin rocjitsu_shared rocjitsu_memory -j + - name: ccache stats + if: ${{ !cancelled() }} + shell: bash + run: ccache --show-stats --verbose || true + - name: Package rocJitsu shell: bash run: | From 78f66220adbbcba64d094ae51303928d56e8e252 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 6 Nov 2026 15:29:35 +0100 Subject: [PATCH 88/89] HRR tests: size host budget caps from the archive's cached blobs The host budget roundtrip tests hardcoded caps of 1, 3 and 5 MiB. Replay charges every blob it loads, including the registered fat binary, and that grows with the arch list the test binary is built for. The CI build for all 14 archs holds about 1.4 MB, so the 1 MiB cap was refused by the blob cache before the allocation under test, and the 3 and 5 MiB caps left too little room. Caps are now the archive's blob and code object bytes plus the allocation budget. Co-Authored-By: Claude Code --- .../integration/hrr_host_alloc_budget_test.cc | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/projects/hrr/tests/integration/hrr_host_alloc_budget_test.cc b/projects/hrr/tests/integration/hrr_host_alloc_budget_test.cc index 70bc3f6857b0..5fd9c1029ead 100644 --- a/projects/hrr/tests/integration/hrr_host_alloc_budget_test.cc +++ b/projects/hrr/tests/integration/hrr_host_alloc_budget_test.cc @@ -13,8 +13,8 @@ * refuses a record that would pass the cap, and gives the charge back on hipFree * and hipMemRelease. Each workload allocates 2 MiB three times in a row, * freeing the buffer in between, so a replay whose cap holds one buffer (3 MiB) - * only gets through if the free returns the charge, and a cap below one buffer - * (1 MiB) refuses the first. + * only gets through if the free returns the charge, and a cap with less than one + * buffer (1 MiB) to spare refuses the first. Caps are relative to the archive's cached blobs. */ #include "hrr_test_common.hh" @@ -22,17 +22,41 @@ #include namespace { +// Replay charges every blob and code object it loads to --max-host-bytes and +// never evicts them. The registered fat binary is among them, and its size +// follows the arch list the test binary was built for (over 1 MiB for the CI +// build of every arch), so a cap is the archive's cached bytes plus the budget +// the test means to give the allocations. +uint64_t hrr_archive_cache_bytes(const fs::path& capture_root) { + // The capture root holds one pid-/ archive; the blobs sit inside it. + const fs::path archive = hrr_single_process_archive(capture_root); + uint64_t total = 0; + for (const char* sub : {"blobs", "code_objects"}) { + std::error_code ec; + if (!fs::is_directory(archive / sub, ec)) continue; + for (fs::recursive_directory_iterator it(archive / sub, ec), end; !ec && it != end; + it.increment(ec)) { + std::error_code fec; + if (it->is_regular_file(fec)) total += fs::file_size(it->path(), fec); + } + } + return total; +} +std::string hrr_host_cap(const fs::path& archive, uint64_t alloc_bytes) { + return std::to_string(hrr_archive_cache_bytes(archive) + alloc_bytes); +} + constexpr size_t kBudgetBuf = 2u << 20; constexpr int kBudgetRounds = 3; // Capture is not capped; only replay is. -void hrr_budget_replay(const char* direct_case, const char* dir, const char* cap_bytes, - const char* api) { +void hrr_budget_replay(const char* direct_case, const char* dir, const char* api) { ScopedDir cap{hrr_temp_dir(dir)}; hrr_capture_direct(direct_case, cap.path, 5); for (const bool over : {false, true}) { const std::string cap_arg = - std::string("--max-host-bytes ") + (over ? "1048576" : cap_bytes); + std::string("--max-host-bytes ") + + hrr_host_cap(cap.path, over ? (1u << 20) : (3u << 20)); auto [ret, out] = hrr_playback_merged(cap.path, cap_arg); INFO(cap_arg << ": playback exit " << ret << ", output:\n" << out); if (over) { @@ -85,12 +109,12 @@ TEST_CASE("Unit_HRR_HostBudgetMemCreate_Direct", "[.][hrr-direct]") { // Within a one-buffer cap the free returns the charge; under the buffer, the // first allocation is refused. HRR_TEST_CASE(Unit_HRR_HostBudgetManagedRoundtrip) { - hrr_budget_replay("Unit_HRR_HostBudgetManaged_Direct", "hrr_budget_managed", "3145728", + hrr_budget_replay("Unit_HRR_HostBudgetManaged_Direct", "hrr_budget_managed", "hipMallocManaged"); } HRR_TEST_CASE(Unit_HRR_HostBudgetMemCreateRoundtrip) { - hrr_budget_replay("Unit_HRR_HostBudgetMemCreate_Direct", "hrr_budget_memcreate", "3145728", + hrr_budget_replay("Unit_HRR_HostBudgetMemCreate_Direct", "hrr_budget_memcreate", "hipMemCreate (host location)"); } @@ -98,7 +122,7 @@ HRR_TEST_CASE(Unit_HRR_HostBudgetMemCreateRoundtrip) { // hipMemUnmap. The workload maps h1, releases it, then creates h2 before the // unmap: with the hold the 3 MiB replay is refused at h2 (h1's 2 MiB is still // charged); without it the release would have returned the charge and h2 fits. -// A 5 MiB cap takes h1 + h2 plus the replay's cached blobs, so the replay +// A cap of 5 MiB over the replay's cached blobs takes h1 + h2, so the replay // completes. TEST_CASE("Unit_HRR_HostBudgetMemMapHold_Direct", "[.][hrr-direct]") { HRR_HIP_CHECK(hipSetDevice(0)); @@ -138,7 +162,8 @@ HRR_TEST_CASE(Unit_HRR_HostBudgetMemMapHoldRoundtrip) { hrr_capture_direct("Unit_HRR_HostBudgetMemMapHold_Direct", cap.path, 5); for (const bool refuse : {false, true}) { const std::string cap_arg = - std::string("--max-host-bytes ") + (refuse ? "3145728" : "5242880"); + std::string("--max-host-bytes ") + + hrr_host_cap(cap.path, refuse ? (3u << 20) : (5u << 20)); auto [ret, out] = hrr_playback_merged(cap.path, cap_arg); INFO(cap_arg << ": playback exit " << ret << ", output:\n" << out); if (refuse) { @@ -165,7 +190,8 @@ HRR_TEST_CASE(Unit_HRR_HostBudgetManagedLeakRoundtrip) { ScopedDir cap{hrr_temp_dir("hrr_budget_managed_leak")}; hrr_capture_direct("Unit_HRR_HostBudgetManagedLeak_Direct", cap.path, 5); auto [ret, out] = hrr_playback_merged( - cap.path, "--kernel-filter hrr_no_such_kernel --max-host-bytes 3145728"); + cap.path, "--kernel-filter hrr_no_such_kernel --max-host-bytes " + + hrr_host_cap(cap.path, 3u << 20)); INFO("playback exit " << ret << ", output:\n" << out); CHECK(ret == 0); CHECK(out.find("[HRR] Fatal") == std::string::npos); From 9f285d5941c29bd9352ca403e542a2d55acbfc74 Mon Sep 17 00:00:00 2001 From: Daniel Olsson Date: Fri, 6 Nov 2026 16:30:38 +0100 Subject: [PATCH 89/89] HRR tests: make CaptureCrashDuringFork_Direct non-dumpable The CI hang on gfx125x (#15217) is the watchdog killing the parent while Linux writes a 12.8 GiB core. PR_SET_DUMPABLE 0 avoids the dump and the SIGSEGV is still raised, so the test's exit-code checks hold. Re-enable Unit_HRR_CaptureCrashDuringFork on gfx125x. Co-Authored-By: Claude Sonnet 5.5 --- projects/hrr/tests/config/integration.yaml | 7 +------ projects/hrr/tests/integration/hrr_workload_test.cc | 3 +++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/projects/hrr/tests/config/integration.yaml b/projects/hrr/tests/config/integration.yaml index fe3c3f40814d..539cac34681e 100644 --- a/projects/hrr/tests/config/integration.yaml +++ b/projects/hrr/tests/config/integration.yaml @@ -83,12 +83,7 @@ cases: Unit_HRR_CaptureCapabilities_Direct: {} Unit_HRR_CaptureClosesArchiveDirs: {} Unit_HRR_CaptureClosesArchiveDirs_Direct: {} - Unit_HRR_CaptureCrashDuringFork: - skip: - - when: {arch: ["gfx125*"], emulator: true} - kind: disabled - reason: "The parent hangs in the crash path under rocJitsu, the watchdog kills it, and the test sees exit 137 instead of 139" - issue: "https://github.com/ROCm/rocm-systems/issues/15217" + Unit_HRR_CaptureCrashDuringFork: {} Unit_HRR_CaptureCrashDuringFork_Direct: {} Unit_HRR_CaptureCrashOnSmallStack: {} Unit_HRR_CaptureCrashSmallStack_Direct: {} diff --git a/projects/hrr/tests/integration/hrr_workload_test.cc b/projects/hrr/tests/integration/hrr_workload_test.cc index bdd742f26bee..5d9f28cd46f5 100644 --- a/projects/hrr/tests/integration/hrr_workload_test.cc +++ b/projects/hrr/tests/integration/hrr_workload_test.cc @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -4128,6 +4129,8 @@ TEST_CASE("Unit_HRR_CaptureCrashDuringFork_Direct", "[.][hrr-direct]") { // The crashes are the point; core files are not. struct rlimit no_core{0, 0}; (void)setrlimit(RLIMIT_CORE, &no_core); + // Non-dumpable too: the parent's core dump outlasts the CI watchdog (#15217). + (void)prctl(PR_SET_DUMPABLE, 0); HRR_HIP_CHECK(hipSetDevice(0)); int* d = nullptr;