diff --git a/.github/workflows/hrr-ci.yml b/.github/workflows/hrr-ci.yml new file mode 100644 index 000000000000..8d5164c7e103 --- /dev/null +++ b/.github/workflows/hrr-ci.yml @@ -0,0 +1,730 @@ +name: HRR CI + +# Pull requests always start the workflow. A workflow skipped by a top-level +# "paths" filter creates no check at all, so a required "HRR CI Summary" would +# wait forever on every PR that does not touch HRR. The gate job below does the +# path filtering instead, and the summary job always reports. +# +# Pushes to develop run the same suites for one reason beyond post-merge signal: +# a pull request can only restore caches from its own ref and from the base +# branch, so without a develop run every PR would start with a cold ccache. +on: + pull_request: + push: + branches: + - develop + paths: + - "projects/hrr/**" + - "!projects/hrr/skills/**" + - "projects/clr/hipamd/src/hrr/**" + - ".github/workflows/hrr-ci.yml" + - ".github/actions/rocprofiler-sdk-util/resolve-ci-config/**" + workflow_dispatch: + inputs: + capture: + description: "Build CLR with HRR capture enabled (false skips every HRR CI job)" + type: boolean + default: true + +concurrency: + group: hrr-ci-${{ github.event.pull_request.number || github.ref }} + # Never cancel a develop run: it is what saves the ccache for later PRs. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + # One fat binary covers every family that has a test runner, so the GPU jobs + # download a single artifact instead of rebuilding per architecture. + # + # Every arch a pool can hand out has to be listed, not one representative per + # family: linux-gfx110X-gpu-rocm and linux-gfx120X-gpu-rocm are wildcard + # pools, and a machine whose arch is missing from the binary fails every + # workload with hipErrorInvalidImage (200) rather than skipping. gfx1250 is + # carried so the binary is ready once its runner is enabled below. + HRR_ARCHS: "gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1153;gfx1200;gfx1201;gfx1250" + ROCM_INDEX_URL: "https://rocm.nightlies.amd.com/whl-multi-arch/" + +jobs: + # --------------------------------------------------------------------------- + # Capture gate. HRR capture is compiled into amdhip64 only when + # HIP_HRR_CAPTURE_ENABLED is defined (hip_capture.cpp, hip_capture_init); the + # tree has no CMake option for it, and this workflow is what defines it for + # the CLR build below. When capture is not enabled there is nothing for HRR CI + # to test, so every job below is skipped, not failed. A skipped job counts as + # passing for branch protection, and "HRR CI Summary" resolves it explicitly. + # + # Capture is off when the repository variable HRR_CAPTURE_ENABLED is "false" + # or "0", when a manual run passes capture=false, or when the capture guard + # is no longer in the CLR sources. The jobs are also skipped when a pull + # request touches nothing HRR owns (the path filter, done here rather than in + # the "on:" block so that HRR CI Summary still reports). + # --------------------------------------------------------------------------- + gate: + name: "Capture enabled?" + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read + outputs: + enabled: ${{ steps.decide.outputs.enabled }} + steps: + - name: Checkout capture sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: projects/clr/hipamd/src/hrr/hip_capture.cpp + sparse-checkout-cone-mode: false + set-safe-directory: true + + - name: Decide + id: decide + env: + VAR_ENABLED: ${{ vars.HRR_CAPTURE_ENABLED }} + INPUT_ENABLED: ${{ github.event.inputs.capture }} + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + enabled=true + reason="capture is enabled" + if [ -n "${PR_NUMBER}" ]; then + # The files API lists at most 3000 files; a PR that large is treated + # as relevant rather than guessed at. + gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files" \ + --jq '.[].filename' > changed-files.txt + if ! python3 - changed-files.txt <<'PY' + import sys + + INCLUDE = ( + "projects/hrr/", + "projects/clr/hipamd/src/hrr/", + ".github/workflows/hrr-ci.yml", + ".github/actions/rocprofiler-sdk-util/resolve-ci-config/", + ) + EXCLUDE = ("projects/hrr/skills/",) + + files = [line.strip() for line in open(sys.argv[1]) if line.strip()] + relevant = len(files) >= 3000 or any( + f.startswith(INCLUDE) and not f.startswith(EXCLUDE) for f in files + ) + sys.exit(0 if relevant else 1) + PY + then + enabled=false + reason="the pull request changes no HRR path" + fi + fi + case "${VAR_ENABLED,,}" in + false|0|off|no) enabled=false; reason="repository variable HRR_CAPTURE_ENABLED=${VAR_ENABLED}" ;; + esac + case "${INPUT_ENABLED,,}" in + false) enabled=false; reason="workflow_dispatch input capture=false" ;; + esac + # Match the macro name alone, outside comments, so a rewrite such as + # "#ifdef HIP_HRR_CAPTURE_ENABLED" still counts as the guard. + if ! grep -vE '^[[:space:]]*//' projects/clr/hipamd/src/hrr/hip_capture.cpp \ + | grep -q 'HIP_HRR_CAPTURE_ENABLED'; then + enabled=false + reason="hip_capture.cpp no longer mentions HIP_HRR_CAPTURE_ENABLED" + echo "::warning::HRR CI skipped: the capture guard HIP_HRR_CAPTURE_ENABLED was not found in hip_capture.cpp. If capture was renamed or made unconditional, update this gate." + fi + echo "enabled=${enabled}" >> "$GITHUB_OUTPUT" + echo "HRR CI: run=${enabled} (${reason})" | tee -a "$GITHUB_STEP_SUMMARY" + # --------------------------------------------------------------------------- + # Host-only unit suite. These 23 cases link nothing but libstdc++/libc: they + # include for types but issue no HIP calls, so they need + # neither a GPU nor the capture library and can run on any platform. + # --------------------------------------------------------------------------- + unit: + name: "Unit (${{ matrix.os }})" + needs: [gate] + if: ${{ needs.gate.outputs.enabled == 'true' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2022] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + projects/hrr + set-safe-directory: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install ROCm SDK (headers and toolchain only) + shell: bash + run: | + pip install --no-cache-dir --index-url "${ROCM_INDEX_URL}" "rocm[devel]" + rocm-sdk init + echo "ROCM_SDK=$(python -c 'import _rocm_sdk_devel,os;print(os.path.dirname(_rocm_sdk_devel.__file__))')" >> "$GITHUB_ENV" + + - name: Configure + shell: bash + run: | + extra=() + if [ "${RUNNER_OS}" = "Windows" ]; then + # The runner builds with MinGW, whose libstdc++/libgcc/winpthread + # DLLs are not on PATH for later steps, so a dynamically linked + # binary fails to start at all. Link them in instead. + extra+=(-DCMAKE_EXE_LINKER_FLAGS=-static) + fi + cmake -S projects/hrr -B build -GNinja -Wno-dev \ + "${extra[@]}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${ROCM_SDK}" \ + -DROCM_PATH="${ROCM_SDK}" \ + -DHRR_BUILD_PLAYBACK=ON \ + -DHRR_BUILD_TESTS=ON \ + -DHRR_BUILD_INTEGRATION_TESTS=OFF + + - name: Build + run: cmake --build build -j + + - name: Assert the expected unit cases are present + shell: bash + run: | + # A case behind an #if leaves the binary with no signal at all -- not + # failed, not skipped, absent -- so a fully green suite can hide it. + # Assert the compiled set before trusting the run below. + if [ "${RUNNER_OS}" = "Windows" ]; then + PATH="$(cygpath -u "${ROCM_SDK}")/bin:${PATH}" + export PATH + fi + shopt -s nullglob + bin=(build/tests/unit/hrr-unit-tests*) + if [ ${#bin[@]} -eq 0 ]; then + echo "::error::no unit binary under build/tests/unit" + exit 1 + fi + python3 projects/hrr/tests/check_expected_cases.py \ + --binary "${bin[0]}" \ + --manifest projects/hrr/tests/expected_cases_unit.txt + + - name: Run unit suite + shell: bash + run: | + # Named hrr-unit-tests.exe on Windows. Resolved with a bash glob + # rather than find, which on a Windows runner resolves to FIND.EXE. + if [ "${RUNNER_OS}" = "Windows" ]; then + # The recovery cases spawn hrr-playback.exe, which loads the SDK's + # HIP DLLs. Without them on PATH it dies with STATUS_DLL_NOT_FOUND + # before running anything. + PATH="$(cygpath -u "${ROCM_SDK}")/bin:${PATH}" + export PATH + fi + shopt -s nullglob + bin=(build/tests/unit/hrr-unit-tests*) + if [ ${#bin[@]} -eq 0 ]; then + echo "::error::no unit binary under build/tests/unit" + ls -R build/tests || true + exit 1 + fi + ls -l "${bin[0]}" + python3 projects/hrr/tests/run_catch2.py \ + --xml unit-${{ matrix.os }}.xml \ + -- "${bin[0]}" + + # The helpers the suites below rely on (JUnit wrapper, summary table, + # expected-case check). They are POSIX-process code; once is enough. + - name: Test the CI helper scripts + if: ${{ runner.os == 'Linux' }} + run: python3 -m unittest discover -s projects/hrr/tests -p 'test_*.py' -v + + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: results-unit-${{ matrix.os }} + path: unit-${{ matrix.os }}.xml + retention-days: 3 + + # --------------------------------------------------------------------------- + # Resolve the GPU runner labels from the shared CI config, the same way the + # rocprofiler-sdk workflows do. + # --------------------------------------------------------------------------- + ci_config: + name: "Resolve runners" + needs: [gate] + if: ${{ needs.gate.outputs.enabled == 'true' }} + runs-on: ubuntu-24.04 + outputs: + gfx90a_runner: ${{ steps.runner.outputs.gfx90a_runner }} + gfx94x_runner: ${{ steps.runner.outputs.gfx94x_runner }} + gfx950_runner: ${{ steps.runner.outputs.gfx950_runner }} + gfx103x_runner: ${{ steps.runner.outputs.gfx103x_runner }} + gfx110x_runner: ${{ steps.runner.outputs.gfx110x_runner }} + gfx120x_runner: ${{ steps.runner.outputs.gfx120x_runner }} + gfx1151_runner: ${{ steps.runner.outputs.gfx1151_runner }} + steps: + - name: Checkout workflow actions + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: .github/actions/rocprofiler-sdk-util/resolve-ci-config + set-safe-directory: true + + - name: Resolve runner labels + id: runner + uses: ./.github/actions/rocprofiler-sdk-util/resolve-ci-config + + # --------------------------------------------------------------------------- + # Builds CLR (which brings in-tree ROCr with it) to get an amdhip64 carrying + # the HRR capture shims, then projects/hrr against the SDK. The capture build + # is substituted at run time via LD_LIBRARY_PATH, not linked here. Produces + # one multi-architecture artifact consumed by every GPU job below. + # --------------------------------------------------------------------------- + build: + name: "Build (multi-arch)" + needs: [gate] + if: ${{ needs.gate.outputs.enabled == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} + # The integration binaries embed absolute paths into this workspace, so a + # containerised GPU job has to recreate it (see "Recreate build path"). + outputs: + workspace: ${{ github.workspace }} + # The nightly index moves every day. The integration jobs install exactly + # this version so the capture amdhip64 never runs against a newer ROCr + # than the one it was built with, however long a GPU queue is. + sdk_version: ${{ steps.sdk.outputs.version }} + runs-on: azure-linux-scale-rocm + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + # Hash the compiler by content, not mtime: the SDK wheel is reinstalled + # every run, so an mtime check would miss on every single object. + CCACHE_COMPILERCHECK: content + CCACHE_MAXSIZE: "5G" + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + projects/hrr + projects/clr + projects/hip + projects/rocr-runtime + set-safe-directory: true + + # ROCclr pulls in OpenGL through ROCclrHSA.cmake even with CLR_BUILD_OCL + # off, so the GL development packages are required to configure CLR. + - name: Install system dependencies + run: | + sudo apt-get update -q + sudo apt-get install -y --no-install-recommends \ + ninja-build pkg-config libnuma-dev libelf-dev libdrm-dev \ + libgl1-mesa-dev libglx-dev mesa-common-dev \ + libclang-rt-18-dev ccache + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install ROCm SDK + id: sdk + run: | + # CppHeaderParser is required by hipamd/src/CMakeLists.txt. + pip install --no-cache-dir CppHeaderParser + pip install --no-cache-dir --index-url "${ROCM_INDEX_URL}" "rocm[libraries,devel]" + rocm-sdk init + echo "version=$(python -c 'import importlib.metadata as m; print(m.version("rocm"))')" >> "$GITHUB_OUTPUT" + echo "ROCM_SDK=$(python -c 'import _rocm_sdk_devel,os;print(os.path.dirname(_rocm_sdk_devel.__file__))')" >> "$GITHUB_ENV" + + # Nothing survived between runs before this: the artifact fan-out only + # reuses a build WITHIN a run. ccache is the house convention (15 + # workflows) and is carried across runs by actions/cache (9 workflows) + # because azure-linux-scale-rocm is ephemeral and keeps no state. + # + # This cache is a SPEED optimisation only and can never make a stale + # binary look fresh. The key exists to pick a good starting point; it + # is ccache itself that decides reuse, and it keys on the preprocessed + # source plus the command line plus the compiler contents. A changed + # input is therefore a miss by construction -- restoring a stale cache + # costs misses, never a wrong object. Do not replace this with a + # "skip the build if the key matches" scheme: that trades a correctness + # property for the same speed. + # + # Restore everywhere, save only from a push to develop. A pull request can + # restore its own caches and the base branch's but never write to + # develop's scope, and a per-run entry of up to 5 GB from every PR run + # would evict other workflows' caches. One develop entry per push seeds + # every PR instead. + - name: Restore ccache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ github.workspace }}/.ccache + key: hrr-ccache-${{ runner.os }}-${{ github.run_id }} + restore-keys: | + hrr-ccache-${{ runner.os }}- + + - name: Zero ccache statistics + run: ccache --zero-stats + + - name: Build amdhip64 with the HRR capture shims + run: | + cmake -S projects/clr -B build/clr -GNinja -Wno-dev \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_FLAGS=-DHIP_HRR_CAPTURE_ENABLED \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${ROCM_SDK}" \ + -DROCM_PATH="${ROCM_SDK}" \ + -DHIP_COMMON_DIR="${GITHUB_WORKSPACE}/projects/hip" \ + -DHIP_PLATFORM=amd \ + -DCLR_BUILD_HIP=ON \ + -DCLR_BUILD_OCL=OFF + cmake --build build/clr --target amdhip64 -j + + - name: Build projects/hrr for every testable architecture + run: | + # C and C++ only. CMAKE_HIP_COMPILER_LAUNCHER is deliberately NOT + # set: HIP device compilation fans one TU out to a code object per + # --offload-arch and bundles them, and ccache's handling of that + # multi-output bundling is not something this workflow has proven. + # A wrong hit there would mean testing a binary that does not match + # its source, which is exactly the failure mode this cache is + # required not to introduce. The C++ half (CLR/amdhip64, the bulk of + # the wall clock) is cached; extend to HIP only after a run proves + # the arch-embedding check still passes on a fully cached build. + cmake -S projects/hrr -B build/hrr -GNinja -Wno-dev \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${ROCM_SDK}" \ + -DROCM_PATH="${ROCM_SDK}" \ + -DCMAKE_HIP_COMPILER="${ROCM_SDK}/bin/amdclang++" \ + -DCMAKE_HIP_FLAGS="--rocm-device-lib-path=${ROCM_SDK}/lib/llvm/amdgcn/bitcode" \ + -DGPU_TARGETS="${HRR_ARCHS}" \ + -DCMAKE_HIP_ARCHITECTURES="${HRR_ARCHS}" \ + -DHRR_BUILD_PLAYBACK=ON \ + -DHRR_BUILD_TESTS=ON \ + -DHRR_BUILD_INTEGRATION_TESTS=ON + cmake --build build/hrr -j + + - name: ccache statistics + if: always() + run: ccache --show-stats --verbose || ccache --show-stats + + - name: Assert the expected integration cases are present + run: | + # Listing needs no GPU: Catch2 registers cases in static initialisers + # and --list-tests never calls into HIP. Checked here rather than in + # the integration jobs so a compiled-out case fails once, straight + # away, instead of nine times after the fan-out. + python3 projects/hrr/tests/check_expected_cases.py \ + --binary build/hrr/tests/integration/hrr-integration-tests \ + --manifest projects/hrr/tests/expected_cases_integration.txt + + - name: Confirm every architecture is embedded + run: | + found=$("${ROCM_SDK}"/lib/llvm/bin/llvm-objdump --offloading \ + build/hrr/tests/integration/hrr-integration-tests \ + | grep -o 'gfx[0-9a-z]*' | sort -u | tr '\n' ' ') + echo "embedded: ${found}" + for arch in ${HRR_ARCHS//;/ }; do + case " ${found} " in + *" ${arch} "*) ;; + *) echo "::error::${arch} missing from the fat binary"; exit 1 ;; + esac + done + + - name: Save ccache + if: ${{ !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/develop' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ github.workspace }}/.ccache + key: hrr-ccache-${{ runner.os }}-${{ github.run_id }} + + - name: Upload build + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hrr-build + path: | + build/hrr + build/clr/hipamd/lib + retention-days: 3 + + # --------------------------------------------------------------------------- + # Integration suite on every family that has a test runner. + # --------------------------------------------------------------------------- + integration: + name: "Integration (${{ matrix.family }})" + needs: [gate, ci_config, build] + if: ${{ needs.gate.outputs.enabled == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} + runs-on: ${{ matrix.runner }} + # The suite runs in about 90 seconds. These are scarce single-GPU machines + # and the default timeout is six hours, so a workload that deadlocks on one + # part would hold a runner for the rest of the day and publish no log to + # say why. Fail instead, while the log still gets uploaded. + timeout-minutes: 30 + # The OSSCI pod pools (gfx90a, gfx950) expose /dev/kfd only to a job with + # a container spec: run bare there and hipGetDeviceCount returns 100 with + # no devices. Every other pool is a bare machine and gets no container, + # since a null image runs the job directly on the runner. + container: + image: ${{ matrix.container && 'ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98' || null }} + options: >- + --ipc host + --group-add video + --device /dev/kfd + --device /dev/dri + --group-add 110 + --group-add 993 + --group-add 992 + --ulimit memlock=-1:-1 + --security-opt seccomp=unconfined + --env-file /etc/podinfo/gha-gpu-isolation-settings + --user 0:0 + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: + - { family: gfx90a, container: true, runner: "${{ needs.ci_config.outputs.gfx90a_runner }}" } + # gfx94x uses the resolved 1-GPU runner: the 8-GPU gfx942 pool + # exposes /dev/kfd only to a job with a container spec. + - { family: gfx94x, runner: "${{ needs.ci_config.outputs.gfx94x_runner }}" } + # The T3 tier needs two visible devices and there is no 2-GPU pool, + # so gfx950 takes an 8-GPU machine; every other family skips T3. + # Do not pin HIP_VISIBLE_DEVICES: on the CCS pool fixed indices can + # be empty even though the job has GPUs. + - { family: gfx950, container: true, runner: "linux-gfx950-8gpu-ccs-ossci-rocm" } + - { family: gfx103x, runner: "${{ needs.ci_config.outputs.gfx103x_runner }}" } + - { family: gfx110x, runner: "${{ needs.ci_config.outputs.gfx110x_runner }}" } + - { family: gfx120x, runner: "${{ needs.ci_config.outputs.gfx120x_runner }}" } + - { family: gfx1151, runner: "${{ needs.ci_config.outputs.gfx1151_runner }}" } + # Not exposed by resolve-ci-config; taken from TheRock's family matrix. + - { family: gfx1150, runner: "linux-gfx1150-gpu-rocm" } + - { family: gfx1153, runner: "linux-gfx1153-gpu-rocm" } + # gfx1250 (MI455) is withheld until linux-mi455-gpu-rocm is healthy. + # If it is wanted sooner, add it as a separate non-blocking job, as + # therock-ci.yml does, so the report job never waits on that queue: + # + # - { family: gfx1250, runner: "linux-mi455-gpu-rocm" } + steps: + # Only the test runner: this job compiles nothing and takes its binaries + # from the build artifact. It has to come first, because checkout cleans + # the workspace and would delete an artifact unpacked before it. + - name: Checkout test runner + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + projects/hrr/tests/run_catch2.py + sparse-checkout-cone-mode: false + set-safe-directory: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + # Invoked through "python -m" and by script path throughout: on some of + # these self-hosted machines the console-script wrappers cannot execute, + # and the system Python has no ensurepip to build a virtualenv with. + - name: Install ROCm SDK + env: + SDK_VERSION: ${{ needs.build.outputs.sdk_version }} + run: | + # devel as well as libraries: rocm-sdk init refuses to run without it. + # The version the build job used, not whatever the index has now. + python -m pip install --no-cache-dir \ + --index-url "${ROCM_INDEX_URL}" "rocm[libraries,devel]${SDK_VERSION:+==${SDK_VERSION}}" + # Run as a module where possible. Falling back to the script means + # locating it: some of these machines install into a --user bin that + # is not on PATH, leaving command -v empty. + if ! python -m rocm_sdk init; then + cli="$(command -v rocm-sdk || true)" + if [ -z "${cli}" ]; then + cli="$(python -c 'import site;print(site.USER_BASE)')/bin/rocm-sdk" + fi + python "${cli}" init + fi + echo "ROCM_SDK=$(python -c 'import _rocm_sdk_devel,os;print(os.path.dirname(_rocm_sdk_devel.__file__))')" >> "$GITHUB_ENV" + + # Unpacked under build/ so the tree matches the build machine's layout. + # The suite spawns each workload by re-execing HRR_TEST_EXE, which CMake + # bakes in as an absolute path, so the binary has to sit where it was + # built or every workload dies in execvp with 127. + - name: Download build + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: hrr-build + path: build + + # In a container the workspace moves (/__w/...), which would leave the + # baked-in HRR_TEST_EXE and HRR_PLAYBACK_EXE pointing nowhere. + - name: Recreate build path + env: + BUILD_WORKSPACE: ${{ needs.build.outputs.workspace }} + run: | + if [ -n "${BUILD_WORKSPACE}" ] && [ "${BUILD_WORKSPACE}" != "${PWD}" ]; then + mkdir -p "$(dirname "${BUILD_WORKSPACE}")" + ln -sfn "${PWD}" "${BUILD_WORKSPACE}" + fi + + - name: Restore executable bits + run: chmod +x build/hrr/playback/hrr-playback build/hrr/tests/*/hrr-*tests + + # CMake gives the binaries a RUNPATH to the SDK's lib directory, where + # the plain amdhip64 lives. LD_LIBRARY_PATH is searched before RUNPATH, so + # putting the capture build first here is what selects it. + - name: Run integration suite + run: | + # hipamd/lib holds the amdhip64 carrying the capture shims and must + # precede the SDK, which ships a plain build under the same soname. + # No ROCr preload here: CLR was built against the SDK's ROCr, so the + # two agree and there is no in-tree runtime to substitute. + # Keep the runner's existing path: setup-python puts libpython there, + # and replacing it makes python3 fail to start on some GPU machines. + export LD_LIBRARY_PATH="${PWD}/build/clr/hipamd/lib:${ROCM_SDK}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + # A GPU-labelled runner with no visible device is infrastructure + # failure, not 67 unsupported capabilities. Still run the three + # CPU-only integration checks and publish their JUnit, then fail the + # job so the table cannot make a broken runner look green. + # Exit 1 is "confirmed no device". Any other probe failure (broken + # ctypes, missing libpython) is not a device count and must not + # skip the GPU suite. + set +e + python3 - <<'PY' + import sys + + try: + import ctypes + hip = ctypes.CDLL("libamdhip64.so.7") + count = ctypes.c_int() + status = hip.hipGetDeviceCount(ctypes.byref(count)) + except Exception as exc: + print(f"GPU probe unavailable: {type(exc).__name__}: {exc}", file=sys.stderr) + sys.exit(2) + print(f"hipGetDeviceCount: status={status}, devices={count.value}") + sys.exit(0 if status == 0 and count.value > 0 else 1) + PY + probe_rc=$? + set -e + if [ "$probe_rc" -eq 1 ]; then + echo "::error::GPU runner has no visible ROCm-capable device" + python3 projects/hrr/tests/run_catch2.py \ + --xml integration-${{ matrix.family }}.xml \ + -- ./build/hrr/tests/integration/hrr-integration-tests "[cpu]" + exit 1 + fi + # Keep passing logs to one result line per case. The wrapper captures + # Catch2 and subprocess output, and only emits it when the suite fails. + # Time out inside the job's 30 minutes, so a hang prints the cases + # that finished instead of being cancelled with no output. + # + # The selector matters: Catch2 leaves out hidden "[.]" cases unless a + # tag names them, and every API-matrix tier, the kernel-argument + # regression and the HSA first-touch case are hidden drivers tagged + # [hrr]. "[hrr]~[direct]" runs them, leaves out the [hrr][direct] + # workloads they spawn as subprocesses, and (being an exact tag match) + # also leaves out the [hrr-direct] workloads and the disabled + # [hrr-repro] cases. + python3 projects/hrr/tests/run_catch2.py \ + --xml integration-${{ matrix.family }}.xml --timeout 1200 \ + -- ./build/hrr/tests/integration/hrr-integration-tests "[hrr]~[direct]" + + - name: Upload results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: results-integration-${{ matrix.family }} + path: integration-${{ matrix.family }}.xml + retention-days: 3 + + # --------------------------------------------------------------------------- + # Per-test reporting for every job above. Collected centrally because the + # publishing action is a container action and so cannot run on the Windows + # runners. + # --------------------------------------------------------------------------- + report: + name: "Test results" + needs: [gate, unit, integration] + if: ${{ always() && needs.gate.outputs.enabled == 'true' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + checks: write + pull-requests: write + steps: + - name: Checkout summary script + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + projects/hrr/tests/summarize_junit.py + sparse-checkout-cone-mode: false + set-safe-directory: true + + - name: Download all results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: results-* + merge-multiple: true + + - name: Platform / family table + if: always() + run: | + # Written first so it sits above EnricoMi's suite dump on the job + # summary and on the HRR test results check page. + python3 projects/hrr/tests/summarize_junit.py \ + --dir . \ + --summary "${GITHUB_STEP_SUMMARY}" + + - name: Publish + # Runs even when the table step exits 1 on unreadable XML. + if: always() + uses: EnricoMi/publish-unit-test-result-action@d0a4676d0e0b938bc201470d88276b7c74c712b3 # v2.24.0 + with: + files: "*.xml" + check_name: "HRR test results" + comment_mode: "off" + # Fork pull requests get a read-only token, and creating a check run + # then fails with "Resource not accessible by integration". They keep + # the job summary, which needs no token. + check_run: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork) }} + job_summary: true + report_individual_runs: true + deduplicate_classes_by_file_name: true + + # --------------------------------------------------------------------------- + # Single required-check target. Passes when capture is off (everything above + # was skipped) and when every job that ran succeeded; fails on any failure or + # cancellation, and on a skip that is not explained by the gate. Fork PRs skip + # build and integration on purpose, so those skips are accepted there. + # --------------------------------------------------------------------------- + hrr_ci_summary: + name: "HRR CI Summary" + if: always() + needs: [gate, unit, ci_config, build, integration, report] + runs-on: ubuntu-24.04 + steps: + - name: Evaluate + env: + NEEDS_JSON: ${{ toJSON(needs) }} + IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork }} + run: | + python3 - <<'PY' + import json, os, sys + + needs = json.loads(os.environ["NEEDS_JSON"]) + result = {name: job["result"] for name, job in needs.items()} + print(json.dumps(result, indent=2)) + + if result["gate"] != "success": + print("::error::capture gate did not complete: " + result["gate"]) + sys.exit(1) + if needs["gate"]["outputs"].get("enabled") != "true": + print("HRR CI was skipped by the gate (capture off, or no HRR path changed).") + sys.exit(0) + + fork = os.environ["IS_FORK"] == "true" + optional = {"build", "integration"} if fork else set() + bad = [n for n, r in result.items() + if r not in ("success",) and not (r == "skipped" and n in optional)] + if bad: + print("::error::HRR CI jobs not successful: " + ", ".join(sorted(bad))) + sys.exit(1) + PY diff --git a/projects/clr/hipamd/src/hrr/hip_capture_generated.cpp b/projects/clr/hipamd/src/hrr/hip_capture_generated.cpp index 5ab1e7017207..de58e05cdbce 100644 --- a/projects/clr/hipamd/src/hrr/hip_capture_generated.cpp +++ b/projects/clr/hipamd/src/hrr/hip_capture_generated.cpp @@ -7788,6 +7788,19 @@ static hipError_t capture_hipModuleEnumerateFunctions(hipFunction_t* functions, return r; } +// Generated shim +static hipError_t capture_hipDeviceFlushGPUDirectRDMAWrites(enum hipFlushGPUDirectRDMAWritesTarget target, enum hipFlushGPUDirectRDMAWritesScope scope) { + hipError_t r = g_real_table.hipDeviceFlushGPUDirectRDMAWrites_fn(target, scope); + if (r == hipSuccess) { + hrr_args_hipDeviceFlushGPUDirectRDMAWrites a{}; + a.ret = static_cast(r); + a.target = static_cast(target); + a.scope = static_cast(scope); + hrr_cap::writer::write_event_raw(HRR_API_HIPDEVICEFLUSHGPUDIRECTRDMAWRITES, &a.hdr, sizeof(a)); + } + return r; +} + // Generated shim static hipError_t capture___hipPopCallConfiguration(dim3* gridDim, dim3* blockDim, size_t* sharedMem, hipStream_t* stream) { hipError_t r = g_real_compiler_table.__hipPopCallConfiguration_fn(gridDim, blockDim, sharedMem, stream); @@ -8523,6 +8536,7 @@ void hip_capture_build_table() { g_cap_table.hipDeviceGetLuid_fn = capture_hipDeviceGetLuid; g_cap_table.hipInitDevice_fn = capture_hipInitDevice; g_cap_table.hipModuleEnumerateFunctions_fn = capture_hipModuleEnumerateFunctions; + g_cap_table.hipDeviceFlushGPUDirectRDMAWrites_fn = capture_hipDeviceFlushGPUDirectRDMAWrites; } void hip_capture_build_compiler_table() { diff --git a/projects/hrr/include/hrr/hrr_api_args.h b/projects/hrr/include/hrr/hrr_api_args.h index 5ed6e4e78c76..5244c89bd7b2 100644 --- a/projects/hrr/include/hrr/hrr_api_args.h +++ b/projects/hrr/include/hrr/hrr_api_args.h @@ -5820,6 +5820,14 @@ typedef struct { uint64_t module; } hrr_args_hipModuleEnumerateFunctions; +/* hipError_t hipDeviceFlushGPUDirectRDMAWrites(enum hipFlushGPUDirectRDMAWritesTarget target, enum hipFlushGPUDirectRDMAWritesScope scope) */ +typedef struct { + hrr_event_header hdr; + int32_t ret; + uint64_t /* enum hipFlushGPUDirectRDMAWritesTarget */ target; + uint64_t /* enum hipFlushGPUDirectRDMAWritesScope */ scope; +} hrr_args_hipDeviceFlushGPUDirectRDMAWrites; + /* ---- API id enumeration ---- */ typedef enum hrr_api_id { HRR_API_HIPAPINAME = 0, @@ -6367,16 +6375,17 @@ typedef enum hrr_api_id { HRR_API_HIPDEVICEGETLUID = 542, HRR_API_HIPINITDEVICE = 543, HRR_API_HIPMODULEENUMERATEFUNCTIONS = 544, - HRR_API_HIPPOPCALLCONFIGURATION = 545, - HRR_API_HIPPUSHCALLCONFIGURATION = 546, - HRR_API_HIPREGISTERFATBINARY = 547, - HRR_API_HIPREGISTERFUNCTION = 548, - HRR_API_HIPREGISTERMANAGEDVAR = 549, - HRR_API_HIPREGISTERSURFACE = 550, - HRR_API_HIPREGISTERTEXTURE = 551, - HRR_API_HIPREGISTERVAR = 552, - HRR_API_HIPUNREGISTERFATBINARY = 553, - HRR_API_COUNT = 554 + HRR_API_HIPDEVICEFLUSHGPUDIRECTRDMAWRITES = 545, + HRR_API_HIPPOPCALLCONFIGURATION = 546, + HRR_API_HIPPUSHCALLCONFIGURATION = 547, + HRR_API_HIPREGISTERFATBINARY = 548, + HRR_API_HIPREGISTERFUNCTION = 549, + HRR_API_HIPREGISTERMANAGEDVAR = 550, + HRR_API_HIPREGISTERSURFACE = 551, + HRR_API_HIPREGISTERTEXTURE = 552, + HRR_API_HIPREGISTERVAR = 553, + HRR_API_HIPUNREGISTERFATBINARY = 554, + HRR_API_COUNT = 555 } hrr_api_id_t; /* Array of API names indexed by hrr_api_id_t */ @@ -6927,6 +6936,7 @@ const char* const hrr_api_names[HRR_API_COUNT] = { "hipDeviceGetLuid", "hipInitDevice", "hipModuleEnumerateFunctions", + "hipDeviceFlushGPUDirectRDMAWrites", "__hipPopCallConfiguration", "__hipPushCallConfiguration", "__hipRegisterFatBinary", diff --git a/projects/hrr/playback/hip_playback_generated.cpp b/projects/hrr/playback/hip_playback_generated.cpp index ca58638644fa..ae82ce0aac65 100644 --- a/projects/hrr/playback/hip_playback_generated.cpp +++ b/projects/hrr/playback/hip_playback_generated.cpp @@ -5997,6 +5997,16 @@ static hipError_t playback_hipModuleEnumerateFunctions(PlaybackContext& ctx, con return hipSuccess; } +static hipError_t playback_hipDeviceFlushGPUDirectRDMAWrites(PlaybackContext& ctx, const uint8_t* payload) { + const auto* a = reinterpret_cast(payload); + hipError_t _r = (hipError_t)hipDeviceFlushGPUDirectRDMAWrites((enum hipFlushGPUDirectRDMAWritesTarget)a->target, (enum hipFlushGPUDirectRDMAWritesScope)a->scope); + if (_r != hipSuccess && a->ret != 0 && static_cast(_r) == a->ret) { + hrr_note_recorded_error(ctx, "hipDeviceFlushGPUDirectRDMAWrites", a->ret); + return hipSuccess; + } + return _r; +} + static hipError_t playback___hipPopCallConfiguration(PlaybackContext& ctx, const uint8_t* payload) { (void)ctx; (void)payload; return hipSuccess; @@ -6588,15 +6598,16 @@ const uint32_t hrr_api_min_payload_size[HRR_API_COUNT] = { static_cast(sizeof(hrr_args_hipDeviceGetLuid)), // [542] HRR_API_HIPDEVICEGETLUID static_cast(sizeof(hrr_args_hipInitDevice)), // [543] HRR_API_HIPINITDEVICE static_cast(sizeof(hrr_args_hipModuleEnumerateFunctions)), // [544] HRR_API_HIPMODULEENUMERATEFUNCTIONS - static_cast(sizeof(hrr_args___hipPopCallConfiguration)), // [545] HRR_API_HIPPOPCALLCONFIGURATION - static_cast(sizeof(hrr_args___hipPushCallConfiguration)), // [546] HRR_API_HIPPUSHCALLCONFIGURATION - static_cast(sizeof(hrr_args___hipRegisterFatBinary)), // [547] HRR_API_HIPREGISTERFATBINARY - static_cast(sizeof(hrr_args___hipRegisterFunction)), // [548] HRR_API_HIPREGISTERFUNCTION - static_cast(sizeof(hrr_args___hipRegisterManagedVar)), // [549] HRR_API_HIPREGISTERMANAGEDVAR - static_cast(sizeof(hrr_args___hipRegisterSurface)), // [550] HRR_API_HIPREGISTERSURFACE - static_cast(sizeof(hrr_args___hipRegisterTexture)), // [551] HRR_API_HIPREGISTERTEXTURE - static_cast(sizeof(hrr_args___hipRegisterVar)), // [552] HRR_API_HIPREGISTERVAR - static_cast(sizeof(hrr_args___hipUnregisterFatBinary)), // [553] HRR_API_HIPUNREGISTERFATBINARY + static_cast(sizeof(hrr_args_hipDeviceFlushGPUDirectRDMAWrites)), // [545] HRR_API_HIPDEVICEFLUSHGPUDIRECTRDMAWRITES + static_cast(sizeof(hrr_args___hipPopCallConfiguration)), // [546] HRR_API_HIPPOPCALLCONFIGURATION + static_cast(sizeof(hrr_args___hipPushCallConfiguration)), // [547] HRR_API_HIPPUSHCALLCONFIGURATION + static_cast(sizeof(hrr_args___hipRegisterFatBinary)), // [548] HRR_API_HIPREGISTERFATBINARY + static_cast(sizeof(hrr_args___hipRegisterFunction)), // [549] HRR_API_HIPREGISTERFUNCTION + static_cast(sizeof(hrr_args___hipRegisterManagedVar)), // [550] HRR_API_HIPREGISTERMANAGEDVAR + static_cast(sizeof(hrr_args___hipRegisterSurface)), // [551] HRR_API_HIPREGISTERSURFACE + static_cast(sizeof(hrr_args___hipRegisterTexture)), // [552] HRR_API_HIPREGISTERTEXTURE + static_cast(sizeof(hrr_args___hipRegisterVar)), // [553] HRR_API_HIPREGISTERVAR + static_cast(sizeof(hrr_args___hipUnregisterFatBinary)), // [554] HRR_API_HIPUNREGISTERFATBINARY }; // ============================================================ @@ -7148,13 +7159,14 @@ hrr_playback_fn_t hrr_playback_dispatch[HRR_API_COUNT] = { playback_hipDeviceGetLuid, // [542] HRR_API_HIPDEVICEGETLUID playback_hipInitDevice, // [543] HRR_API_HIPINITDEVICE playback_hipModuleEnumerateFunctions, // [544] HRR_API_HIPMODULEENUMERATEFUNCTIONS - playback___hipPopCallConfiguration, // [545] HRR_API_HIPPOPCALLCONFIGURATION - playback___hipPushCallConfiguration, // [546] HRR_API_HIPPUSHCALLCONFIGURATION - playback___hipRegisterFatBinary, // [547] HRR_API_HIPREGISTERFATBINARY - playback___hipRegisterFunction, // [548] HRR_API_HIPREGISTERFUNCTION - playback___hipRegisterManagedVar, // [549] HRR_API_HIPREGISTERMANAGEDVAR - playback___hipRegisterSurface, // [550] HRR_API_HIPREGISTERSURFACE - playback___hipRegisterTexture, // [551] HRR_API_HIPREGISTERTEXTURE - playback___hipRegisterVar, // [552] HRR_API_HIPREGISTERVAR - playback___hipUnregisterFatBinary, // [553] HRR_API_HIPUNREGISTERFATBINARY + playback_hipDeviceFlushGPUDirectRDMAWrites, // [545] HRR_API_HIPDEVICEFLUSHGPUDIRECTRDMAWRITES + playback___hipPopCallConfiguration, // [546] HRR_API_HIPPOPCALLCONFIGURATION + playback___hipPushCallConfiguration, // [547] HRR_API_HIPPUSHCALLCONFIGURATION + playback___hipRegisterFatBinary, // [548] HRR_API_HIPREGISTERFATBINARY + playback___hipRegisterFunction, // [549] HRR_API_HIPREGISTERFUNCTION + playback___hipRegisterManagedVar, // [550] HRR_API_HIPREGISTERMANAGEDVAR + playback___hipRegisterSurface, // [551] HRR_API_HIPREGISTERSURFACE + playback___hipRegisterTexture, // [552] HRR_API_HIPREGISTERTEXTURE + playback___hipRegisterVar, // [553] HRR_API_HIPREGISTERVAR + playback___hipUnregisterFatBinary, // [554] HRR_API_HIPUNREGISTERFATBINARY }; diff --git a/projects/hrr/tests/check_expected_cases.py b/projects/hrr/tests/check_expected_cases.py new file mode 100755 index 000000000000..2457d7faf1d3 --- /dev/null +++ b/projects/hrr/tests/check_expected_cases.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Assert that a Catch2 binary registers exactly the test cases we expect. + +A case behind an #if can leave a binary with no signal whatsoever -- not failed, +not skipped, simply absent -- so a fully green run can hide it. That is how +Unit_HRR_Regions_MemcpyFirstTouchMaterialization stayed invisible through a +106/106 gfx950 run: HRR_HAVE_HSA was never defined anywhere in the tree, so the +case compiled out and no report it could have appeared in ever existed. + +This check asserts on the *compiled artifact* rather than on the source, which +is what makes it catch that class of problem. It fails in both directions: a +case that disappears is an error, and so is one that appears unannounced, +because an unreviewed addition is how a manifest silently stops meaning +anything. + +Case names come from `--list-tests "*" --reporter xml`. The "*" is required -- +without it Catch2 omits hidden `[.]` cases, which is most of the integration +suite. The XML reporter is used rather than the console listing because the +console form is a formatted table, and formatting is not a contract. + +Usage: + check_expected_cases.py --binary --manifest + check_expected_cases.py --binary --manifest --update +""" + +import argparse +import subprocess +import sys +import xml.etree.ElementTree as ET + +HEADER = [ + "# Test cases expected to be present in this binary, one per line, sorted.", + "# Generated by check_expected_cases.py --update; do not hand-edit casually.", + "#", + "# A diff here is a real finding either way. Cases vanish when a guard stops", + "# being satisfied (a missing -D, a find_package that quietly failed), and", + "# that vanishing is otherwise completely silent. Regenerate only when you", + "# intended to add or remove a case, and say so in the commit message.", +] + + +def listed_cases(binary: str) -> list[str]: + """Every case the binary registers, hidden ones included.""" + proc = subprocess.run( + [binary, "--list-tests", "*", "--reporter", "xml"], + capture_output=True, text=True, + ) + if proc.returncode != 0: + sys.exit(f"error: {binary} --list-tests failed ({proc.returncode})\n" + f"{proc.stderr.strip()}") + try: + root = ET.fromstring(proc.stdout) + except ET.ParseError as exc: + sys.exit(f"error: could not parse --list-tests XML from {binary}: {exc}") + names = [e.text or "" for e in root.iter("Name")] + if not names: + sys.exit(f"error: {binary} listed no test cases at all -- that is itself " + f"the failure this check exists to catch") + return sorted(names) + + +def read_manifest(path: str) -> list[str]: + try: + with open(path, encoding="utf-8") as fh: + return sorted(ln.strip() for ln in fh + if ln.strip() and not ln.startswith("#")) + except FileNotFoundError: + sys.exit(f"error: manifest {path} not found; create it with --update") + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--binary", required=True, help="Catch2 test executable") + ap.add_argument("--manifest", required=True, help="expected-cases file") + ap.add_argument("--update", action="store_true", + help="rewrite the manifest from the binary instead of checking") + args = ap.parse_args(argv) + + actual = listed_cases(args.binary) + + if args.update: + with open(args.manifest, "w", encoding="utf-8") as fh: + fh.write("\n".join(HEADER) + "\n") + fh.write("\n".join(actual) + "\n") + print(f"wrote {len(actual)} case names to {args.manifest}") + return 0 + + expected = read_manifest(args.manifest) + missing = [n for n in expected if n not in set(actual)] + unexpected = [n for n in actual if n not in set(expected)] + + if not missing and not unexpected: + print(f"OK: {len(actual)} test cases present, matching {args.manifest}") + return 0 + + print(f"FAIL: {args.binary} does not match {args.manifest}", file=sys.stderr) + print(f" expected {len(expected)} cases, found {len(actual)}", file=sys.stderr) + for n in missing: + print(f" MISSING (compiled out or deleted): {n}", file=sys.stderr) + for n in unexpected: + print(f" UNEXPECTED (added without updating manifest): {n}", file=sys.stderr) + if missing: + print("\nA MISSING case is usually a guard that stopped being satisfied -- " + "check that every -D its #if needs is still passed by CMake.", + file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/hrr/tests/expected_cases_integration.txt b/projects/hrr/tests/expected_cases_integration.txt new file mode 100644 index 000000000000..486e2c5cc275 --- /dev/null +++ b/projects/hrr/tests/expected_cases_integration.txt @@ -0,0 +1,149 @@ +# Test cases expected to be present in this binary, one per line, sorted. +# Generated by check_expected_cases.py --update; do not hand-edit casually. +# +# A diff here is a real finding either way. Cases vanish when a guard stops +# being satisfied (a missing -D, a find_package that quietly failed), and +# that vanishing is otherwise completely silent. Regenerate only when you +# intended to add or remove a case, and say so in the commit message. +Unit_HRR_AllApisRoundtrip +Unit_HRR_AllApis_Direct +Unit_HRR_ApiMatrix_Breadth2_Direct +Unit_HRR_ApiMatrix_Breadth_Direct +Unit_HRR_ApiMatrix_Deprioritised_Direct +Unit_HRR_ApiMatrix_GraphNodes_Direct +Unit_HRR_ApiMatrix_InfoBreakdownParse +Unit_HRR_ApiMatrix_LegacyCtx_Direct +Unit_HRR_ApiMatrix_LegacyLaunch_Direct +Unit_HRR_ApiMatrix_LegacySurface_Direct +Unit_HRR_ApiMatrix_ManifestWellFormed +Unit_HRR_ApiMatrix_MemPoolShare_Direct +Unit_HRR_ApiMatrix_MultiGpu_Direct +Unit_HRR_ApiMatrix_PayloadLoss_Direct +Unit_HRR_ApiMatrix_ProcAddress_Direct +Unit_HRR_ApiMatrix_ReplayClassMarkers +Unit_HRR_ApiMatrix_Reset_Direct +Unit_HRR_ApiMatrix_ShareableHandle_Direct +Unit_HRR_ApiMatrix_SilentFailure_Direct +Unit_HRR_ApiMatrix_SptCoopLaunch_Direct +Unit_HRR_ApiMatrix_SptKernelLaunch_Direct +Unit_HRR_ApiMatrix_T0_Roundtrip +Unit_HRR_ApiMatrix_T1_Roundtrip +Unit_HRR_ApiMatrix_T2_Roundtrip +Unit_HRR_ApiMatrix_T3_Roundtrip +Unit_HRR_ApiMatrix_T4_Roundtrip +Unit_HRR_ApiMatrix_T5_Roundtrip +Unit_HRR_ApiMatrix_UC1Dense_Direct +Unit_HRR_CaptureReplayRoundtrip +Unit_HRR_ChevronLaunchRoundtrip +Unit_HRR_ChevronLaunch_Direct +Unit_HRR_ConfigureCallRoundtrip +Unit_HRR_ConfigureCall_Direct +Unit_HRR_ContextRoundtrip +Unit_HRR_Context_Direct +Unit_HRR_DeviceExtraRoundtrip +Unit_HRR_DeviceExtra_Direct +Unit_HRR_DeviceInfoRoundtrip +Unit_HRR_DeviceInfo_Direct +Unit_HRR_DeviceMemPoolRoundtrip +Unit_HRR_DeviceMemPool_Direct +Unit_HRR_DivergenceAbortRoundtrip +Unit_HRR_DrvMemcpy2DUnalignedRoundtrip +Unit_HRR_DrvMemcpy2DUnaligned_Direct +Unit_HRR_DrvMemcpy3DRoundtrip +Unit_HRR_DrvMemcpy3D_Direct +Unit_HRR_DrvMemcpyRoundtrip +Unit_HRR_DrvMemcpy_Direct +Unit_HRR_EmbeddedPtrRoundtrip +Unit_HRR_EmbeddedPtrStruct_Direct +Unit_HRR_ExtMallocRoundtrip +Unit_HRR_ExtMalloc_Direct +Unit_HRR_ExtModuleLaunchKernelRoundtrip +Unit_HRR_ExtModuleLaunchKernel_Direct +Unit_HRR_FirstMallocRoundtrip +Unit_HRR_FirstMalloc_Direct +Unit_HRR_GpuWorkload_Direct +Unit_HRR_GraphExplicitRoundtrip +Unit_HRR_GraphExplicit_Direct +Unit_HRR_GraphLaunchSptRoundtrip +Unit_HRR_GraphLaunchSpt_Direct +Unit_HRR_GraphRoundtrip +Unit_HRR_GraphWorkload_Direct +Unit_HRR_HostAliasesRoundtrip +Unit_HRR_HostAliases_Direct +Unit_HRR_HostFreeRoundtrip +Unit_HRR_HostFree_Direct +Unit_HRR_HostMemRoundtrip +Unit_HRR_HostMemWorkload_Direct +Unit_HRR_HostRegLaunchRoundtrip +Unit_HRR_HostRegLaunch_Direct +Unit_HRR_KernelArgs_Direct +Unit_HRR_KernelArgs_SentinelKeptLostPointerNulled +Unit_HRR_LoggingRoundtrip +Unit_HRR_Logging_Direct +Unit_HRR_MemPoolExtendedRoundtrip +Unit_HRR_MemPoolExtended_Direct +Unit_HRR_Memcpy3DSptRoundtrip +Unit_HRR_Memcpy3DSpt_Direct +Unit_HRR_MemcpyExtraRoundtrip +Unit_HRR_MemcpyExtra_Direct +Unit_HRR_MemcpyPeerRoundtrip +Unit_HRR_MemcpyPeer_Direct +Unit_HRR_MemcpySptRoundtrip +Unit_HRR_MemcpySpt_Direct +Unit_HRR_Memset3DSptRoundtrip +Unit_HRR_Memset3DSpt_Direct +Unit_HRR_MemsetD2DPitchAllocRoundtrip +Unit_HRR_MemsetD2DPitchAlloc_Direct +Unit_HRR_MemsetD2DRoundtrip +Unit_HRR_MemsetD2D_Direct +Unit_HRR_MemsetExtraRoundtrip +Unit_HRR_MemsetExtra_Direct +Unit_HRR_MemsetSptRoundtrip +Unit_HRR_MemsetSpt_Direct +Unit_HRR_MemsetVariantsRoundtrip +Unit_HRR_MemsetVariants_Direct +Unit_HRR_MetadataManifest +Unit_HRR_MiscAPIsRoundtrip +Unit_HRR_MiscAPIs_Direct +Unit_HRR_ModuleAPIRoundtrip +Unit_HRR_ModuleAPI_Direct +Unit_HRR_ModuleExtraRoundtrip +Unit_HRR_ModuleExtra_Direct +Unit_HRR_MultiThreadRoundtrip +Unit_HRR_NullOptionalPtrRoundtrip +Unit_HRR_NullOptionalPtr_Direct +Unit_HRR_OccupancyRoundtrip +Unit_HRR_Occupancy_Direct +Unit_HRR_Regions_Direct +Unit_HRR_Regions_EmbeddedPointerMaterialization +Unit_HRR_Regions_MemcpyDirect +Unit_HRR_Regions_MemcpyFirstTouchMaterialization +Unit_HRR_Regions_Roundtrip +Unit_HRR_Regions_SegmentMaterialization +Unit_HRR_Regions_StreamFraming +Unit_HRR_ReplaceKernelBadSpec +Unit_HRR_ReplaceKernelMissingCO +Unit_HRR_ReplaceKernelRoundtrip +Unit_HRR_StreamAdvanced2Roundtrip +Unit_HRR_StreamAdvanced2_Direct +Unit_HRR_StreamAdvancedRoundtrip +Unit_HRR_StreamAdvanced_Direct +Unit_HRR_StreamCaptureBeginSptRoundtrip +Unit_HRR_StreamCaptureBeginSpt_Direct +Unit_HRR_StreamCaptureQuerySptRoundtrip +Unit_HRR_StreamCaptureQuerySpt_Direct +Unit_HRR_StreamQuerySptRoundtrip +Unit_HRR_StreamQuerySpt_Direct +Unit_HRR_StreamWaitEventSptRoundtrip +Unit_HRR_StreamWaitEventSpt_Direct +Unit_HRR_StreamWriteValueRoundtrip +Unit_HRR_StreamWriteValue_Direct +Unit_HRR_StressApisRoundtrip +Unit_HRR_StressApis_Direct +Unit_HRR_TextureRoundtrip +Unit_HRR_Texture_Direct +Unit_HRR_UncapturedHostWrite_Direct +Unit_HRR_VMMRoundtrip +Unit_HRR_VMM_Direct +Unit_HRR_ZeroInitRead_Direct +Unit_HRR_ZeroInitRoundtrip diff --git a/projects/hrr/tests/expected_cases_unit.txt b/projects/hrr/tests/expected_cases_unit.txt new file mode 100644 index 000000000000..e5cde064ddd1 --- /dev/null +++ b/projects/hrr/tests/expected_cases_unit.txt @@ -0,0 +1,30 @@ +# Test cases expected to be present in this binary, one per line, sorted. +# Generated by check_expected_cases.py --update; do not hand-edit casually. +# +# A diff here is a real finding either way. Cases vanish when a guard stops +# being satisfied (a missing -D, a find_package that quietly failed), and +# that vanishing is otherwise completely silent. Regenerate only when you +# intended to add or remove a case, and say so in the commit message. +Unit_HRR_AllocKind_FreeDispatch +Unit_HRR_Format_BadMagic +Unit_HRR_Format_EventTypeName +Unit_HRR_Format_HashHex +Unit_HRR_Format_Roundtrip +Unit_HRR_Format_SmallPayloadForType +Unit_HRR_Format_TruncatedEvent +Unit_HRR_Format_UnknownEventType +Unit_HRR_Playback_ReplayedH2DDrainGraphGuard +Unit_HRR_Playback_VersionOption +Unit_HRR_Playback_ZeroInitDrainGuard +Unit_HRR_Recovery_CompleteArchive +Unit_HRR_Recovery_ImplausiblePayloadLength +Unit_HRR_Recovery_MultiProcessRootInfo +Unit_HRR_Recovery_NoTrailer +Unit_HRR_Recovery_RepairWritesProcessManifest +Unit_HRR_Recovery_TornHeader +Unit_HRR_Recovery_TornPayload +Unit_HRR_TranslatePtr_ExactMatch +Unit_HRR_TranslatePtr_OutOfRange_ReturnsNull +Unit_HRR_TranslatePtr_SubAlloc_ReturnsOffset +Unit_HRR_TranslatePtr_TightestEnclosing +Unit_HRR_TranslatePtr_Zero_ReturnsNull diff --git a/projects/hrr/tests/integration/CMakeLists.txt b/projects/hrr/tests/integration/CMakeLists.txt index 82c61cb84f5f..d7f26a911d3f 100644 --- a/projects/hrr/tests/integration/CMakeLists.txt +++ b/projects/hrr/tests/integration/CMakeLists.txt @@ -102,6 +102,11 @@ target_link_libraries(hrr-integration-tests PRIVATE if(NOT WIN32) find_package(hsa-runtime64 CONFIG REQUIRED) target_link_libraries(hrr-integration-tests PRIVATE hsa-runtime64::hsa-runtime64) + # hsa-runtime64 is REQUIRED and linked just above, so the cases that call HSA + # directly can compile here. Define the macro they are guarded by next to the + # dependency so the two cannot drift apart; without it those cases vanish from + # the binary with no signal. tests/expected_cases_integration.txt is the backstop. + target_compile_definitions(hrr-integration-tests PRIVATE HRR_HAVE_HSA) endif() # Optional: hip_raw_trace, a genuine multi-threaded capture workload used only by diff --git a/projects/hrr/tests/run_catch2.py b/projects/hrr/tests/run_catch2.py new file mode 100644 index 000000000000..31b2387ef727 --- /dev/null +++ b/projects/hrr/tests/run_catch2.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Run Catch2 quietly while preserving console diagnostics and JUnit results.""" + +from __future__ import annotations + +import argparse +import os +import signal +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--xml", required=True, type=Path) + parser.add_argument( + "--timeout", + type=float, + help="seconds before the suite and its workloads are killed", + ) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + if args.command[:1] == ["--"]: + args.command = args.command[1:] + if not args.command: + parser.error("a Catch2 command is required after --") + return args + + +def test_result(case: ET.Element) -> str: + if case.find("failure") is not None or case.find("error") is not None: + return "FAIL" + if case.find("skipped") is not None: + return "SKIP" + return "PASS" + + +def group_cases(root: ET.Element) -> dict[str, list[ET.Element]]: + """Group records by top-level case, in order of first appearance. + + Catch2 writes one record per SECTION named "Case/Section", and an assertion + that fails inside a section is attached to that record only. A case whose + assertions all live in sections has no record under its own name at all. + Counting only slash-free names would therefore lose both. + """ + groups: dict[str, list[ET.Element]] = {} + for case in root.iterfind(".//testcase"): + name = case.attrib.get("name") + if name: + groups.setdefault(name.split("/", 1)[0], []).append(case) + return groups + + +def group_result(records: list[ET.Element]) -> str: + """A failing section fails its case; a skip beats a pass.""" + results = {test_result(record) for record in records} + for result in ("FAIL", "SKIP"): + if result in results: + return result + return "PASS" + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + # Durations name every case as it finishes, so the transcript of a hung + # suite ends just before the case that hung. + command = args.command + [ + "--durations", + "yes", + "--reporter", + "console", + "--reporter", + f"junit::out={args.xml}", + ] + # A session of its own, so a timeout also kills the workloads the suite + # spawned: they hold the pipe open and would block the read forever. + with subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) as process: + try: + stdout, _ = process.communicate(timeout=args.timeout) + except subprocess.TimeoutExpired: + # SIGTERM first: Catch2's fatal-signal handler names the running + # case and flushes the console reporter, which SIGKILL would lose. + stdout = "" + for sig, grace in ((signal.SIGTERM, 10), (signal.SIGKILL, None)): + try: + os.killpg(process.pid, sig) + except ProcessLookupError: + pass + try: + stdout, _ = process.communicate(timeout=grace) + break + except subprocess.TimeoutExpired: + continue + print(f"Catch2 timed out after {args.timeout:g} s; it hung after:") + print(stdout, end="") + return 1 + completed = subprocess.CompletedProcess(command, process.returncode, stdout) + + try: + groups = group_cases(ET.parse(args.xml).getroot()) + except (ET.ParseError, OSError) as error: + print(f"Could not read JUnit results: {error}", file=sys.stderr) + print(completed.stdout, end="") + return completed.returncode or 1 + + counts = {"PASS": 0, "FAIL": 0, "SKIP": 0} + for name, records in groups.items(): + result = group_result(records) + counts[result] += 1 + print(f"{result}: {name}") + if result == "FAIL": + for record in records: + for failure in record.findall("failure") + record.findall("error"): + if failure.text: + print(failure.text.strip()) + print( + f"{counts['PASS']} passed, {counts['FAIL']} failed, " + f"{counts['SKIP']} skipped" + ) + + # A crash can leave syntactically valid, but incomplete, JUnit without a + # failed case. Preserve the raw transcript only for that exceptional path. + # Catch2 returns 4 when the selected cases were all skipped; that is not a + # crash and must not dump the banner. Any other non-zero status with no + # failed case is one, skipped cases or not (a crash at exit after some + # cases skipped, for instance, must still leave a log). + if completed.returncode not in (0, 4) and counts["FAIL"] == 0: + print("\nCatch2 terminated without a JUnit failure:") + print(completed.stdout, end="") + # Catch2 uses 4 for "tests were skipped / none ran". That is a skip, not a + # failed suite: returning it would paint gfx90a red after every GPU case + # correctly skipped for hipErrorNoDevice. + if completed.returncode == 4 and counts["FAIL"] == 0: + return 0 + return completed.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/projects/hrr/tests/summarize_junit.py b/projects/hrr/tests/summarize_junit.py new file mode 100644 index 000000000000..06b10679d44b --- /dev/null +++ b/projects/hrr/tests/summarize_junit.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Build a platform/family table from Catch2 JUnit files. + +CI names the files unit-.xml and integration-.xml. Catch2 also +emits one per SECTION, named "Case/Section", and a failure inside a +section is attached to that record only. Counts here are per top-level case +(the part before the first '/'), a case failing if any of its records fails and +being skipped if any is skipped, matching run_catch2.py. The two scripts are +checked out separately by the workflow, so the grouping is repeated rather than +imported; test_summarize_junit.py asserts they agree. +""" + +from __future__ import annotations + +import argparse +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +# Display order matches .github/workflows/hrr-ci.yml. +COLUMNS = ( + ("unit-ubuntu-24.04.xml", "Unit", "Linux"), + ("unit-windows-2022.xml", "Unit", "Windows"), + ("integration-gfx90a.xml", "Integration", "gfx90a"), + ("integration-gfx94x.xml", "Integration", "gfx94x"), + ("integration-gfx950.xml", "Integration", "gfx950"), + ("integration-gfx103x.xml", "Integration", "gfx103x"), + ("integration-gfx110x.xml", "Integration", "gfx110x"), + ("integration-gfx120x.xml", "Integration", "gfx120x"), + ("integration-gfx1151.xml", "Integration", "gfx1151"), + ("integration-gfx1150.xml", "Integration", "gfx1150"), + ("integration-gfx1153.xml", "Integration", "gfx1153"), +) + +MAX_FAILED_NAMES = 8 + + +def case_result(case: ET.Element) -> str: + if case.find("failure") is not None or case.find("error") is not None: + return "FAIL" + if case.find("skipped") is not None: + return "SKIP" + return "PASS" + + +def group_cases(root: ET.Element) -> dict[str, list[ET.Element]]: + groups: dict[str, list[ET.Element]] = {} + for case in root.iterfind(".//testcase"): + name = case.attrib.get("name") + if name: + groups.setdefault(name.split("/", 1)[0], []).append(case) + return groups + + +def group_result(records: list[ET.Element]) -> str: + results = {case_result(record) for record in records} + for result in ("FAIL", "SKIP"): + if result in results: + return result + return "PASS" + + +def summarize_xml(path: Path) -> dict[str, object]: + groups = group_cases(ET.parse(path).getroot()) + counts = {"PASS": 0, "FAIL": 0, "SKIP": 0} + failed: list[str] = [] + for name, records in groups.items(): + result = group_result(records) + counts[result] += 1 + if result == "FAIL": + failed.append(name) + return {"counts": counts, "failed": failed, "total": len(groups)} + + +def try_summarize(path: Path, errors: list[str]) -> dict[str, object]: + # A job killed mid-run (timeout, lost runner) can upload a truncated or + # empty file. Report it as unreadable rather than abort the whole table. + try: + return summarize_xml(path) + except (ET.ParseError, OSError) as error: + errors.append(f"{path.name}: {error}") + return {"error": str(error)} + + +def cell(value: object) -> str: + return "—" if value is None else str(value) + + +def failed_cell(names: list[str]) -> str: + if not names: + return "" + shown = names[:MAX_FAILED_NAMES] + text = ", ".join(f"`{name}`" for name in shown) + extra = len(names) - len(shown) + if extra: + text += f", … +{extra}" + return text + + +def render(directory: Path, errors: list[str]) -> str: + rows: list[tuple[str, str, dict[str, object] | None]] = [] + for filename, suite, platform in COLUMNS: + path = directory / filename + if path.is_file(): + rows.append((suite, platform, try_summarize(path, errors))) + else: + rows.append((suite, platform, None)) + + extra = sorted( + path.name + for path in directory.glob("*.xml") + if path.name not in {filename for filename, _, _ in COLUMNS} + ) + for filename in extra: + rows.append(("Other", filename, try_summarize(directory / filename, errors))) + + lines = [ + "## HRR results by platform", + "", + "| Suite | Platform / family | Pass | Fail | Skip | Failed tests |", + "| --- | --- | ---: | ---: | ---: | --- |", + ] + for suite, platform, data in rows: + if data is None: + lines.append(f"| {suite} | {platform} | — | — | — | _no results_ |") + continue + if "error" in data: + lines.append( + f"| {suite} | {platform} | — | — | — | _unreadable XML: {data['error']}_ |" + ) + continue + counts = data["counts"] + lines.append( + "| {suite} | {platform} | {passed} | {failed} | {skipped} | {names} |".format( + suite=suite, + platform=platform, + passed=counts["PASS"], + failed=counts["FAIL"], + skipped=counts["SKIP"], + names=failed_cell(data["failed"]), + ) + ) + + lines.extend( + [ + "", + "Counts are top-level Catch2 cases. Missing XML means that job did not upload results (cancelled or not started). Unreadable XML is usually a job killed mid-run; its results are incomplete.", + "", + "### Counts grid", + "", + ] + ) + + headers = ["", *(platform for _, platform, _ in rows)] + lines.append("| " + " | ".join(headers) + " |") + lines.append("| --- |" + " ---: |" * (len(headers) - 1)) + for label, key in (("Pass", "PASS"), ("Fail", "FAIL"), ("Skip", "SKIP")): + values = [] + for _, _, data in rows: + values.append( + cell(None if data is None or "error" in data else data["counts"][key]) + ) + lines.append("| " + " | ".join([label, *values]) + " |") + + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--dir", type=Path, default=Path(".")) + parser.add_argument( + "--summary", + type=Path, + help="Append markdown to this file (GITHUB_STEP_SUMMARY).", + ) + args = parser.parse_args(argv) + if not args.dir.is_dir(): + print(f"not a directory: {args.dir}", file=sys.stderr) + return 1 + errors: list[str] = [] + markdown = render(args.dir, errors) + sys.stdout.write(markdown) + if args.summary is not None: + with args.summary.open("a", encoding="utf-8") as out: + out.write(markdown) + for error in errors: + print(f"::error::unreadable JUnit XML {error}") + return 1 if errors else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/projects/hrr/tests/test_check_expected_cases.py b/projects/hrr/tests/test_check_expected_cases.py new file mode 100644 index 000000000000..58dc40c105ef --- /dev/null +++ b/projects/hrr/tests/test_check_expected_cases.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Tests for check_expected_cases.py. + +subprocess.run is replaced so no Catch2 binary is needed. +""" + +import contextlib +import io +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import check_expected_cases as cec # noqa: E402 + + +def listing(*names: str) -> str: + body = "".join(f"{n}" for n in names) + return f"{body}" + + +def completed(stdout: str = "", returncode: int = 0, stderr: str = ""): + return subprocess.CompletedProcess([], returncode, stdout, stderr) + + +class ListedCasesTest(unittest.TestCase): + def test_sorted_names_and_the_star_selector(self): + with mock.patch.object(cec.subprocess, "run", return_value=completed(listing("b", "a"))) as run: + self.assertEqual(cec.listed_cases("./bin"), ["a", "b"]) + # Without "*" Catch2 omits hidden cases, which is most of the suite. + self.assertEqual(run.call_args.args[0], ["./bin", "--list-tests", "*", "--reporter", "xml"]) + + def test_failing_listing_exits(self): + with mock.patch.object(cec.subprocess, "run", return_value=completed("", 2, "bad flag")): + with self.assertRaises(SystemExit) as caught: + cec.listed_cases("./bin") + self.assertIn("failed (2)", str(caught.exception)) + self.assertIn("bad flag", str(caught.exception)) + + def test_unparseable_listing_exits(self): + with mock.patch.object(cec.subprocess, "run", return_value=completed(" str: + return "" + "".join(records) + "" + + +def record(name: str, kind: str | None = None, text: str = "") -> str: + if kind is None: + return f'' + return f'<{kind}>{text}' + + +class ParseArgsTest(unittest.TestCase): + def test_strips_separator_and_reads_options(self): + args = run_catch2.parse_args( + ["--xml", "r.xml", "--timeout", "2.5", "--", "./bin", "[cpu]"] + ) + self.assertEqual(args.xml, Path("r.xml")) + self.assertEqual(args.timeout, 2.5) + self.assertEqual(args.command, ["./bin", "[cpu]"]) + + def test_timeout_is_optional(self): + args = run_catch2.parse_args(["--xml", "r.xml", "--", "./bin"]) + self.assertIsNone(args.timeout) + + def test_requires_a_command(self): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + run_catch2.parse_args(["--xml", "r.xml", "--"]) + + def test_requires_xml(self): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + run_catch2.parse_args(["--", "./bin"]) + + +class ResultTest(unittest.TestCase): + def parse(self, text: str) -> ET.Element: + return ET.fromstring(text) + + def test_test_result(self): + self.assertEqual(run_catch2.test_result(self.parse(record("a"))), "PASS") + self.assertEqual(run_catch2.test_result(self.parse(record("a", "skipped"))), "SKIP") + self.assertEqual(run_catch2.test_result(self.parse(record("a", "failure"))), "FAIL") + self.assertEqual(run_catch2.test_result(self.parse(record("a", "error"))), "FAIL") + + def test_group_cases_folds_sections_into_their_case(self): + root = self.parse( + junit(record("A"), record("B/x"), record("B/y"), record("C/s/t"), '') + ) + groups = run_catch2.group_cases(root) + self.assertEqual(list(groups), ["A", "B", "C"]) + self.assertEqual(len(groups["B"]), 2) + + def test_group_result_precedence(self): + def result(*kinds): + root = self.parse(junit(*(record("A/" + str(i), k) for i, k in enumerate(kinds)))) + return run_catch2.group_result(run_catch2.group_cases(root)["A"]) + + self.assertEqual(result(None, None), "PASS") + self.assertEqual(result(None, "skipped"), "SKIP") + self.assertEqual(result("skipped", "failure", None), "FAIL") + self.assertEqual(result("error"), "FAIL") + + +class MainTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.dir = Path(self.tmp.name) + self.fake = self.dir / "fake_catch2.py" + self.fake.write_text(FAKE) + self.xml = self.dir / "out.xml" + + def run_main(self, spec: dict, *extra: str): + spec_path = self.dir / "spec.json" + spec_path.write_text(json.dumps(spec)) + argv = ["--xml", str(self.xml), *extra, "--", sys.executable, str(self.fake), str(spec_path)] + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + rc = run_catch2.main(argv) + return rc, out.getvalue(), err.getvalue() + + def test_passing_run(self): + rc, out, _ = self.run_main({"xml": junit(record("A"), record("B"))}) + self.assertEqual(rc, 0) + self.assertIn("PASS: A", out) + self.assertIn("2 passed, 0 failed, 0 skipped", out) + self.assertNotIn("BANNER", out) + + def test_reporters_are_appended(self): + rc, out, _ = self.run_main({"xml": junit(record("A")), "echo_args": True}) + self.assertEqual(rc, 0) + self.assertNotIn("ARGS=", out) # the transcript is only shown on a crash + spec = {"xml": junit(record("A")), "echo_args": True, "rc": 139} + _, out, _ = self.run_main(spec) + args = json.loads(out.split("ARGS=")[1].splitlines()[0]) + self.assertIn("--durations", args) + self.assertIn(f"junit::out={self.xml}", args) + self.assertIn("console", args) + + def test_failing_section_fails_its_passing_parent(self): + xml = junit( + record("Case"), + record("Case/ok"), + record("Case/bad", "failure", "assertion went wrong"), + ) + rc, out, _ = self.run_main({"xml": xml, "rc": 1}) + self.assertEqual(rc, 1) + self.assertIn("FAIL: Case", out) + self.assertIn("assertion went wrong", out) + self.assertIn("0 passed, 1 failed, 0 skipped", out) + self.assertNotIn("BANNER", out) + + def test_case_with_only_sections_is_counted(self): + xml = junit(record("OnlySections/a", "failure", "boom"), record("Fine")) + rc, out, _ = self.run_main({"xml": xml, "rc": 1}) + self.assertEqual(rc, 1) + self.assertIn("FAIL: OnlySections", out) + self.assertIn("1 passed, 1 failed, 0 skipped", out) + + def test_all_skipped_exit_4_is_success_without_transcript(self): + rc, out, _ = self.run_main({"xml": junit(record("A", "skipped")), "rc": 4}) + self.assertEqual(rc, 0) + self.assertIn("SKIP: A", out) + self.assertNotIn("BANNER", out) + + def test_exit_4_with_a_failure_stays_failed(self): + xml = junit(record("A", "skipped"), record("B", "failure", "x")) + rc, out, _ = self.run_main({"xml": xml, "rc": 4}) + self.assertEqual(rc, 4) + self.assertIn("1 failed", out) + + def test_crash_after_skips_keeps_the_transcript(self): + rc, out, _ = self.run_main({"xml": junit(record("A", "skipped"), record("B")), "rc": 139}) + self.assertEqual(rc, 139) + self.assertIn("Catch2 terminated without a JUnit failure", out) + self.assertIn("BANNER", out) + + def test_crash_without_skips_keeps_the_transcript(self): + rc, out, _ = self.run_main({"xml": junit(record("A")), "rc": 134}) + self.assertEqual(rc, 134) + self.assertIn("BANNER", out) + + def test_unreadable_junit_prints_transcript(self): + rc, out, err = self.run_main({"xml": " str: + return "" + "".join(records) + "" + + +def record(name: str, kind: str | None = None) -> str: + if kind is None: + return f'' + return f'<{kind}>text' + + +class ScratchDir(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.dir = Path(self.tmp.name) + + def write(self, name: str, text: str) -> Path: + path = self.dir / name + path.write_text(text) + return path + + +class CaseTest(unittest.TestCase): + def test_case_result(self): + parse = ET.fromstring + self.assertEqual(summarize_junit.case_result(parse(record("a"))), "PASS") + self.assertEqual(summarize_junit.case_result(parse(record("a", "skipped"))), "SKIP") + self.assertEqual(summarize_junit.case_result(parse(record("a", "failure"))), "FAIL") + self.assertEqual(summarize_junit.case_result(parse(record("a", "error"))), "FAIL") + + def test_group_cases_and_result(self): + root = ET.fromstring(junit(record("A"), record("B/x"), record("B/y", "failure"), '')) + groups = summarize_junit.group_cases(root) + self.assertEqual(list(groups), ["A", "B"]) + self.assertEqual(summarize_junit.group_result(groups["A"]), "PASS") + self.assertEqual(summarize_junit.group_result(groups["B"]), "FAIL") + + def test_grouping_agrees_with_run_catch2(self): + # The workflow checks the two scripts out separately, so they cannot + # share code; they must not drift apart. + samples = [ + junit(record("A"), record("A/s", "failure")), + junit(record("A/s", "skipped"), record("A/t")), + junit(record("A", "skipped"), record("B"), record("C/x", "error")), + ] + for sample in samples: + root = ET.fromstring(sample) + mine = {k: summarize_junit.group_result(v) for k, v in summarize_junit.group_cases(root).items()} + theirs = {k: run_catch2.group_result(v) for k, v in run_catch2.group_cases(root).items()} + self.assertEqual(mine, theirs) + + +class FormatTest(unittest.TestCase): + def test_cell(self): + self.assertEqual(summarize_junit.cell(None), "—") + self.assertEqual(summarize_junit.cell(0), "0") + + def test_failed_cell(self): + self.assertEqual(summarize_junit.failed_cell([]), "") + self.assertEqual(summarize_junit.failed_cell(["a", "b"]), "`a`, `b`") + names = [f"n{i}" for i in range(summarize_junit.MAX_FAILED_NAMES + 3)] + text = summarize_junit.failed_cell(names) + self.assertIn("`n7`", text) + self.assertNotIn("`n8`", text) + self.assertTrue(text.endswith("… +3")) + + +class SummarizeTest(ScratchDir): + def test_summarize_xml_counts_cases_not_sections(self): + path = self.write( + "x.xml", + junit( + record("Pass"), + record("Skip", "skipped"), + record("PassWithBadSection"), + record("PassWithBadSection/s", "failure"), + record("OnlySection/s", "failure"), + ), + ) + data = summarize_junit.summarize_xml(path) + self.assertEqual(data["counts"], {"PASS": 1, "FAIL": 2, "SKIP": 1}) + self.assertEqual(data["failed"], ["PassWithBadSection", "OnlySection"]) + self.assertEqual(data["total"], 4) + + def test_try_summarize_reports_unreadable_files(self): + path = self.write("bad.xml", "