diff --git a/.github/workflows/rocjitsu-corpus-tests.yml b/.github/workflows/rocjitsu-corpus-tests.yml index 224a6b081cda..4c0cfe74b53d 100644 --- a/.github/workflows/rocjitsu-corpus-tests.yml +++ b/.github/workflows/rocjitsu-corpus-tests.yml @@ -102,6 +102,7 @@ jobs: # Each simulator process can use many host threads. Bound Release # fanout so long FPSAN cases do not starve under runner load. corpus_worker_limit: 4 + rocjitsu_thread_budget: 8 corpus_soft_timeout: 360 corpus_hard_timeout: 600 corpus_rerun_timeout: 600 @@ -123,7 +124,8 @@ jobs: sanitizer_runtime: SHARED run_simulator_corpus: true corpus_worker_limit: 4 - corpus_soft_timeout: 240 + rocjitsu_thread_budget: 4 + corpus_soft_timeout: 300 corpus_hard_timeout: 480 corpus_rerun_timeout: 1200 corpus_sanitizer: clang-asan @@ -143,6 +145,7 @@ jobs: sanitizer_runtime: SHARED run_simulator_corpus: false corpus_worker_limit: 16 + rocjitsu_thread_budget: 4 corpus_soft_timeout: 0 corpus_hard_timeout: 0 corpus_rerun_timeout: 0 @@ -165,6 +168,7 @@ jobs: cxx_compiler: g++ run_simulator_corpus: true corpus_worker_limit: 4 + rocjitsu_thread_budget: 8 corpus_soft_timeout: 240 corpus_hard_timeout: 480 corpus_rerun_timeout: 1200 @@ -468,6 +472,9 @@ jobs: uv pip install --system -e lib/python python -m pytest lib/python/amdisa/tests + - name: Test corpus harness options + run: python3 "${ROCJITSU_SOURCE_DIR}/tests/corpus/test-run-corpus-tests.py" + - name: Run corpus tests # Keep the HIP corpus signal even if an independent test step fails. if: ${{ !cancelled() && matrix.run_simulator_corpus && steps.build_rocjitsu.outcome == 'success' && steps.install_corpus_dependencies.outcome == 'success' }} @@ -492,8 +499,15 @@ jobs: if (( WORKERS > WORKER_LIMIT )); then WORKERS="${WORKER_LIMIT}" fi + ROCJITSU_THREAD_BUDGET_LIMIT=${{ matrix.rocjitsu_thread_budget }} + ROCJITSU_THREAD_BUDGET="$(nproc)" + if (( ROCJITSU_THREAD_BUDGET > ROCJITSU_THREAD_BUDGET_LIMIT )); then + ROCJITSU_THREAD_BUDGET="${ROCJITSU_THREAD_BUDGET_LIMIT}" + fi + echo "Corpus CPU plan: pytest-workers: ${WORKERS}, rocjitsu-thread-budget: ${ROCJITSU_THREAD_BUDGET}" corpus_test_flags=( --workers "${WORKERS}" + --rocjitsu-thread-budget "${ROCJITSU_THREAD_BUDGET}" --soft-timeout ${{ matrix.corpus_soft_timeout }} --hard-timeout ${{ matrix.corpus_hard_timeout }} --rerun-timeout ${{ matrix.corpus_rerun_timeout }} diff --git a/emulation/rocjitsu/tests/corpus/README.md b/emulation/rocjitsu/tests/corpus/README.md index 2c981bf4ee4a..0f8cd45df94f 100644 --- a/emulation/rocjitsu/tests/corpus/README.md +++ b/emulation/rocjitsu/tests/corpus/README.md @@ -56,6 +56,23 @@ retains only its size and SHA-256. Its `finalize` command requires every pinned corpus input to have either a successful pair or a matching declared exclusion, which prevents partial runs from becoming a develop baseline. +## CPU allocation + +Every corpus case is its own ROCjitsu process, and each of those sizes its +thread allocation from the CPUs it can see rather than from the share of the +runner it is entitled to — so several workers on one runner each plan as if +they had the whole machine. The workflow matrix therefore names both numbers +per lane: `corpus_worker_limit` for pytest-xdist fanout and +`rocjitsu_thread_budget` for one ROCjitsu process, capped at `nproc`. + +The optional `--rocjitsu-thread-budget N` is forwarded as +`rocjitsu --cpu-thread-budget`, which replaces the JSON `cpu_thread_budget` +for that launch. Omitting it preserves the configured budget and automatic CPU +detection; an explicit `0` selects the launcher's automatic sizing. CI supplies +an explicit per-process budget for each lane. The option does not restrict process +affinity: ROCjitsu picks the largest configured thread allocation that fits, +and Linux still schedules those threads on any CPU available to the CI VM. + ## Near-timeout reporting With `--warn-perf`, `run-corpus-tests.sh` warns about passing tests whose @@ -143,8 +160,8 @@ The Clang and GCC ASan+UBSan lanes run the same target-qualified corpus as the release lane against a sanitizer-instrumented RocJITsu build. They use longer per-test timeouts to accommodate the instrumented simulator. -Each case runs through `env` → `setpriv` → the process supervisor → `timeout` → -`rocjitsu` → the optional HIP preload helper → the corpus executable. The +Each case runs through `env` → `setpriv` → the process supervisor → `timeout` +→ `rocjitsu` → the optional HIP preload helper → the corpus executable. The run-wrapper `timeout` owns the per-test deadline and retains command output; pytest gets 15 seconds of cleanup headroom as a failsafe. Each target's failed-test rerun also has a 20-minute budget in CI, within the 60-minute diff --git a/emulation/rocjitsu/tests/corpus/run-corpus-tests.sh b/emulation/rocjitsu/tests/corpus/run-corpus-tests.sh index a37c3069c12d..f167fc441bf1 100755 --- a/emulation/rocjitsu/tests/corpus/run-corpus-tests.sh +++ b/emulation/rocjitsu/tests/corpus/run-corpus-tests.sh @@ -11,6 +11,8 @@ # # Options: # --workers N Number of pytest-xdist workers (default: 8) +# --rocjitsu-thread-budget N Override ROCjitsu's per-process thread budget (0: automatic) +# Default: preserve the configured budget and CPU detection # --soft-timeout N Per-test timeout for the first run (default: 30) # --hard-timeout N Per-test timeout for failed-test reruns (default: 60) # --rerun-timeout N Overall failed-test rerun budget (default: 1200) @@ -33,6 +35,7 @@ set -euo pipefail : "${ROCJITSU_SOURCE_DIR:?ROCJITSU_SOURCE_DIR must be set}" worker_count=8 +rocjitsu_thread_budget="" soft_timeout_seconds=30 hard_timeout_seconds=60 rerun_timeout_seconds=1200 @@ -41,7 +44,8 @@ warn_perf=false sanitizer_mode=none usage() { - echo "Usage: $0 [--workers N] [--soft-timeout N] [--hard-timeout N] [--rerun-timeout N]" \ + echo "Usage: $0 [--workers N] [--rocjitsu-thread-budget N] [--soft-timeout N]" \ + "[--hard-timeout N] [--rerun-timeout N]" \ "[--sanitizer none|clang-asan|gcc-asan] [--rerun-failed] [--warn-perf]" >&2 } @@ -64,6 +68,19 @@ while (( $# )); do worker_count="$2" shift 2 ;; + --rocjitsu-thread-budget) + if (( $# < 2 )); then + echo "--rocjitsu-thread-budget requires a value" >&2 + usage + exit 1 + fi + rocjitsu_thread_budget="$2" + if [[ ! "${rocjitsu_thread_budget}" =~ ^[0-9]+$ ]]; then + echo "--rocjitsu-thread-budget requires a non-negative integer" >&2 + exit 1 + fi + shift 2 + ;; --soft-timeout) if (( $# < 2 )); then echo "--soft-timeout requires a value" >&2 @@ -138,6 +155,11 @@ for numeric_option in "${numeric_options[@]}"; do fi done +rocjitsu_thread_budget_args=() +if [[ -n "${rocjitsu_thread_budget}" ]]; then + rocjitsu_thread_budget_args=(--cpu-thread-budget "${rocjitsu_thread_budget}") +fi + corpus_test_status=0 corpus_work_dir="$(pwd -P)" junit_dir="${corpus_work_dir}/.pytest-artifacts/junit" @@ -203,7 +225,8 @@ if [[ "${sanitizer_mode}" == clang-asan ]]; then # does not construct the expected shared-runtime/interposer preload order. preflight_config="${ROCJITSU_SOURCE_DIR}/configs/gfx942_cdna3.json" "${run_wrapper_prefix[@]}" \ - "${rocjitsu_launcher}" --config "${preflight_config}" -- \ + "${rocjitsu_launcher}" --config "${preflight_config}" \ + "${rocjitsu_thread_budget_args[@]}" -- \ "${child_command_prefix[@]}" true fi @@ -227,6 +250,7 @@ run_pytest() { timeout --foreground --signal=TERM --kill-after=5s "${timeout_seconds}s" "${rocjitsu_launcher}" --config "${config_path}" + "${rocjitsu_thread_budget_args[@]}" -- "${child_command_prefix[@]}" ) diff --git a/emulation/rocjitsu/tests/corpus/test-run-corpus-tests.py b/emulation/rocjitsu/tests/corpus/test-run-corpus-tests.py new file mode 100644 index 000000000000..8cac68015967 --- /dev/null +++ b/emulation/rocjitsu/tests/corpus/test-run-corpus-tests.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +# Test corpus budget forwarding without launching simulator workloads. +# Usage: python tests/corpus/test-run-corpus-tests.py + +import os +from pathlib import Path +import shlex +import subprocess +import tempfile +import unittest + +SOURCE = Path(__file__).resolve().parents[2] +SCRIPT = SOURCE / 'tests/corpus/run-corpus-tests.sh' + + +class ThreadBudgetTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + binary = self.root / 'bin' + binary.mkdir() + stubs = { + 'rocjitsu': '#!/bin/bash\nprintf "%s\\0" "$@" >> "$PREFLIGHT_LOG"\n', + 'pytest': ( + '#!/bin/bash\n' + 'printf "%s\\0" "$@" >> "$ARGV_LOG"\n' + 'printf "\\0" >> "$ARGV_LOG"\n' + 'if [[ "$*" == *--junitxml* && "${FORCE_RETRY:-0}" == 1 ]]; then exit 1; fi\n' + ), + } + for name, content in stubs.items(): + path = binary / name + path.write_text(content) + path.chmod(0o755) + symbolizer = self.root / 'lib/llvm/bin/llvm-symbolizer' + symbolizer.parent.mkdir(parents=True) + symbolizer.write_text('#!/bin/bash\nexit 0\n') + symbolizer.chmod(0o755) + (self.root / 'lib/libamdhip64.so').touch() + self.env = dict( + os.environ, + PATH=f'{binary}:{os.environ["PATH"]}', + ROCM_PATH=str(self.root), + ROCJITSU_SOURCE_DIR=str(SOURCE), + ARGV_LOG=str(self.root / 'argv'), + PREFLIGHT_LOG=str(self.root / 'preflight'), + ) + + def run_script(self, *args): + return subprocess.run( + ['bash', str(SCRIPT), *args], + cwd=self.root, + env=self.env, + capture_output=True, + text=True, + ) + + def assert_budget(self, argv, budget): + self.assertNotIn('--rocjitsu-thread-budget', argv) + if budget is None: + self.assertNotIn('--cpu-thread-budget', argv) + else: + self.assertEqual(argv.count('--cpu-thread-budget'), 1) + self.assertEqual(argv[argv.index('--cpu-thread-budget') + 1], budget) + + def test_budget_across_initial_runs_reruns_and_preflight(self): + for budget in (None, '0', '3'): + for sanitizer in ('none', 'clang-asan', 'gcc-asan'): + with self.subTest(budget=budget, sanitizer=sanitizer): + (self.root / 'argv').write_bytes(b'') + (self.root / 'preflight').write_bytes(b'') + self.env['FORCE_RETRY'] = '1' + args = ['--rerun-failed', '--sanitizer', sanitizer] + if budget is not None: + args += ['--rocjitsu-thread-budget', budget] + result = self.run_script(*args) + # A successful retry must not hide the original failure. + self.assertEqual(result.returncode, 1, result.stderr) + calls = (self.root / 'argv').read_bytes().split(b'\0\0') + wrappers = [] + for call in calls: + argv = [item.decode() for item in call.split(b'\0')] + if '--run-wrapper' in argv: + wrapper = shlex.split(argv[argv.index('--run-wrapper') + 1]) + self.assert_budget(wrapper, budget) + wrappers.append(wrapper) + self.assertEqual(len(wrappers), 10) + preflight = (self.root / 'preflight').read_bytes() + if sanitizer == 'clang-asan': + self.assertTrue(preflight) + self.assert_budget(preflight.decode().split('\0'), budget) + else: + self.assertFalse(preflight) + + def test_invalid_budget(self): + for value in ('', '-1', 'abc'): + with self.subTest(value=value): + result = self.run_script('--rocjitsu-thread-budget', value) + self.assertNotEqual(result.returncode, 0) + self.assertIn('requires a non-negative integer', result.stderr) + result = self.run_script('--rocjitsu-thread-budget') + self.assertNotEqual(result.returncode, 0) + self.assertIn('--rocjitsu-thread-budget requires a value', result.stderr) + + +if __name__ == '__main__': + unittest.main()