From 7e931a01851685005d978fb062adcdb3221fb1c2 Mon Sep 17 00:00:00 2001 From: Luis Machado Date: Tue, 25 Aug 2026 06:30:26 -0500 Subject: [PATCH 1/4] ci: report KFD IOCTL version in driver/GPU sanity check Extends print_driver_gpu_info.py to query the KFD IOCTL version from /dev/kfd and log it alongside the existing amd-smi and rocminfo output. The check is informational only: it warns when the version falls outside the range rocdbgapi requires (>= 1.13, < 2.0) but never fails the step. Removes the standalone print_kfd_version.py and the separate KFD version check step from test_component.yml, test_pytorch_wheels.yml, and test_multi_arch_linux_jax_wheels.yml. Also emits a run_gpu_sanity_check boolean from fetch_test_configurations.py so the sanity job in test_artifacts.yml is skipped cleanly for CPU-only families. Co-Authored-By: Claude --- .github/workflows/test_artifacts.yml | 2 + .../fetch_test_configurations.py | 3 ++ build_tools/print_driver_gpu_info.py | 43 ++++++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_artifacts.yml b/.github/workflows/test_artifacts.yml index d0e7c7eb859..50b2dff40c3 100644 --- a/.github/workflows/test_artifacts.yml +++ b/.github/workflows/test_artifacts.yml @@ -129,6 +129,7 @@ jobs: runs-on: ubuntu-24.04 outputs: sanity_component: ${{ steps.configure.outputs.sanity_component }} + run_gpu_sanity_check: ${{ steps.configure.outputs.run_gpu_sanity_check }} components: ${{ steps.configure.outputs.components }} platform: ${{ steps.configure.outputs.platform }} shard_arr: ${{ steps.configure.outputs.shard_arr }} @@ -193,6 +194,7 @@ jobs: test_sanity_check: name: 'Test Sanity Check' needs: configure_test_matrix + if: ${{ needs.configure_test_matrix.outputs.run_gpu_sanity_check == 'true' }} uses: './.github/workflows/test_component.yml' secrets: inherit with: diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 2a375d8830e..8ba85414679 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -1179,6 +1179,9 @@ def run(): gha_set_output( { "sanity_component": json.dumps(sanity_component), + "run_gpu_sanity_check": str( + bool(sanity_component and sanity_component.get("test_runner")) + ).lower(), "components": json.dumps(output_matrix), "platform": platform, } diff --git a/build_tools/print_driver_gpu_info.py b/build_tools/print_driver_gpu_info.py index 7284377ee14..00175fa6a24 100644 --- a/build_tools/print_driver_gpu_info.py +++ b/build_tools/print_driver_gpu_info.py @@ -8,21 +8,43 @@ On Linux: - run "amd-smi static" - run "rocminfo" + - report the KFD IOCTL version and warn if outside the range required by rocdbgapi (>= 1.13 and < 2.0) On Windows: - run "hipInfo.exe" -This script prints only raw command output. +This script always exits 0; the KFD version check is informational only. """ +import fcntl import os from pathlib import Path import platform import shlex import shutil +import struct import subprocess import sys -from typing import List, Optional +from typing import List, Optional, Tuple + + +# AMDKFD_IOC_GET_VERSION = _IOR('K', 0x01, struct { u32 major; u32 minor; }) +# _IOR: direction=0x80, size=8, type='K'=0x4b, nr=0x01 -> 0x80084b01 +_AMDKFD_IOC_GET_VERSION = 0x80084B01 +_KFD_DEVICE = "/dev/kfd" +_KFD_VERSION_MIN = (1, 13) +_KFD_VERSION_MAX = (2, 0) # exclusive + + +def _get_kfd_version() -> Tuple[int, int]: + fd = os.open(_KFD_DEVICE, os.O_RDWR) + try: + buf = bytearray(8) + fcntl.ioctl(fd, _AMDKFD_IOC_GET_VERSION, buf) + major, minor = struct.unpack("II", buf) + return major, minor + finally: + os.close(fd) def log(*args, **kwargs): @@ -125,6 +147,23 @@ def run_sanity(os_name: str) -> None: extra_command_search_paths=[bin_dir], ) + log("\n=== KFD IOCTL version ===") + if not os.path.exists(_KFD_DEVICE): + log(f"warning: {_KFD_DEVICE} not found — is the AMDGPU driver loaded?") + else: + try: + major, minor = _get_kfd_version() + in_range = _KFD_VERSION_MIN <= (major, minor) < _KFD_VERSION_MAX + status = "supported" if in_range else "WARNING: NOT supported" + log(f"KFD IOCTL version: {major}.{minor} ({status})") + log( + f"Required range for rocdbgapi: " + f">= {_KFD_VERSION_MIN[0]}.{_KFD_VERSION_MIN[1]}" + f" and < {_KFD_VERSION_MAX[0]}.{_KFD_VERSION_MAX[1]}" + ) + except OSError as e: + log(f"warning: failed to query KFD version: {e}") + log("\n=== End of sanity check ===") From 385d9f5ce6d6d026ab9a8b2b0d54a74ad7df8658 Mon Sep 17 00:00:00 2001 From: Luis Machado Date: Thu, 27 Aug 2026 03:49:27 -0500 Subject: [PATCH 2/4] ci: address review feedback on KFD version check PR Drop the run_gpu_sanity_check output variable from fetch_test_configurations.py and test_artifacts.yml per Scott's suggestion. The test_sanity_check job now gates directly on fromJSON(sanity_component).test_runner, which is already non-empty only on GPU runners. Also fix the print_driver_gpu_info.py docstring, which incorrectly claimed the script always exits 0. Since PR #6604 the run_command() calls use check=True, so driver failures produce a non-zero exit. Co-Authored-By: Claude --- .github/workflows/test_artifacts.yml | 3 +-- build_tools/github_actions/fetch_test_configurations.py | 7 +++---- build_tools/print_driver_gpu_info.py | 4 +++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_artifacts.yml b/.github/workflows/test_artifacts.yml index 50b2dff40c3..e27a561be69 100644 --- a/.github/workflows/test_artifacts.yml +++ b/.github/workflows/test_artifacts.yml @@ -129,7 +129,6 @@ jobs: runs-on: ubuntu-24.04 outputs: sanity_component: ${{ steps.configure.outputs.sanity_component }} - run_gpu_sanity_check: ${{ steps.configure.outputs.run_gpu_sanity_check }} components: ${{ steps.configure.outputs.components }} platform: ${{ steps.configure.outputs.platform }} shard_arr: ${{ steps.configure.outputs.shard_arr }} @@ -194,7 +193,7 @@ jobs: test_sanity_check: name: 'Test Sanity Check' needs: configure_test_matrix - if: ${{ needs.configure_test_matrix.outputs.run_gpu_sanity_check == 'true' }} + if: ${{ fromJSON(needs.configure_test_matrix.outputs.sanity_component).test_runner != '' }} uses: './.github/workflows/test_component.yml' secrets: inherit with: diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 8ba85414679..384cc7169f8 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -6,7 +6,9 @@ Outputs (written to $GITHUB_OUTPUT): - sanity_component: JSON object for the sanity component, always present as a - prerequisite that must pass before other components are run. + prerequisite that must pass before other components are run. The + ``test_runner`` field within this object is non-empty only on GPU runners, + so callers can gate GPU-only steps on that field. - components: JSON array of component configs for the regular test matrix (excludes sanity, which is output separately above). - platform: lowercase OS name derived from RUNNER_OS. @@ -1179,9 +1181,6 @@ def run(): gha_set_output( { "sanity_component": json.dumps(sanity_component), - "run_gpu_sanity_check": str( - bool(sanity_component and sanity_component.get("test_runner")) - ).lower(), "components": json.dumps(output_matrix), "platform": platform, } diff --git a/build_tools/print_driver_gpu_info.py b/build_tools/print_driver_gpu_info.py index 00175fa6a24..e4ef6af2f53 100644 --- a/build_tools/print_driver_gpu_info.py +++ b/build_tools/print_driver_gpu_info.py @@ -13,7 +13,9 @@ On Windows: - run "hipInfo.exe" -This script always exits 0; the KFD version check is informational only. +Driver commands (amd-smi, rocminfo, hipInfo) are run with check=True and will +cause this script to exit non-zero if they fail. The KFD version check is +informational only and never causes a non-zero exit. """ import fcntl From cb34cb321eb14619a857ba49e2e4eaa3f70ff26b Mon Sep 17 00:00:00 2001 From: Luis Machado Date: Thu, 27 Aug 2026 03:54:20 -0500 Subject: [PATCH 3/4] ci: make KFD IOCTL version check gating The check now returns 1 (and the script exits non-zero) when /dev/kfd is missing, when the ioctl fails, or when the version falls outside the supported range for rocdbgapi (>= 1.13, < 2.0). Previously these were logged as warnings but never failed the sanity job. Co-Authored-By: Claude --- build_tools/print_driver_gpu_info.py | 42 +++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/build_tools/print_driver_gpu_info.py b/build_tools/print_driver_gpu_info.py index e4ef6af2f53..9c69b5cd52e 100644 --- a/build_tools/print_driver_gpu_info.py +++ b/build_tools/print_driver_gpu_info.py @@ -14,8 +14,9 @@ - run "hipInfo.exe" Driver commands (amd-smi, rocminfo, hipInfo) are run with check=True and will -cause this script to exit non-zero if they fail. The KFD version check is -informational only and never causes a non-zero exit. +cause this script to exit non-zero if they fail. The KFD version check also +causes a non-zero exit when the version is outside the supported range or +cannot be queried. """ import fcntl @@ -113,7 +114,7 @@ def run_command_with_search( log(f"{command}: command not found") -def run_sanity(os_name: str) -> None: +def run_sanity(os_name: str) -> int: THIS_SCRIPT_DIR = Path(__file__).resolve().parent THEROCK_DIR = THIS_SCRIPT_DIR.parent bin_dir = Path(os.getenv("THEROCK_BIN_DIR", THEROCK_DIR / "build" / "bin")) @@ -151,28 +152,31 @@ def run_sanity(os_name: str) -> None: log("\n=== KFD IOCTL version ===") if not os.path.exists(_KFD_DEVICE): - log(f"warning: {_KFD_DEVICE} not found — is the AMDGPU driver loaded?") - else: - try: - major, minor = _get_kfd_version() - in_range = _KFD_VERSION_MIN <= (major, minor) < _KFD_VERSION_MAX - status = "supported" if in_range else "WARNING: NOT supported" - log(f"KFD IOCTL version: {major}.{minor} ({status})") - log( - f"Required range for rocdbgapi: " - f">= {_KFD_VERSION_MIN[0]}.{_KFD_VERSION_MIN[1]}" - f" and < {_KFD_VERSION_MAX[0]}.{_KFD_VERSION_MAX[1]}" - ) - except OSError as e: - log(f"warning: failed to query KFD version: {e}") + log(f"error: {_KFD_DEVICE} not found — is the AMDGPU driver loaded?") + return 1 + try: + major, minor = _get_kfd_version() + in_range = _KFD_VERSION_MIN <= (major, minor) < _KFD_VERSION_MAX + status = "supported" if in_range else "NOT supported" + log(f"KFD IOCTL version: {major}.{minor} ({status})") + log( + f"Required range for rocdbgapi: " + f">= {_KFD_VERSION_MIN[0]}.{_KFD_VERSION_MIN[1]}" + f" and < {_KFD_VERSION_MAX[0]}.{_KFD_VERSION_MAX[1]}" + ) + if not in_range: + return 1 + except OSError as e: + log(f"error: failed to query KFD version: {e}") + return 1 log("\n=== End of sanity check ===") + return 0 def main(argv: Optional[List[str]] = None) -> int: detected = platform.system() - run_sanity(detected) - return 0 + return run_sanity(detected) if __name__ == "__main__": From bfbeba1a4581bacda15e518f0880050f15ae7772 Mon Sep 17 00:00:00 2001 From: Luis Machado Date: Fri, 28 Aug 2026 03:48:56 -0500 Subject: [PATCH 4/4] ci: warn instead of failing when KFD version exceeds the maximum A version below the minimum still hard-fails because it is known to produce broken rocdbgapi output. A version above the rocdbgapi tested ceiling may still work and should not block testing, so it now logs a warning and continues. Co-Authored-By: Claude --- build_tools/print_driver_gpu_info.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/build_tools/print_driver_gpu_info.py b/build_tools/print_driver_gpu_info.py index 9c69b5cd52e..6339e041e4a 100644 --- a/build_tools/print_driver_gpu_info.py +++ b/build_tools/print_driver_gpu_info.py @@ -15,8 +15,9 @@ Driver commands (amd-smi, rocminfo, hipInfo) are run with check=True and will cause this script to exit non-zero if they fail. The KFD version check also -causes a non-zero exit when the version is outside the supported range or -cannot be queried. +causes a non-zero exit when the version is below the supported minimum or +cannot be queried. A version above the supported maximum produces a warning +but does not fail. """ import fcntl @@ -156,15 +157,21 @@ def run_sanity(os_name: str) -> int: return 1 try: major, minor = _get_kfd_version() - in_range = _KFD_VERSION_MIN <= (major, minor) < _KFD_VERSION_MAX - status = "supported" if in_range else "NOT supported" + too_old = (major, minor) < _KFD_VERSION_MIN + too_new = (major, minor) >= _KFD_VERSION_MAX + if too_old: + status = "NOT supported (too old)" + elif too_new: + status = "NOT supported (warning: newer than tested range)" + else: + status = "supported" log(f"KFD IOCTL version: {major}.{minor} ({status})") log( f"Required range for rocdbgapi: " f">= {_KFD_VERSION_MIN[0]}.{_KFD_VERSION_MIN[1]}" f" and < {_KFD_VERSION_MAX[0]}.{_KFD_VERSION_MAX[1]}" ) - if not in_range: + if too_old: return 1 except OSError as e: log(f"error: failed to query KFD version: {e}")