Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/test_artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ jobs:
test_sanity_check:
name: 'Test Sanity Check'
needs: configure_test_matrix
if: ${{ fromJSON(needs.configure_test_matrix.outputs.sanity_component).test_runner != '' }}
uses: './.github/workflows/test_component.yml'
secrets: inherit
with:
Expand Down
4 changes: 3 additions & 1 deletion build_tools/github_actions/fetch_test_configurations.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@

Outputs (written to $GITHUB_OUTPUT):
- sanity_component: JSON object for the sanity component, always present as a
prerequisite that must pass before other components are run.
prerequisite that must pass before other components are run. The
``test_runner`` field within this object is non-empty only on GPU runners,
so callers can gate GPU-only steps on that field.
- components: JSON array of component configs for the regular test matrix
(excludes sanity, which is output separately above).
- platform: lowercase OS name derived from RUNNER_OS.
Expand Down
62 changes: 57 additions & 5 deletions build_tools/print_driver_gpu_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,47 @@
On Linux:
- run "amd-smi static"
- run "rocminfo"
- report the KFD IOCTL version and warn if outside the range required by rocdbgapi (>= 1.13 and < 2.0)

On Windows:
- run "hipInfo.exe"

This script prints only raw command output.
Driver commands (amd-smi, rocminfo, hipInfo) are run with check=True and will
cause this script to exit non-zero if they fail. The KFD version check also
causes a non-zero exit when the version is below the supported minimum or
cannot be queried. A version above the supported maximum produces a warning
but does not fail.
"""

import fcntl

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This import fails on Windows, and sanity checks failed on Windows on this PR's presubmit run: https://github.com/ROCm/TheRock/actions/runs/33499470761/job/99867709885.

Fixing in #7820

cc @amd-justchen , see https://github.com/ROCm/TheRock/blob/main/docs/rfcs/RFC0002-MonoRepo-Gardener-Rotations.md#responsibilities-and-priorities

Responsibilities and priorities

  1. When regressions occur that break CI workflows
    • Communicate that something is broken
    • Identify the culprit commit(s)
    • Notify the authors/reviewers of those commits
    • Revert those changes
    • If it's not obvious how to, facilitate getting changes in to get back to green (see 2)

The general sequence is "comment on the PR about the break" --> "revert" --> "let the author fix"

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can also add a unit test for this script that at least checks that it can be run on CPU runners without failing at import time (CPU runners won't have the tools, so those calls could be mocked out)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These shouldn't be running on CPU runners at all.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We missed this because the "Windows::release / Test gfx110X-all / Test Sanity Check / Test sanity (shard 1/1) (gfx110X-all)" job that failed ran 2 hours after the CI started and was not blocking. The "unit tests" job runs in < 5 minutes and is blocking, but it only runs on CPU (github-hosted) runners. A unit test that checked that the script could import on windows and linux would have caught this and blocked the merge.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, yeah that makes sense. I had the context wrong at first. Let me craft a PR for that.

import os
from pathlib import Path
import platform
import shlex
import shutil
import struct
import subprocess
import sys
from typing import List, Optional
from typing import List, Optional, Tuple


# AMDKFD_IOC_GET_VERSION = _IOR('K', 0x01, struct { u32 major; u32 minor; })
# _IOR: direction=0x80, size=8, type='K'=0x4b, nr=0x01 -> 0x80084b01
_AMDKFD_IOC_GET_VERSION = 0x80084B01
_KFD_DEVICE = "/dev/kfd"
_KFD_VERSION_MIN = (1, 13)
_KFD_VERSION_MAX = (2, 0) # exclusive


def _get_kfd_version() -> Tuple[int, int]:
fd = os.open(_KFD_DEVICE, os.O_RDWR)
try:
buf = bytearray(8)
fcntl.ioctl(fd, _AMDKFD_IOC_GET_VERSION, buf)
major, minor = struct.unpack("II", buf)
return major, minor
finally:
os.close(fd)


def log(*args, **kwargs):
Expand Down Expand Up @@ -89,7 +115,7 @@ def run_command_with_search(
log(f"{command}: command not found")


def run_sanity(os_name: str) -> None:
def run_sanity(os_name: str) -> int:
THIS_SCRIPT_DIR = Path(__file__).resolve().parent
THEROCK_DIR = THIS_SCRIPT_DIR.parent
bin_dir = Path(os.getenv("THEROCK_BIN_DIR", THEROCK_DIR / "build" / "bin"))
Expand Down Expand Up @@ -125,13 +151,39 @@ def run_sanity(os_name: str) -> None:
extra_command_search_paths=[bin_dir],
)

log("\n=== KFD IOCTL version ===")
if not os.path.exists(_KFD_DEVICE):
log(f"error: {_KFD_DEVICE} not found — is the AMDGPU driver loaded?")
return 1
try:
major, minor = _get_kfd_version()
too_old = (major, minor) < _KFD_VERSION_MIN
too_new = (major, minor) >= _KFD_VERSION_MAX
if too_old:
status = "NOT supported (too old)"
elif too_new:
status = "NOT supported (warning: newer than tested range)"
else:
status = "supported"
log(f"KFD IOCTL version: {major}.{minor} ({status})")
log(
f"Required range for rocdbgapi: "
f">= {_KFD_VERSION_MIN[0]}.{_KFD_VERSION_MIN[1]}"
f" and < {_KFD_VERSION_MAX[0]}.{_KFD_VERSION_MAX[1]}"
)
if too_old:
return 1
except OSError as e:
log(f"error: failed to query KFD version: {e}")
return 1

log("\n=== End of sanity check ===")
return 0


def main(argv: Optional[List[str]] = None) -> int:
detected = platform.system()
run_sanity(detected)
return 0
return run_sanity(detected)


if __name__ == "__main__":
Expand Down
Loading