diff --git a/.github/workflows/rdc-dynolog-ci.yml b/.github/workflows/rdc-dynolog-ci.yml new file mode 100644 index 000000000000..b3b2db615008 --- /dev/null +++ b/.github/workflows/rdc-dynolog-ci.yml @@ -0,0 +1,255 @@ +# dynolog <-> RDC Integration CI +# +# dynolog embeds RDC in-process through the shim in dynolog/src/gpumon/amd (no +# rdcd). Compiling and linking that shim against a freshly-built RDC is the +# primary signal: it catches RDC API/ABI/build changes that would break the +# downstream integration. Running the example on a GPU is an extra check that +# soft-passes when no GPU metrics are available. +# +# Embedded mode needs librdc.so and its bootstrap/modules, but not rdcd, rdci or +# librdc_client.so -- the only RDC components that link gRPC. So RDC is built +# with BUILD_STANDALONE=OFF and this workflow never builds gRPC at all; RDC's +# standalone/gRPC surface is rdc-ci.yml's job. +# +# The heavy lifting lives in projects/rdc/tests/dynolog_integration/, runnable +# locally with `python3 -m dynolog_integration `. See its README. +name: RDC dynolog Integration CI + +on: + pull_request: + branches: [develop] + paths: + - 'projects/rdc/include/**' + - 'projects/rdc/rdc_libs/**' + - 'projects/rdc/common/**' + - 'projects/rdc/CMakeLists.txt' + - 'projects/rdc/cmake_modules/**' + - 'projects/rdc/tests/dynolog_integration/**' + - '.github/workflows/rdc-dynolog-ci.yml' + workflow_dispatch: + inputs: + dynolog_ref: + description: 'dynolog ref to build (branch, tag, or commit SHA)' + default: 'b301ce05b3b25e2f48e4525bfc52a0ba6fe77446' + type: string + +permissions: + contents: read + +concurrency: + group: rdc-dynolog-integration-${{ github.ref }} + cancel-in-progress: true + +# Force bash for every `run:` step. The default container shell on +# Debian/Ubuntu images is /bin/sh (dash), which does not support +# `set -o pipefail` or other bashisms used below. +defaults: + run: + shell: bash + +env: + DEBIAN_FRONTEND: noninteractive + DEBCONF_NONINTERACTIVE_SEEN: true + BUILD_TYPE: RelWithDebInfo + ROCM_DIR: /opt/rocm + GPU_TARGETS: ${{ vars.GPU_TARGETS || 'gfx942' }} + DYNOLOG_REPO: https://github.com/facebookincubator/dynolog.git + # Automatic runs pin an immutable SHA so a re-tagged upstream cannot silently + # change what CI builds. Manual runs may override via workflow_dispatch. + DYNOLOG_REF: ${{ inputs.dynolog_ref || 'b301ce05b3b25e2f48e4525bfc52a0ba6fe77446' }} + DYNOLOG_DIR: /tmp/dynolog + EXAMPLE_RUN_SECONDS: '30' + +jobs: + integration-test: + name: RDC dynolog Integration + # Privileged self-hosted GPU runner: never auto-run untrusted fork-PR code on it. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} + runs-on: + - self-hosted + # vars.RUNNER_TYPE must be set to a self-hosted AMD GPU runner label; + # with it unset the job silently never schedules. + - ${{ vars.RUNNER_TYPE }} + timeout-minutes: 60 + container: + image: ${{ vars.RDC_DOCKER_IMAGE || 'rocm/dev-ubuntu-22.04:7.2' }} + options: >- + --rm --privileged + --device=/dev/kfd + --device=/dev/dri + --group-add video + + steps: + - name: Checkout Super-repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + projects/amdsmi + projects/rdc + + - name: Build metadata + run: echo "TODAY=$(date +%Y_%m_%d)" >> "$GITHUB_ENV" + + - name: Install build tools + run: | + set -euo pipefail + ok=0 + for _ in 1 2 3; do + if apt-get update; then ok=1; break; fi + sleep 5 + done + if [ "${ok}" -ne 1 ]; then + echo "::error::apt-get update failed after 3 attempts" + exit 1 + fi + # RDC build deps mirror rdc-ci.yml, minus gRPC's autotools submodules. + # ninja-build is for the dynolog shim's CMake project; glog is not + # installable here and is built from source below. + apt-get install -y --no-install-recommends \ + build-essential ccache clang cmake curl git gzip \ + hip-dev hip-runtime-amd hipcc hsa-rocr-dev \ + libcap-dev libc++-dev libdrm-dev libgflags-dev \ + libmnl-dev libnl-3-dev libnl-genl-3-dev \ + ninja-build pkg-config python3 \ + rocm-core rocm-developer-tools rocm-device-libs rocm-smi-lib \ + rocprofiler-dev rocprofiler-plugins rocprofiler-register rocprofiler-sdk \ + tar zstd + + # Gate the CI helper logic (rocm_version_code / evaluate) before the long + # build so regressions in the helpers are caught fast and never merge + # unexercised. + - name: Run helper unit tests + working-directory: projects/rdc/tests + run: | + set -euo pipefail + python3 -m unittest \ + dynolog_integration.test_build \ + dynolog_integration.test_run_example -v + + # dynolog's shim links glog. The ROCm dev image cannot install + # libgoogle-glog-dev (its libunwind-dev dependency is unavailable there), + # so build a minimal shared glog from source with stack unwinding and the + # optional gflags integration disabled. + - name: Build and install glog + run: | + set -euo pipefail + git clone --depth 1 --branch v0.6.0 https://github.com/google/glog.git /tmp/glog + cmake -S /tmp/glog -B /tmp/glog/build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DWITH_GFLAGS=OFF \ + -DWITH_UNWIND=OFF \ + -DCMAKE_INSTALL_PREFIX=/usr/local + cmake --build /tmp/glog/build --parallel "$(nproc)" + cmake --install /tmp/glog/build + ldconfig + + - name: Cache ccache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.cache/ccache + key: ${{ runner.os }}-rdc-dynolog-ccache-${{ github.ref_name }}-${{ env.TODAY }} + restore-keys: | + ${{ runner.os }}-rdc-dynolog-ccache-${{ github.ref_name }}- + ${{ runner.os }}-rdc-dynolog-ccache- + + - name: Build and install amd-smi + working-directory: projects/amdsmi + run: | + set -euo pipefail + # RDC's metric fetcher links amdsmi_get_cpu_* (ESMI/CPU), so amd-smi + # must be built with the E-SMI library enabled or librdc fails to link. + cmake -B build \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DCMAKE_INSTALL_PREFIX="${ROCM_DIR}" \ + -DENABLE_ESMI_LIB=ON \ + -DBUILD_TESTS=OFF + make -C build -j "$(nproc)" + make -C build install + ldconfig + + - name: Build and install RDC + working-directory: projects/rdc + run: | + set -euo pipefail + # BUILD_STANDALONE=OFF drops rdcd/rdci/librdc_client.so, the only RDC + # components that link gRPC. dynolog embeds RDC in-process and links + # the `rdc` target, whose interface is rdc_bootstrap/pthread/amd_smi/cap + # -- so nothing here needs gRPC, and librdc.so is unchanged either way. + cmake -B build \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DBUILD_STANDALONE=OFF \ + -DBUILD_RUNTIME=ON \ + -DBUILD_PROFILER=ON \ + -DBUILD_RVS=OFF \ + -DBUILD_TESTS=OFF \ + -DGPU_TARGETS="${GPU_TARGETS}" \ + -DCMAKE_INSTALL_PREFIX="${ROCM_DIR}" + make -C build -j "$(nproc)" + make -C build install + ldconfig + # librdc_rocp.so is dlopened at runtime, so find_package(rdc) would not + # catch it going missing. Assert the whole set the shim needs up front. + missing=0 + for f in \ + "${ROCM_DIR}/lib/librdc.so" \ + "${ROCM_DIR}/lib/librdc_bootstrap.so" \ + "${ROCM_DIR}/lib/rdc/librdc_rocp.so" \ + "${ROCM_DIR}/lib/cmake/rdc/rdc-config.cmake" \ + "${ROCM_DIR}/include/rdc/rdc.h"; do + [ -e "$f" ] || { echo "::error::missing expected RDC artifact: $f"; missing=1; } + done + [ "$missing" -eq 0 ] + + - name: Clone dynolog + working-directory: projects/rdc/tests + run: | + python3 -m dynolog_integration clone \ + --repo "${DYNOLOG_REPO}" \ + --ref "${DYNOLOG_REF}" \ + --dest "${DYNOLOG_DIR}" + + - name: Build dynolog RDC wrapper and example + working-directory: projects/rdc/tests + run: | + python3 -m dynolog_integration build-wrapper \ + --dynolog-dir "${DYNOLOG_DIR}" \ + --rocm-dir "${ROCM_DIR}" + + - name: Run RdcWrapperExample + working-directory: projects/rdc/tests + run: | + python3 -m dynolog_integration run-example \ + --binary "${DYNOLOG_DIR}/dynolog/src/gpumon/amd/build/examples/RdcWrapperExample" \ + --rocm-dir "${ROCM_DIR}" \ + --duration "${EXAMPLE_RUN_SECONDS}" \ + --ld-library-path "${ROCM_DIR}/lib:${ROCM_DIR}/lib/rdc" \ + --log-file /tmp/rdc-dynolog-example.log + + - name: Collect logs + if: always() + run: | + set -euo pipefail + mkdir -p /tmp/integration-logs + cp -f /tmp/rdc-dynolog-example.log /tmp/integration-logs/ 2>/dev/null || true + { + echo "=== System ===" + uname -a + echo "=== GPU devices ===" + ls -la /dev/kfd /dev/dri/ 2>&1 || true + echo "=== RDC / AMDSMI libs ===" + ldconfig -p | grep -E 'rdc|amd_smi' || true + } > /tmp/integration-logs/system-info.txt 2>&1 + + - name: Upload integration logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: rdc-dynolog-integration-logs + path: /tmp/integration-logs/ + retention-days: 7 diff --git a/projects/rdc/tests/dynolog_integration/README.md b/projects/rdc/tests/dynolog_integration/README.md new file mode 100644 index 000000000000..c2d5a1f1cf47 --- /dev/null +++ b/projects/rdc/tests/dynolog_integration/README.md @@ -0,0 +1,135 @@ +# dynolog ↔ RDC integration CI helpers + +**dynolog** is a system-level telemetry daemon that runs RDC in-process. dynolog does +not spawn `rdcd`; it embeds RDC in-process through a thin C++ shim, +[`RdcWrapper`](https://github.com/facebookincubator/dynolog/tree/main/dynolog/src/gpumon/amd), +which `#include`s `rdc/rdc.h`, links `librdc`, and creates one GPU group + field +group to watch a fixed set of `RDC_FI_*` fields. + +These helpers back the [`rdc-dynolog-ci.yml`](../../../../.github/workflows/rdc-dynolog-ci.yml) +workflow. They build that shim (and its `RdcWrapperExample`) against an RDC that +was just built from `projects/rdc`, so CI catches RDC changes that would break +the downstream dynolog integration **before** they ship. + +``` +RdcWrapperExample (C++) + -> dynolog RdcWrapper (dynolog's integration shim) + -> librdc / librdc_bootstrap (embedded RDC, built from projects/rdc) + -> libamd_smi.so -> /dev/kfd +``` + +The **primary signal is the shim compile + link** against the freshly-installed +`rdc/rdc.h` and `librdc.so`: an RDC API/ABI/build change that the dynolog +integration cannot tolerate fails the build. Running the example on a GPU is an +additional runtime check that soft-passes when no GPU metrics are available. + +## Subcommands + +Run any phase locally (from `projects/rdc/tests`) with: + +```bash +python3 -m dynolog_integration [options] +``` + +| Subcommand | Purpose | +| --- | --- | +| `clone` | Shallow-fetch dynolog at a pinned ref (branch, tag, or 40-char SHA). | +| `build-wrapper` | Configure + build `dynolog/src/gpumon/amd` (shim + example) against an installed RDC. Auto-detects `ROCM_VERSION` from `/.info/version`. | +| `run-example` | Run `RdcWrapperExample` for a bounded window and verify it initialized RDC, discovered GPU entities, and collected metrics. | + +Example end-to-end run against an RDC already installed under `/opt/rocm`: + +```bash +cd projects/rdc/tests +python3 -m dynolog_integration clone \ + --ref b301ce05b3b25e2f48e4525bfc52a0ba6fe77446 --dest /tmp/dynolog +python3 -m dynolog_integration build-wrapper \ + --dynolog-dir /tmp/dynolog --rocm-dir /opt/rocm +python3 -m dynolog_integration run-example \ + --binary /tmp/dynolog/dynolog/src/gpumon/amd/build/examples/RdcWrapperExample \ + --rocm-dir /opt/rocm \ + --ld-library-path /opt/rocm/lib:/opt/rocm/lib/rdc +``` + +## No gRPC is needed + +gRPC is an RDC *standalone*-mode dependency: only `rdcd`, `rdci` and +`librdc_client.so` link it, and `rdc_libs/CMakeLists.txt` installs +`librdc_client.so` into the exported target set only under `BUILD_STANDALONE`. +dynolog uses embedded mode and links the `rdc` target, whose exported interface +is `rdc_bootstrap;pthread;amd_smi;cap`. + +So the workflow builds RDC with `-DBUILD_STANDALONE=OFF`: `librdc.so`, +`librdc_bootstrap.so` and `librdc_rocp.so` are unchanged (nothing under +`rdc_libs/` other than the `rdc_client` gate reads that option), and the +generated `rdcTargets.cmake` contains no `gRPC::grpc++` reference at all. RDC's +standalone/gRPC surface is `rdc-ci.yml`'s job, not this workflow's. + +Avoiding gRPC here also avoids pinning a second gRPC version: TheRock builds +RDC against `THEROCK_GRPC_VERSION` (`third-party/grpc/`, currently 1.78.1, +statically linked), which is also RDC's own `GRPC_DESIRED_VERSION` default. A +hand-rolled gRPC in this workflow would drift from what actually ships. + +## Prerequisites for the build + +The shim's standalone CMake requires an RDC SDK + `amd_smi` (normally under +`/opt/rocm`), plus `glog` and `gflags`. On a stock Ubuntu host: + +```bash +apt-get install -y ninja-build cmake libgoogle-glog-dev libgflags-dev +``` + +`libcap` is additionally needed to *build RDC itself* (`find_library(LIB_CAP)`), +and is pulled in transitively when linking the shim against `librdc.so`. + +> **CI note:** the ROCm dev container the workflow runs in +> (`rocm/dev-ubuntu-22.04`) cannot install `libgoogle-glog-dev` — its +> `libunwind-dev` dependency is unavailable there — so the workflow builds +> **glog v0.6.0 from source** (`-DWITH_UNWIND=OFF`) into `/usr/local` instead. +> See the workflow's *Build and install glog* step. + +## Tests + +Stdlib-only unit tests cover the ROCm-version math and the example-output +verification (no GPU required): + +```bash +cd projects/rdc/tests +python3 -m pytest dynolog_integration # or: +python3 -m unittest dynolog_integration.test_build dynolog_integration.test_run_example +``` + +## CI configuration + +The workflow runs on a **self-hosted AMD GPU runner** and needs these repo/org +settings (the same GPU runner convention the DME integration CI uses): + +| Setting | Required | Purpose | +| --- | --- | --- | +| `vars.RUNNER_TYPE` | **Yes** | Second `runs-on` label selecting the self-hosted GPU runner. If unset, the job never schedules and eventually times out. | +| `vars.GPU_TARGETS` | No (default `gfx942`) | `GPU_TARGETS` for the RDC build; set to the runner's arch (e.g. `gfx90a`, `gfx950`) so profiler kernels load. | +| `vars.RDC_DOCKER_IMAGE` | No (default `rocm/dev-ubuntu-22.04:7.2`) | ROCm build container image. | + +The runtime `RdcWrapperExample` step is intentionally lenient: it **soft-passes** +(with a `::warning::`) when a started example collects no metrics or throws on an +unsupported field, since those depend on the runner's GPU arch rather than the +RDC build/link surface. The build step is the primary gate. Pass `--strict` to +`run-example` to gate on metric collection on a known-good runner. + +It still hard-fails on the cases no GPU arch explains: the binary not starting +at all (runtime link/load break), a non-capability fatal from `RdcWrapper`, the +process exiting on its own (the poll loop is meant to run until terminated), and +setup never completing within the window (a hang). + +## Keeping the dynolog pin current + +Automatic runs build dynolog at the immutable SHA pinned in `DYNOLOG_REF` +(`.github/workflows/rdc-dynolog-ci.yml`). Because the shim is built against +`develop`'s RDC, a **deliberate** RDC API change that the pinned dynolog has not +yet adopted will (correctly) fail the build until the integration is updated. To +resolve such drift: + +* re-run against a newer dynolog via **Actions -> Run workflow** and set the + `dynolog_ref` input, then bump `DYNOLOG_REF` to that ref; or +* coordinate the corresponding change upstream in + `facebookincubator/dynolog` first. diff --git a/projects/rdc/tests/dynolog_integration/__init__.py b/projects/rdc/tests/dynolog_integration/__init__.py new file mode 100644 index 000000000000..dc83de309136 --- /dev/null +++ b/projects/rdc/tests/dynolog_integration/__init__.py @@ -0,0 +1,15 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""dynolog <-> RDC integration CI helpers. + +The **dynolog** system-level telemetry daemon drives RDC through a thin C++ +shim, ``RdcWrapper`` (``dynolog/src/gpumon/amd``), which embeds RDC in +process (no separate ``rdcd``). These helpers build that shim and its example +against a freshly-built RDC so CI catches RDC changes that would break the +downstream dynolog integration. Each phase is a subcommand:: + + python3 -m dynolog_integration [options] + +See ``__main__.py`` for the available subcommands. +""" diff --git a/projects/rdc/tests/dynolog_integration/__main__.py b/projects/rdc/tests/dynolog_integration/__main__.py new file mode 100644 index 000000000000..cc5b730fb033 --- /dev/null +++ b/projects/rdc/tests/dynolog_integration/__main__.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Single CLI entrypoint dispatching to ``dynolog_integration`` subcommands. + +Usage from a workflow step:: + + python3 -m dynolog_integration clone --ref --dest /tmp/dynolog + python3 -m dynolog_integration build-wrapper --dynolog-dir /tmp/dynolog \ + --rocm-dir /opt/rocm + python3 -m dynolog_integration run-example --binary --duration 20 + +Each subcommand simply forwards to the matching module's ``main``. +""" + +from __future__ import annotations + +import sys + +from . import build, clone, run_example + +_SUBCOMMANDS = {"clone": clone.main, "build-wrapper": build.main, "run-example": run_example.main} + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if not argv or argv[0] in {"-h", "--help"}: + print(__doc__) + print("Available subcommands:") + for name in _SUBCOMMANDS: + print(f" {name}") + return 0 if argv else 2 + + sub = argv[0] + handler = _SUBCOMMANDS.get(sub) + if handler is None: + print(f"Unknown subcommand: {sub}", file=sys.stderr) + print(f"Available: {', '.join(_SUBCOMMANDS)}", file=sys.stderr) + return 2 + return handler(argv[1:]) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/rdc/tests/dynolog_integration/_common.py b/projects/rdc/tests/dynolog_integration/_common.py new file mode 100644 index 000000000000..ceb87147781c --- /dev/null +++ b/projects/rdc/tests/dynolog_integration/_common.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Shared utilities for the dynolog <-> RDC integration CI helpers.""" + +from __future__ import annotations + +import logging +import shlex +import subprocess +import sys +from pathlib import Path + +_LOG_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" + + +def configure_logging(verbose: bool = False) -> None: + level = logging.DEBUG if verbose else logging.INFO + logging.basicConfig(level=level, format=_LOG_FORMAT, stream=sys.stderr) + + +def run(cmd: list[str], *, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + """Run a subprocess, logging the command for CI visibility. + + Fails fast on a non-zero exit. Output streams to the parent process so + GitHub Actions log grouping works correctly. + """ + logger = logging.getLogger("dynolog.run") + logger.info("$ %s", " ".join(shlex.quote(c) for c in cmd)) + return subprocess.run(cmd, cwd=str(cwd) if cwd else None, check=True, text=True) + + +def gh_error(message: str) -> None: + print(f"::error::{message}", flush=True) + + +def gh_warning(message: str) -> None: + print(f"::warning::{message}", flush=True) diff --git a/projects/rdc/tests/dynolog_integration/build.py b/projects/rdc/tests/dynolog_integration/build.py new file mode 100644 index 000000000000..7e4045c47998 --- /dev/null +++ b/projects/rdc/tests/dynolog_integration/build.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Build the dynolog RDC shim (``dynolog/src/gpumon/amd``) against a built RDC. + +Configures and builds the standalone target defined by dynolog's +``src/gpumon/amd/CMakeLists.txt`` -- the ``dynolog_rdc_lib`` shim plus the +``RdcWrapperExample`` binary -- against the RDC that was installed under +``--rocm-dir``. This compile+link is the primary CI signal: it fails when an +RDC change breaks the header/library surface the dynolog integration depends +on. + +The standalone build gates its metric set on ``DYNOLOG_ROCM_VERSION``; if that +is left undefined every metric is preprocessed out and the example aborts at +runtime with "enabledMetrics is empty". The version code is derived from the +installed ROCm the same way dynolog's own CMake derives it -- from +``/.info/version`` as ``major*10000 + minor*100 + patch`` (e.g. +``70200`` for ROCm 7.2.0) -- and passed explicitly as ``-DROCM_VERSION``. +""" + +from __future__ import annotations + +import argparse +import logging +import os +import re +import sys +from pathlib import Path + +from ._common import configure_logging, gh_error, gh_warning, run + +logger = logging.getLogger("dynolog.build") + +# Relative path from a dynolog checkout to the standalone RDC shim CMake project. +_SHIM_CMAKE_SUBDIR = Path("dynolog/src/gpumon/amd") +# Where the example binary lands inside the shim build tree. +_EXAMPLE_SUBPATH = Path("examples/RdcWrapperExample") + +# Fallback used only when the installed ROCm version cannot be read. 70000 keeps +# the full ROCm 7.0 metric set enabled so the example is still meaningful. +_DEFAULT_ROCM_VERSION_CODE = 70000 + +_VERSION_RE = re.compile(r"\d+") + + +def rocm_version_code(rocm_dir: Path) -> int: + """Return ``major*10000 + minor*100 + patch`` for the ROCm at ``rocm_dir``. + + Reads ``/.info/version`` (e.g. ``7.2.0-...``) and mirrors the + version-code math in dynolog's ``src/gpumon/amd/CMakeLists.txt``. Falls back + to ``_DEFAULT_ROCM_VERSION_CODE`` with a warning if the file is missing or + unparsable. + """ + version_file = rocm_dir / ".info" / "version" + try: + raw = version_file.read_text().splitlines()[0] + except (OSError, IndexError): + gh_warning( + f"Could not read {version_file}; defaulting ROCM_VERSION to " + f"{_DEFAULT_ROCM_VERSION_CODE}. Pass --rocm-version to override." + ) + return _DEFAULT_ROCM_VERSION_CODE + + parts = _VERSION_RE.findall(raw) + if len(parts) < 3: + gh_warning( + f"Unexpected version string {raw!r} in {version_file}; defaulting " + f"ROCM_VERSION to {_DEFAULT_ROCM_VERSION_CODE}." + ) + return _DEFAULT_ROCM_VERSION_CODE + + major, minor, patch = (int(p) for p in parts[:3]) + code = major * 10000 + minor * 100 + patch + logger.info("Detected ROCm %s.%s.%s -> ROCM_VERSION=%d", major, minor, patch, code) + return code + + +def build( + *, + dynolog_dir: Path, + rocm_dir: Path, + rocm_version: int, + build_dir: Path, + build_type: str, + jobs: int, +) -> Path: + """Configure + build the shim/example, returning the example binary path.""" + shim_dir = dynolog_dir / _SHIM_CMAKE_SUBDIR + if not (shim_dir / "CMakeLists.txt").is_file(): + gh_error(f"dynolog RDC shim not found at {shim_dir}") + raise SystemExit(1) + + # The shim links the `rdc` target only (embedded mode), whose interface is + # rdc_bootstrap/pthread/amd_smi/cap -- no gRPC -- so the ROCm prefix is the + # only one find_package(rdc/amd_smi) needs. + run( + [ + "cmake", + "-S", + str(shim_dir), + "-B", + str(build_dir), + "-G", + "Ninja", + f"-DROCM_VERSION={rocm_version}", + f"-DCMAKE_BUILD_TYPE={build_type}", + f"-DCMAKE_PREFIX_PATH={rocm_dir}", + ] + ) + run(["cmake", "--build", str(build_dir), "--parallel", str(jobs)]) + + example = build_dir / _EXAMPLE_SUBPATH + if not example.is_file(): + gh_error(f"Build succeeded but example binary is missing: {example}") + raise SystemExit(1) + logger.info("Built dynolog RDC example: %s", example) + return example + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dynolog-dir", required=True, type=Path) + parser.add_argument("--rocm-dir", default=Path("/opt/rocm"), type=Path) + parser.add_argument( + "--rocm-version", + type=int, + default=None, + help="ROCM_VERSION code (major*10000+minor*100+patch). Auto-detected if omitted.", + ) + parser.add_argument("--build-dir", default=None, type=Path) + parser.add_argument("--build-type", default="RelWithDebInfo") + parser.add_argument("--jobs", type=int, default=os.cpu_count() or 1) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args(argv) + configure_logging(verbose=args.verbose) + + build_dir = args.build_dir or (args.dynolog_dir / _SHIM_CMAKE_SUBDIR / "build") + rocm_version = ( + args.rocm_version if args.rocm_version is not None else rocm_version_code(args.rocm_dir) + ) + build( + dynolog_dir=args.dynolog_dir, + rocm_dir=args.rocm_dir, + rocm_version=rocm_version, + build_dir=build_dir, + build_type=args.build_type, + jobs=args.jobs, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/rdc/tests/dynolog_integration/clone.py b/projects/rdc/tests/dynolog_integration/clone.py new file mode 100644 index 000000000000..a66a75a95cf7 --- /dev/null +++ b/projects/rdc/tests/dynolog_integration/clone.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Clone dynolog at a pinned ref for the RDC integration build. + +Only the in-tree RDC shim (``dynolog/src/gpumon/amd``) is needed, so this does +a shallow, single-commit fetch and skips submodules. The ref may be a branch, +tag, or full 40-char commit SHA; GitHub allows fetching an arbitrary SHA +directly, which keeps a pinned automatic run reproducible without cloning the +whole (large) dynolog history. +""" + +from __future__ import annotations + +import argparse +import logging +import shutil +import sys +from pathlib import Path + +from ._common import configure_logging, gh_error, run + +logger = logging.getLogger("dynolog.clone") + +# Force SSH->HTTPS on CI hosts without SSH keys, without mutating the runner's +# persistent global git config (per-command ``-c`` entries only). Only the +# fetch resolves the remote URL, so only it needs these. +_INSTEAD_OF = [ + "-c", + "url.https://github.com/.insteadOf=git@github.com:", + "-c", + "url.https://github.com/.insteadOf=ssh://git@github.com/", +] + +# The RDC shim source lives here; a successful clone must contain it. +_RDC_SHIM_SUBPATH = Path("dynolog/src/gpumon/amd/RdcWrapper.cpp") + + +def clone(*, repo: str, ref: str, dest: Path) -> Path: + """Shallow-clone ``repo`` at ``ref`` into ``dest`` and return ``dest``.""" + if ref.startswith("-"): + gh_error(f"refusing suspicious ref that looks like an option: {ref!r}") + raise SystemExit(1) + if dest.exists(): + logger.info("Removing existing dynolog dir: %s", dest) + shutil.rmtree(dest) + dest.mkdir(parents=True, exist_ok=True) + + run(["git", "init", "--quiet", str(dest)]) + run(["git", "remote", "add", "origin", repo], cwd=dest) + + # ``git fetch --depth 1 `` accepts a branch, tag, or (on GitHub) a full + # commit SHA, fetching just that one commit instead of the whole history. + run(["git", *_INSTEAD_OF, "fetch", "--depth", "1", "origin", ref], cwd=dest) + run(["git", "checkout", "--detach", "FETCH_HEAD"], cwd=dest) + + shim = dest / _RDC_SHIM_SUBPATH + if not shim.is_file(): + gh_error(f"dynolog clone missing RDC shim: {shim} (bad ref '{ref}'?)") + raise SystemExit(1) + + logger.info("dynolog ready at %s (ref=%s)", dest, ref) + return dest + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", default="https://github.com/facebookincubator/dynolog.git") + parser.add_argument( + "--ref", required=True, help="Branch, tag, or full 40-char commit SHA to check out." + ) + parser.add_argument("--dest", required=True, type=Path) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args(argv) + configure_logging(verbose=args.verbose) + clone(repo=args.repo, ref=args.ref, dest=args.dest) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/rdc/tests/dynolog_integration/run_example.py b/projects/rdc/tests/dynolog_integration/run_example.py new file mode 100644 index 000000000000..309bd117b5ec --- /dev/null +++ b/projects/rdc/tests/dynolog_integration/run_example.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Run dynolog's ``RdcWrapperExample`` and verify it drives RDC end to end. + +The example embeds RDC exactly as dynolog does: it constructs ``RdcWrapper``, +discovers GPU entities, watches the RDC field set, then loops forever polling +metrics. This helper launches it with ``ROCM_PATH`` / ``LD_LIBRARY_PATH`` set, +lets it run for a bounded window, terminates it (the poll loop never returns by +design), and inspects the captured output. + +The RDC build/link step is the primary integration signal; this runtime check is +a bonus smoke test. Gating: + +* Not started (no ``Initializing RDC example`` banner) -> hard fail: the binary + could not load/run against the freshly-built RDC (a missing shared library or + an undefined symbol at load time) -- a real runtime-linkage break. +* Fatal setup error (``RDC example failed: ...``): soft-pass only when no GPU is + present or the message is a recognized capability/field limitation (e.g. an + unsupported field on this arch). Any other fatal -- ``rdc_init``, + ``rdc_start_embedded``, GPU group create/add, field-watch, ... -- is a genuine + RDC regression and hard-fails. +* Self-exit before we stop it: the poll loop is meant to run until terminated, + so a process that exits on its own (e.g. a crash after a healthy-looking + sample) hard-fails even if some metrics were printed first. +* Healthy (started, >=1 entity, >=1 metric, still running) -> pass. +* Never reported an entity count on a GPU box -> hard fail: RdcWrapper setup + neither finished nor threw within the window, which is a hang, not arch + variance. +* Started, discovered entities, but collected no metric on a GPU box -> + soft-pass with a warning (usually all enabled fields are N/A for the arch); + ``--strict`` turns this into a hard failure. +""" + +from __future__ import annotations + +import argparse +import logging +import os +import re +import signal +import subprocess +import sys +from pathlib import Path + +from ._common import configure_logging, gh_error, gh_warning + +logger = logging.getLogger("dynolog.run_example") + +_KFD_DEVICE = Path("/dev/kfd") + +_STARTED_RE = re.compile(r"Initializing RDC example", re.MULTILINE) +_FATAL_RE = re.compile(r"RDC example failed:\s*(?P.*)", re.MULTILINE) +_ENTITIES_RE = re.compile(r"Monitoring (?P\d+) entities", re.MULTILINE) +_COLLECTED_RE = re.compile(r"collected (?P\d+) metrics", re.MULTILINE) + +# A fatal message matching this is an acceptable capability/arch limitation +# (the build already validated the integration surface), not an RDC regression. +_CAPABILITY_RE = re.compile(r"not[ _]?supported|unsupported", re.IGNORECASE) + + +def gpu_present() -> bool: + """True if an AMD GPU compute device (``/dev/kfd``) is available.""" + return _KFD_DEVICE.exists() + + +def _monitored_entities(log: str) -> int | None: + matches = _ENTITIES_RE.findall(log) + return int(matches[-1]) if matches else None + + +def _max_metrics_collected(log: str) -> int: + return max((int(m) for m in _COLLECTED_RE.findall(log)), default=0) + + +def evaluate( + log: str, *, gpu_available: bool, self_exited: bool, strict: bool = False +) -> tuple[bool, bool, str]: + """Assess an example run from its captured output. + + Returns ``(ok, soft_pass, reason)``. A pure function of the inputs so it is + unit-testable without hardware. See the module docstring for the gating + rationale. + """ + if not _STARTED_RE.search(log): + return ( + False, + False, + "example binary did not start (no 'Initializing RDC example' banner) " + "-- possible runtime link/load failure against the built RDC", + ) + + fatal = _FATAL_RE.search(log) + if fatal is not None: + msg = fatal.group("msg").strip() or "no detail" + # Acceptable: no hardware, or a recognized capability/field limitation. + if not gpu_available or _CAPABILITY_RE.search(msg): + return (True, True, f"setup did not complete ({msg})") + # Anything else (init / embedded-start / group / field-watch) is a real + # RDC runtime regression. + return (False, False, f"RdcWrapper reported a fatal error: {msg}") + + # The free-running poll loop should only stop when we terminate it; a + # self-exit (e.g. a crash after a healthy-looking prefix) is unexpected. + if self_exited: + return ( + False, + False, + "example exited on its own before it was stopped -- unexpected for the " + "free-running poll loop (possible crash after start)", + ) + + entities = _monitored_entities(log) + metrics = _max_metrics_collected(log) + + # Healthy run: discovered entities and read at least one metric. + if entities and metrics > 0: + return (True, False, f"collected up to {metrics} metrics from {entities} entities") + + if entities is None: + # Never printed an entity count and never threw: RdcWrapper construction + # did not return. No arch explains that, so on a GPU box it is a hang. + reason = ( + "example never reported an entity count -- RdcWrapper setup neither " + "completed nor failed within the run window (possible hang in " + "rdc_init / rdc_start_embedded / field-watch)" + ) + if gpu_available: + return (False, False, reason) + elif entities == 0: + reason = "no GPU entities were discovered (Monitoring 0 entities)" + else: + # Usually every enabled field is N/A for this arch. + reason = "no metrics were collected from any entity" + + if not gpu_available: + return (True, True, f"no GPU present; runtime validation skipped ({reason})") + if strict: + return (False, False, reason) + return (True, True, reason) + + +def run( + *, + binary: Path, + duration: float, + rocm_dir: Path, + ld_library_path: str | None, + log_file: Path, + strict: bool = False, + extra_args: list[str] | None = None, +) -> int: + if not binary.is_file(): + gh_error(f"example binary not found: {binary}") + return 1 + + env = os.environ.copy() + env["ROCM_PATH"] = str(rocm_dir) + if ld_library_path: + env["LD_LIBRARY_PATH"] = ld_library_path + + cmd = [str(binary), *(extra_args or [])] + logger.info("running example for %.0fs: %s", duration, " ".join(cmd)) + log_file.parent.mkdir(parents=True, exist_ok=True) + + self_exited = False + with log_file.open("wb") as log_handle: + proc = subprocess.Popen( + cmd, stdout=log_handle, stderr=subprocess.STDOUT, env=env, start_new_session=True + ) + try: + # The example loops forever on success, so a return within the window + # means it stopped on its own (threw, or crashed). TimeoutExpired is + # the healthy path. + proc.wait(timeout=duration) + self_exited = True + logger.info("example exited on its own with code %s", proc.returncode) + except subprocess.TimeoutExpired: + logger.info("example still running after %.0fs -- stopping it", duration) + _terminate(proc) + + log = log_file.read_text(errors="replace") + _echo_log(log) + + present = gpu_present() + ok, soft, reason = evaluate(log, gpu_available=present, self_exited=self_exited, strict=strict) + ctx = "GPU present" if present else "no GPU detected" + if not ok: + gh_error(f"dynolog RdcWrapperExample validation failed ({ctx}): {reason}") + return 1 + if soft: + gh_warning(f"dynolog RdcWrapperExample soft-passed ({ctx}): {reason}") + else: + logger.info("dynolog RdcWrapperExample OK (%s): %s", ctx, reason) + return 0 + + +def _terminate(proc: subprocess.Popen) -> None: + for sig in (signal.SIGTERM, signal.SIGKILL): + if proc.poll() is not None: + break + try: + os.killpg(os.getpgid(proc.pid), sig) + except ProcessLookupError: + break + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + continue + + +def _echo_log(log: str, lines: int = 40) -> None: + tail = log.splitlines()[-lines:] + if tail: + logger.info("---- RdcWrapperExample output (last %d lines) ----", len(tail)) + for line in tail: + print(line, file=sys.stderr) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", required=True, type=Path) + parser.add_argument("--duration", type=float, default=30.0) + parser.add_argument("--rocm-dir", default=Path("/opt/rocm"), type=Path) + parser.add_argument("--ld-library-path", default=None) + parser.add_argument("--log-file", default=Path("/tmp/rdc-dynolog-example.log"), type=Path) + parser.add_argument( + "--strict", + action="store_true", + help="Hard-fail when a started example collects no metrics on a GPU box " + "(default: soft-pass, since that is usually arch field-availability).", + ) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args(argv) + configure_logging(verbose=args.verbose) + return run( + binary=args.binary, + duration=args.duration, + rocm_dir=args.rocm_dir, + ld_library_path=args.ld_library_path, + log_file=args.log_file, + strict=args.strict, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/rdc/tests/dynolog_integration/test_build.py b/projects/rdc/tests/dynolog_integration/test_build.py new file mode 100644 index 000000000000..8a62ca94b0ce --- /dev/null +++ b/projects/rdc/tests/dynolog_integration/test_build.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Stdlib-only unit tests for ``dynolog_integration.build``.""" + +import contextlib +import io +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +# Allow running directly as well as via unittest discovery from +# ``projects/rdc/tests``. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from dynolog_integration import build + + +def _version_code_quiet(rocm_dir: Path) -> int: + """Call rocm_version_code, swallowing its ``::warning::`` stdout. + + The fallback paths emit a GitHub workflow ``::warning::`` command; without + this, running the tests inside the workflow would surface those as + misleading run-level annotations. + """ + with contextlib.redirect_stdout(io.StringIO()): + return build.rocm_version_code(rocm_dir) + + +class RocmVersionCodeTest(unittest.TestCase): + def _rocm_dir_with_version(self, contents: str) -> Path: + tmp = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + info = tmp / ".info" + info.mkdir() + (info / "version").write_text(contents) + return tmp + + def test_parses_major_minor_patch(self): + rocm_dir = self._rocm_dir_with_version("7.2.0-12345\n") + self.assertEqual(build.rocm_version_code(rocm_dir), 70200) + + def test_parses_two_digit_minor(self): + rocm_dir = self._rocm_dir_with_version("10.1.3\n") + self.assertEqual(build.rocm_version_code(rocm_dir), 100103) + + def test_missing_version_file_falls_back(self): + empty = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, empty, ignore_errors=True) + self.assertEqual(_version_code_quiet(empty), build._DEFAULT_ROCM_VERSION_CODE) + + def test_unparseable_version_falls_back(self): + rocm_dir = self._rocm_dir_with_version("not-a-version\n") + self.assertEqual(_version_code_quiet(rocm_dir), build._DEFAULT_ROCM_VERSION_CODE) + + +if __name__ == "__main__": + unittest.main() diff --git a/projects/rdc/tests/dynolog_integration/test_run_example.py b/projects/rdc/tests/dynolog_integration/test_run_example.py new file mode 100644 index 000000000000..c9d918536142 --- /dev/null +++ b/projects/rdc/tests/dynolog_integration/test_run_example.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Stdlib-only unit tests for ``dynolog_integration.run_example`` verification.""" + +import sys +import unittest +from pathlib import Path + +# Allow running directly as well as via unittest discovery from +# ``projects/rdc/tests``. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from dynolog_integration import run_example + +# A representative healthy run: started, discovered entities, collected metrics. +_HEALTHY_LOG = """\ +Initializing RDC example, ROCM version: 70200 +RDC watch config: update_interval=1000ms, max_keep_age=10s, max_keep_samples=100 +Collecting 30 metrics: + metric RDC_FI_GPU_UTIL (id=500) +Monitoring 8 entities (physical GPUs + partitions): + entity index=0 +entity 0: collected 30 metrics + entity 0 RDC_FI_GPU_UTIL (id=500)=42 +entity 1: collected 30 metrics +""" + +# Fatal due to a capability/field gap on this arch (acceptable on a GPU box). +_CAPABILITY_FATAL_LOG = """\ +Initializing RDC example, ROCM version: 70200 +Collecting 30 metrics: +RDC example failed: rdc_field_watch failed: RDC_ST_NOT_SUPPORTED +""" + +# Fatal due to a genuine setup failure (a real RDC regression). +_INIT_FATAL_LOG = """\ +Initializing RDC example, ROCM version: 70200 +Collecting 30 metrics: +RDC example failed: rdc_start_embedded failed: RDC_ST_FAIL_LOAD_MODULE +""" + +# Started and discovered GPUs but never read a metric. +_NO_METRICS_LOG = """\ +Initializing RDC example, ROCM version: 70200 +Monitoring 8 entities (physical GPUs + partitions): +entity 0: collected 0 metrics +""" + +# Started, then hung in RdcWrapper setup: no entity count, no fatal, no exit. +_HUNG_LOG = """\ +Initializing RDC example, ROCM version: 70200 +Collecting 30 metrics: + metric RDC_FI_GPU_UTIL (id=500) +""" + + +class EvaluateTest(unittest.TestCase): + def test_healthy_run_passes(self): + ok, soft, reason = run_example.evaluate(_HEALTHY_LOG, gpu_available=True, self_exited=False) + self.assertTrue(ok) + self.assertFalse(soft) + self.assertIn("30 metrics", reason) + + def test_never_started_hard_fails(self): + for gpu in (True, False): + ok, soft, _ = run_example.evaluate( + "garbage output\n", gpu_available=gpu, self_exited=False + ) + self.assertFalse(ok) + self.assertFalse(soft) + + def test_capability_fatal_soft_passes_on_gpu(self): + ok, soft, _ = run_example.evaluate( + _CAPABILITY_FATAL_LOG, gpu_available=True, self_exited=True + ) + self.assertTrue(ok) + self.assertTrue(soft) + + def test_genuine_fatal_hard_fails_on_gpu(self): + ok, soft, reason = run_example.evaluate( + _INIT_FATAL_LOG, gpu_available=True, self_exited=True + ) + self.assertFalse(ok) + self.assertFalse(soft) + self.assertIn("fatal error", reason) + + def test_any_fatal_soft_passes_without_gpu(self): + ok, soft, _ = run_example.evaluate(_INIT_FATAL_LOG, gpu_available=False, self_exited=True) + self.assertTrue(ok) + self.assertTrue(soft) + + def test_self_exit_after_healthy_hard_fails(self): + # Designed to loop forever; a self-exit after a healthy prefix is a crash. + ok, soft, reason = run_example.evaluate(_HEALTHY_LOG, gpu_available=True, self_exited=True) + self.assertFalse(ok) + self.assertFalse(soft) + self.assertIn("exited on its own", reason) + + def test_no_metrics_soft_passes_by_default_on_gpu(self): + ok, soft, reason = run_example.evaluate( + _NO_METRICS_LOG, gpu_available=True, self_exited=False + ) + self.assertTrue(ok) + self.assertTrue(soft) + self.assertIn("no metrics", reason) + + def test_no_metrics_hard_fails_under_strict(self): + ok, soft, _ = run_example.evaluate( + _NO_METRICS_LOG, gpu_available=True, self_exited=False, strict=True + ) + self.assertFalse(ok) + self.assertFalse(soft) + + def test_no_metrics_soft_passes_without_gpu(self): + ok, soft, _ = run_example.evaluate(_NO_METRICS_LOG, gpu_available=False, self_exited=False) + self.assertTrue(ok) + self.assertTrue(soft) + + def test_hang_hard_fails_on_gpu(self): + # No entity count and no fatal means setup never returned; no GPU arch + # explains that, so it must not be excused as field availability. + ok, soft, reason = run_example.evaluate(_HUNG_LOG, gpu_available=True, self_exited=False) + self.assertFalse(ok) + self.assertFalse(soft) + self.assertIn("never reported an entity count", reason) + + def test_hang_soft_passes_without_gpu(self): + ok, soft, _ = run_example.evaluate(_HUNG_LOG, gpu_available=False, self_exited=False) + self.assertTrue(ok) + self.assertTrue(soft) + + +class ParseHelpersTest(unittest.TestCase): + def test_monitored_entities_reads_last(self): + self.assertEqual(run_example._monitored_entities(_HEALTHY_LOG), 8) + + def test_monitored_entities_none_when_absent(self): + self.assertIsNone(run_example._monitored_entities("no entities here")) + + def test_max_metrics_collected(self): + self.assertEqual(run_example._max_metrics_collected(_HEALTHY_LOG), 30) + + def test_max_metrics_zero_when_absent(self): + self.assertEqual(run_example._max_metrics_collected("nothing"), 0) + + +if __name__ == "__main__": + unittest.main()