diff --git a/.github/requirements-hipdnn-superbuild-ci.txt b/.github/requirements-hipdnn-superbuild-ci.txt index b83d2610d325..002f05191e06 100644 --- a/.github/requirements-hipdnn-superbuild-ci.txt +++ b/.github/requirements-hipdnn-superbuild-ci.txt @@ -7,9 +7,17 @@ # numpy -- imported by frontend_wheel_package/tests, not by the build. # jsonschema -- the descriptor-packaging suite validates UHD sidecars against # uhd.schema.json; without it those checks skip. +# PyYAML, Jinja2 -- the DescriptorGenerator/IngestorGenerator tooling suites (HIPDNN_ENABLE_TOOLING_TESTS). +# clang-format -- IngestorGenerator tests/test_packs_formatting.py, which skips (and a strict +# run fails the skip) without one. Pinned to the pre-commit hook's rev, because a formatter +# fixed-point check is only meaningful against the formatter that defines the style. +# tools/CMakeLists.txt hands the venv's copy to the suite as HIPDNN_CLANG_FORMAT. # -# Installed with --upgrade, so nothing is pinned here. +# Installed with --upgrade, so nothing else is pinned here. pytest build numpy jsonschema +PyYAML +Jinja2 +clang-format==18.1.4 diff --git a/.github/scripts/check_probe_junit.py b/.github/scripts/check_probe_junit.py deleted file mode 100755 index f8f4b379715f..000000000000 --- a/.github/scripts/check_probe_junit.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -# SPDX-License-Identifier: MIT - -"""Prove that every declared hipDNN packaging probe ran and passed. - -The superbuild lanes run the whole ctest suite once with --output-junit. ctest -exits 0 when a test is disabled or not run, so this script reads that junit file -and checks it against the manifest the CMake configure wrote (one ctest test -name per line). Testcases not named in the manifest are ignored. - -The manifest always names the probe tooling test "hkp-probe-tools", plus one entry -per declared probe. A lane whose GPU_TARGETS cover every architecture the -descriptors name declares no probe, so "hkp-probe-tools" alone is a valid manifest; -an empty one means the configure did not write it properly. - -The manifest is written by the same configure that discovers the probes, so on its -own it cannot tell "this lane builds every architecture" from "discovery found -nothing". A lane that must always declare probes passes --min-probes N: the check -fails when the manifest names fewer than N tests besides "hkp-probe-tools". The -default is 0, which accepts a manifest with no probe. - -For every manifest name, the junit must hold exactly one testcase with that -name, and that testcase must have run (status "run" or "fail"; without a status -attribute, no child) and must have no or child. - -Usage: - check_probe_junit.py --junit build/ctest-junit.xml \ - --manifest build/hkp-probes/manifest.txt [--min-probes N] - -Exit codes: - 0 every manifest name ran exactly once and passed - 1 one or more checks failed; each prints - "check_probe_junit: FAIL : ", where is one of - manifest-unreadable, manifest-too-small, probes-too-few, junit-unreadable, - name-missing, name-duplicate, not-run, failure, error -""" - -import argparse -import sys -import xml.etree.ElementTree as ET -from pathlib import Path - -# The "hkp-probe-tools" test; probes are optional unless --min-probes asks for them. -MIN_MANIFEST_ENTRIES = 1 -TOOLS_TEST = "hkp-probe-tools" - - -def read_manifest(path: Path) -> list[str]: - return [line.strip() for line in path.read_text().splitlines() if line.strip()] - - -def ran(case: ET.Element) -> bool: - """Return whether ctest ran the testcase. - - ctest writes status "run", "fail", "disabled" or "notrun"; only the first - two mean the test executed. - """ - status = case.get("status") - if status is None: - return case.find("skipped") is None - return status in ("run", "fail") - - -def check( - junit_path: Path, manifest_path: Path, min_probes: int = 0 -) -> list[tuple[str, str]]: - """Return (id, detail) for every failed check; empty means green.""" - try: - names = read_manifest(manifest_path) - except OSError as exc: - return [("manifest-unreadable", f"{manifest_path}: {exc}")] - - failures: list[tuple[str, str]] = [] - if len(names) < MIN_MANIFEST_ENTRIES: - failures.append( - ( - "manifest-too-small", - f"{manifest_path} lists {len(names)} test(s), expected at least " - f"{MIN_MANIFEST_ENTRIES} (hkp-probe-tools)", - ) - ) - - probes = [name for name in names if name != TOOLS_TEST] - if len(probes) < min_probes: - failures.append( - ( - "probes-too-few", - f"{manifest_path} declares {len(probes)} probe(s), expected at " - f"least {min_probes}; if this lane now builds every architecture " - f"the descriptors name, drop --min-probes for it, otherwise probe " - f"discovery found nothing", - ) - ) - - try: - root = ET.parse(junit_path).getroot() - except (OSError, ET.ParseError) as exc: - failures.append(("junit-unreadable", f"{junit_path}: {exc}")) - return failures - - cases: dict[str, list[ET.Element]] = {name: [] for name in names} - for case in root.iter("testcase"): - found = cases.get(case.get("name", "")) - if found is not None: - found.append(case) - - for name, found in cases.items(): - if not found: - failures.append( - ("name-missing", f"declared test '{name}' is not in the junit") - ) - continue - if len(found) > 1: - failures.append( - ( - "name-duplicate", - f"declared test '{name}' appears {len(found)} times in the junit", - ) - ) - for case in found: - if not ran(case): - status = case.get("status") - failures.append( - ("not-run", f"test '{name}' did not run (status={status})") - ) - if case.get("status") == "fail" or case.find("failure") is not None: - failures.append(("failure", f"test '{name}' failed")) - if case.find("error") is not None: - failures.append(("error", f"test '{name}' errored")) - - return failures - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument( - "--junit", required=True, type=Path, help="ctest --output-junit file" - ) - parser.add_argument( - "--manifest", - required=True, - type=Path, - help="file listing one declared ctest test name per line", - ) - parser.add_argument( - "--min-probes", - type=int, - default=0, - help="fail when the manifest names fewer probes than this, not counting " - f"{TOOLS_TEST} (default 0: no probe is acceptable)", - ) - args = parser.parse_args(argv) - - failures = check(args.junit, args.manifest, args.min_probes) - for check_id, detail in failures: - print(f"check_probe_junit: FAIL {check_id}: {detail}") - if failures: - return 1 - count = len(read_manifest(args.manifest)) - print(f"check_probe_junit: OK {count} declared test(s) ran and passed") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/scripts/tests/test_check_probe_junit.py b/.github/scripts/tests/test_check_probe_junit.py deleted file mode 100644 index 0cfa93080774..000000000000 --- a/.github/scripts/tests/test_check_probe_junit.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -# SPDX-License-Identifier: MIT - -import os -import sys -import tempfile -import unittest -from pathlib import Path - -sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) -import check_probe_junit as cpj - -MANIFEST = ["hkp-probe-gfx950", "hkp-probe-tools"] - - -def case_xml(name: str, child: str = "", status: str | None = None) -> str: - body = f"<{child}/>" if child else "" - if status is None: - status = {"skipped": "notrun", "failure": "fail"}.get(child, "run") - status_attr = f' status="{status}"' if status else "" - return ( - f'' - f"{body}" - ) - - -def junit_xml(cases: list[str]) -> str: - return ( - '' - f'{"".join(cases)}' - ) - - -class CheckProbeJunitTest(unittest.TestCase): - def setUp(self): - tmp = tempfile.TemporaryDirectory() - self.addCleanup(tmp.cleanup) - self.dir = Path(tmp.name) - - def run_check( - self, xml: str, manifest: list[str], min_probes: int = 0 - ) -> list[str]: - junit_path = self.dir / "ctest-junit.xml" - manifest_path = self.dir / "manifest.txt" - junit_path.write_text(xml) - manifest_path.write_text("".join(f"{n}\n" for n in manifest)) - return [ - check_id for check_id, _ in cpj.check(junit_path, manifest_path, min_probes) - ] - - def test_all_declared_tests_passed(self): - xml = junit_xml([case_xml(n) for n in MANIFEST]) - self.assertEqual(self.run_check(xml, MANIFEST), []) - - def test_skipped_test_fails(self): - xml = junit_xml([case_xml(MANIFEST[0]), case_xml(MANIFEST[1], "skipped")]) - self.assertEqual(self.run_check(xml, MANIFEST), ["not-run"]) - - def test_disabled_test_without_skipped_child_fails(self): - xml = junit_xml( - [case_xml(MANIFEST[0], status="disabled"), case_xml(MANIFEST[1])] - ) - self.assertEqual(self.run_check(xml, MANIFEST), ["not-run"]) - - def test_unknown_status_fails(self): - xml = junit_xml([case_xml(MANIFEST[0], status="bogus"), case_xml(MANIFEST[1])]) - self.assertEqual(self.run_check(xml, MANIFEST), ["not-run"]) - - def test_skipped_child_without_status_fails(self): - xml = junit_xml( - [case_xml(MANIFEST[0], "skipped", status=""), case_xml(MANIFEST[1])] - ) - self.assertEqual(self.run_check(xml, MANIFEST), ["not-run"]) - - def test_missing_status_without_skipped_child_passes(self): - xml = junit_xml([case_xml(n, status="") for n in MANIFEST]) - self.assertEqual(self.run_check(xml, MANIFEST), []) - - def test_unrelated_testcases_are_ignored(self): - xml = junit_xml( - [ - case_xml("hipdnn-unit-tests"), - case_xml(MANIFEST[0]), - case_xml("other-disabled", status="disabled"), - case_xml("other-failed", "failure"), - case_xml(MANIFEST[1]), - ] - ) - self.assertEqual(self.run_check(xml, MANIFEST), []) - - def test_duplicate_declared_name_fails(self): - xml = junit_xml([case_xml(n) for n in MANIFEST] + [case_xml(MANIFEST[0])]) - self.assertEqual(self.run_check(xml, MANIFEST), ["name-duplicate"]) - - def test_declared_name_missing_fails(self): - xml = junit_xml([case_xml(MANIFEST[0]), case_xml("hipdnn-unit-tests")]) - self.assertEqual(self.run_check(xml, MANIFEST), ["name-missing"]) - - def test_empty_manifest_fails(self): - xml = junit_xml([case_xml(n) for n in MANIFEST]) - self.assertIn("manifest-too-small", self.run_check(xml, [])) - - def test_manifest_without_probe_passes(self): - xml = junit_xml([case_xml("hkp-probe-tools")]) - self.assertEqual(self.run_check(xml, ["hkp-probe-tools"]), []) - - def test_failure_fails(self): - xml = junit_xml([case_xml(MANIFEST[0], "failure"), case_xml(MANIFEST[1])]) - self.assertEqual(self.run_check(xml, MANIFEST), ["failure"]) - - def test_fail_status_without_failure_child_fails(self): - xml = junit_xml([case_xml(MANIFEST[0], status="fail"), case_xml(MANIFEST[1])]) - self.assertEqual(self.run_check(xml, MANIFEST), ["failure"]) - - def test_error_fails(self): - xml = junit_xml([case_xml(MANIFEST[0]), case_xml(MANIFEST[1], "error")]) - self.assertEqual(self.run_check(xml, MANIFEST), ["error"]) - - def test_unparseable_junit_fails(self): - self.assertEqual(self.run_check("> "$GITHUB_PATH" - # --output-junit is relative to --test-dir: this writes build/ctest-junit.xml. - name: Run tests - id: run-tests - run: ctest --test-dir build --output-on-failure --output-junit ctest-junit.xml - - # ctest counts a disabled or not-run test as success. Read the junit - # "Run tests" wrote and fail if a declared packaging probe (listed in the - # manifest configure wrote) did not run or failed. Runs whenever "Run - # tests" ran, pass or fail, so a probe failure is reported by name. - # --min-probes 1: this lane never builds every architecture the descriptors - # name, so a manifest with no probe means discovery broke. Drop the flag if - # the lane starts building them all. - - name: Check packaging probes ran - if: ${{ !cancelled() && steps.run-tests.outcome != 'skipped' }} - run: | - "$VENV_PYTHON" .github/scripts/check_probe_junit.py \ - --junit build/ctest-junit.xml \ - --manifest build/hkp-probes/manifest.txt \ - --min-probes 1 + run: ctest --test-dir build --output-on-failure --no-tests=error - name: Patch ROCm SDK with superbuild outputs shell: pwsh @@ -513,6 +499,21 @@ jobs: test "$count" -gt 0 || { echo "dvc pull produced no golden blobs" >&2; exit 1; } echo "Fetched $count golden tensor blob(s)" + # test_registry_regeneration.py regenerates the gfx950 attention catalog and + # diffs it against the shipped KDP. The rocKE KDPs are DVC-managed (each pointer + # names the `ingestor` remote), and this lane packs the example descriptors + # rather than the production tree, so nothing else fetches them. Without them + # both tests skip, and the tooling suite fails skips inside pytest. Same pull as + # the Windows lane, which packs the production tree. + - name: Fetch rocKE descriptors + run: | + set -euo pipefail + LOGNAME=github-runner dvc pull --recursive dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/descriptors + # A pull can exit 0 having fetched nothing; count what it should produce. + count=$(find dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/descriptors -name '*.kdp.json' -path '*artifacts*' | wc -l) + test "$count" -gt 0 || { echo "dvc pull produced no rocKE KDPs" >&2; exit 1; } + echo "Fetched $count rocKE KDP(s)" + # Must precede configure: configuration fails if pytest is not importable # by the interpreter descriptor-packaging registers its tests under. The # interpreter is named explicitly because it must be the one passed as @@ -534,6 +535,7 @@ jobs: -DHIPKERNELPROVIDER_ENABLE_ROCKE=ON \ -DHIPDNN_ENABLE_CUDNN_COMPATIBILITY=ON \ -DHIPDNN_ENABLE_KERNEL_INGESTOR=ON \ + -DHIPDNN_ENABLE_TOOLING_TESTS=ON \ -DHIPDNN_ENABLE_SDPA=ON \ -DHIPKERNELPROVIDER_PRODUCTION_SOURCE_ROOT="$GITHUB_WORKSPACE/dnn-providers/hip-kernel-provider/descriptor-packaging/examples/descriptors" \ -DHIPKERNELPROVIDER_ROCKE_COMGR_LIB="$HKP_ROCKE_COMGR_LIB" \ @@ -543,7 +545,8 @@ jobs: -DFETCHCONTENT_SOURCE_DIR_TSL_ROBIN_MAP="$THIRD_PARTY_DEPS_DIR/robin-map-1.4.1" \ -DPython_EXECUTABLE="$VENV_PYTHON" \ -DPython3_EXECUTABLE="$VENV_PYTHON" \ - -DHIPKERNELPROVIDER_ENABLE_PACKAGING_PROBES=ON + -DHIPKERNELPROVIDER_ENABLE_PACKAGING_PROBES=ON \ + -DHIPKERNELPROVIDER_MIN_PACKAGING_PROBES=1 - name: Build run: cmake --build build @@ -558,28 +561,11 @@ jobs: test "${#kpacks[@]}" -gt 0 || { echo "no .kpack archives produced" >&2; exit 1; } printf '%s\n' "${kpacks[@]}" - # --output-junit is relative to --test-dir: this writes build/ctest-junit.xml. - name: Run tests - id: run-tests env: # Native rocKE AUTO detection must see the wheel COMGR used by packaging. ROCKE_COMGR_LIB: ${{ env.HKP_ROCKE_COMGR_LIB }} - run: ctest --test-dir build --output-on-failure --output-junit ctest-junit.xml - - # ctest counts a disabled or not-run test as success. Read the junit - # "Run tests" wrote and fail if a declared packaging probe (listed in the - # manifest configure wrote) did not run or failed. Runs whenever "Run - # tests" ran, pass or fail, so a probe failure is reported by name. - # --min-probes 1: this lane never builds every architecture the descriptors - # name, so a manifest with no probe means discovery broke. Drop the flag if - # the lane starts building them all. - - name: Check packaging probes ran - if: ${{ !cancelled() && steps.run-tests.outcome != 'skipped' }} - run: | - "$VENV_PYTHON" .github/scripts/check_probe_junit.py \ - --junit build/ctest-junit.xml \ - --manifest build/hkp-probes/manifest.txt \ - --min-probes 1 + run: ctest --test-dir build --output-on-failure --no-tests=error # Golden-data validation used to live here as a schedule-gated re-run of the # external integration targets with HIPDNN_TEST_VERIFICATION_MODE=golden-check. diff --git a/dnn-providers/hip-kernel-provider/descriptor-packaging/CMakeLists.txt b/dnn-providers/hip-kernel-provider/descriptor-packaging/CMakeLists.txt index ffcc570dd840..9f67c9766e67 100644 --- a/dnn-providers/hip-kernel-provider/descriptor-packaging/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/descriptor-packaging/CMakeLists.txt @@ -8,5 +8,13 @@ include(HkpPackaging) option(HIPKERNELPROVIDER_ENABLE_PACKAGING_PROBES "CI-only, non-shipping: register packaging probes (build-tree ctest entries hkp-probe-*)" OFF) +set(HIPKERNELPROVIDER_MIN_PACKAGING_PROBES 0 CACHE STRING + "Fail configure when fewer than this many packaging probes are declared. A lane that never builds every architecture the descriptors name sets 1, so broken probe discovery cannot pass as a lane with nothing to probe") +if(HIPKERNELPROVIDER_MIN_PACKAGING_PROBES GREATER 0 AND NOT HIPKERNELPROVIDER_ENABLE_PACKAGING_PROBES) + message(FATAL_ERROR + "hkp: HIPKERNELPROVIDER_MIN_PACKAGING_PROBES=${HIPKERNELPROVIDER_MIN_PACKAGING_PROBES} " + "needs -DHIPKERNELPROVIDER_ENABLE_PACKAGING_PROBES=ON: with probes off none are " + "declared, so the minimum could never be met.") +endif() hkp_add_packaging() diff --git a/dnn-providers/hip-kernel-provider/descriptor-packaging/README.md b/dnn-providers/hip-kernel-provider/descriptor-packaging/README.md index 0c26579345d3..216e0f85def7 100644 --- a/dnn-providers/hip-kernel-provider/descriptor-packaging/README.md +++ b/dnn-providers/hip-kernel-provider/descriptor-packaging/README.md @@ -462,7 +462,8 @@ configure. (prebuilt `.co` files) and C++/runtime paths, and install staging. - Device behavior: no kernel is launched, no numerics are checked. - Wildcard descriptors on architectures no lane builds (see below for why none ship). -- That the CI job runs: that is the job of the junit check below. +- That the CI job runs: a lane that must always have a probe sets + `HIPKERNELPROVIDER_MIN_PACKAGING_PROBES` (below). ### Which architectures are probed @@ -482,9 +483,10 @@ hkp: probe gfx950: derived root /hkp-probes/gfx950/root (17 UKD(s) for gf A new architecture, or a new pack under one, is therefore probed with no edit anywhere. A lane whose `GPU_TARGETS` cover every named architecture declares no probe; that is valid, -and the manifest then lists `hkp-probe-tools` alone. A lane that must always have a probe -passes `--min-probes N` to `check_probe_junit.py`, which fails when the manifest names fewer -than N probes; without it, a tools-only manifest is accepted. +and only `hkp-probe-tools` is registered. A lane that must always have a probe configures +with `-DHIPKERNELPROVIDER_MIN_PACKAGING_PROBES=N`, which fails configure when fewer than N +probes are declared, so broken probe discovery cannot pass as a lane with nothing to probe; +the default, 0, accepts no probe. **Wildcards are excluded.** A descriptor with no `arch` (absent or empty) names no architecture and gets no automatic probe. That is safe: a wildcard ships for whatever the @@ -640,10 +642,7 @@ cmake --preset hipdnn-dev-all -GNinja -B build-probe \ -DENABLE_CLANG_FORMAT=OFF -DENABLE_CLANG_TIDY=OFF -DROCM_LIBS_ENABLE_ROOT_CTEST=ON \ -DHIPKERNELPROVIDER_ENABLE_PACKAGING_PROBES=ON cmake --build build-probe --target hkp_packaging_probes # or the full build -ctest --test-dir build-probe -R '^hkp-probe-' --no-tests=error --output-on-failure \ - --output-junit probe-junit.xml -python3 .github/scripts/check_probe_junit.py \ - --junit build-probe/probe-junit.xml --manifest build-probe/hkp-probes/manifest.txt +ctest --test-dir build-probe -R '^hkp-probe-' --no-tests=error --output-on-failure ``` The selector is `^hkp-probe-`, anchored: `-R` is an unanchored regex. It matches every @@ -651,18 +650,14 @@ probe (`hkp-probe-`) and `hkp-probe-tools`, the pytest suite of the probe (`probes/tests/`; the tests that pack rocKE content live in `probes/tests/rocke/` and nowhere else, which a build with `HIPKERNELPROVIDER_ENABLE_ROCKE=OFF` never collects). `--no-tests=error` makes an empty selection a failure instead of a -green run that proved nothing. `manifest.txt` is written at configure and lists the ctest -names that must run; the junit check fails unless every one ran and passed (a skipped, -disabled or missing entry fails), and ignores testcases not in the manifest, so it also -accepts the junit of a full ctest run. Configuring without the option and running the same +green run that proved nothing. Configuring without the option and running the same build and ctest lines fails (unknown target, then "No tests were found"). In CI both superbuild lanes configure with the option `ON`. The probe packs run in the Build step as part of `all`; the probe ctest entries run once, inside the lane's normal -"Run tests" ctest, which writes `build/ctest-junit.xml`. The "Check packaging probes ran" -step then runs only `check_probe_junit.py` on that junit against the manifest, with -`--min-probes 1` because neither lane builds every architecture the production root names -(drop the flag from a lane that does). It runs whenever "Run tests" ran, pass or fail. +"Run tests" ctest (`--no-tests=error`). Both lanes configure with +`HIPKERNELPROVIDER_MIN_PACKAGING_PROBES=1` because neither builds every architecture the +production root names (drop it from a lane that does). Locally on Linux the build is quick (the wheel environment plus the probed UKDs) and each probe ctest entry takes a few seconds; `hkp-probe-tools` takes longer, since it really @@ -716,7 +711,7 @@ does not build it, with the default sample; nothing needs declaring. archive rule). This is a design decision, not an entry. After adding an architecture or a probe line, reconfigure and run the local block above: -the new `hkp-probe-` entry and its manifest line must appear. Then run the +the new `hkp-probe-` ctest entry must appear. Then run the mutation checks below. ### Mutation checks when adding a probe diff --git a/dnn-providers/hip-kernel-provider/descriptor-packaging/cmake/HkpPackagingProbes.cmake b/dnn-providers/hip-kernel-provider/descriptor-packaging/cmake/HkpPackagingProbes.cmake index 38504ee76167..6d9f00062945 100644 --- a/dnn-providers/hip-kernel-provider/descriptor-packaging/cmake/HkpPackagingProbes.cmake +++ b/dnn-providers/hip-kernel-provider/descriptor-packaging/cmake/HkpPackagingProbes.cmake @@ -408,11 +408,11 @@ endfunction() # and UKDs (hkp_probe_derive_root.py --list-arches; wildcard descriptors name none, # generic targets map to a member, see _hkp_probe_automatic_arches) minus # hkp_selected_arches(), each as hkp_add_packaging_probe(ARCH ). Register -# the probe tooling tests, write the manifest of ctest names -# (${CMAKE_BINARY_DIR}/hkp-probes/manifest.txt) and define the aggregate target +# the probe tooling tests and define the aggregate target # hkp_packaging_probes. Zero probes is valid: every explicit arch is a build target, -# so the production pack covers it. Configuration fails when the production root -# cannot be read or pytest is not importable by Python3_EXECUTABLE. +# so the production pack covers it, unless HIPKERNELPROVIDER_MIN_PACKAGING_PROBES asks +# for more. Configuration fails when the production root cannot be read, pytest is not +# importable by Python3_EXECUTABLE, or fewer probes are declared than that minimum. # --------------------------------------------------------------------------- function(hkp_load_packaging_probes rocm_kpack_dir hipcc rocke_comgr_lib) set_property(GLOBAL PROPERTY HKP_PROBE_ROCM_KPACK_DIR "${rocm_kpack_dir}") @@ -439,6 +439,18 @@ function(hkp_load_packaging_probes rocm_kpack_dir hipcc rocke_comgr_lib) get_property(_names GLOBAL PROPERTY HKP_PROBE_NAMES) + # Broken discovery and a build with nothing to probe both declare zero probes; a lane that + # must always have one sets the minimum. + list(LENGTH _names _probe_count) + if(_probe_count LESS HIPKERNELPROVIDER_MIN_PACKAGING_PROBES) + message(FATAL_ERROR + "hkp: ${_probe_count} packaging probe(s) declared, but " + "HIPKERNELPROVIDER_MIN_PACKAGING_PROBES requires at least " + "${HIPKERNELPROVIDER_MIN_PACKAGING_PROBES}. If this build now targets every " + "architecture the descriptors name, lower the minimum; otherwise probe " + "discovery found nothing.") + endif() + execute_process( COMMAND "${Python3_EXECUTABLE}" -c "import pytest" RESULT_VARIABLE _pytest_rc @@ -472,15 +484,10 @@ function(hkp_load_packaging_probes rocm_kpack_dir hipcc rocke_comgr_lib) ${_rocke_ignore}) set_tests_properties(hkp-probe-tools PROPERTIES ENVIRONMENT "${_pyenv}") - set(_test_names hkp-probe-tools) set(_targets "") foreach(_name IN LISTS _names) - list(APPEND _test_names "hkp-probe-${_name}") list(APPEND _targets "hkp_packaging_probe_${_name}") endforeach() - list(SORT _test_names) - string(REPLACE ";" "\n" _manifest "${_test_names}") - file(WRITE "${CMAKE_BINARY_DIR}/hkp-probes/manifest.txt" "${_manifest}\n") add_custom_target(hkp_packaging_probes COMMENT "hkp: packaging probes") if(_targets) diff --git a/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/rocke/test_hkp_pack_parallel_rocke.py b/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/rocke/test_hkp_pack_parallel_rocke.py index bd23694f6d73..8a77e8ca1ba2 100644 --- a/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/rocke/test_hkp_pack_parallel_rocke.py +++ b/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/rocke/test_hkp_pack_parallel_rocke.py @@ -6,6 +6,7 @@ import concurrent.futures import importlib +import multiprocessing import pickle import re import sys @@ -100,14 +101,18 @@ def test_prewarm_failure_names_variant(failing_corpus, tmp_path, monkeypatch): @pytest.mark.quick def test_worker_inherits_the_in_tree_rocke_paths(): - """A pool worker imports rocke and kernels from the paths this directory's + """A spawned pool worker imports rocke and kernels from the paths this directory's conftest inserts, which no `PYTHONPATH` carries, so it can only have them by - inheriting the parent's `sys.path`.""" + inheriting the parent's `sys.path`. The pool is pinned to `spawn`: under the Linux + default (`fork` up to Python 3.13) the child copies the parent's memory and the + check would prove nothing.""" candidates = [_ROCKE_ROOT / "platform" / "python", _ROCKE_ROOT / "library"] expected = _conftest_only_paths(candidates) assert len(expected) == 2, "the premise: conftest inserted both, PYTHONPATH neither" - with concurrent.futures.ProcessPoolExecutor(max_workers=1) as pool: + with concurrent.futures.ProcessPoolExecutor( + max_workers=1, mp_context=multiprocessing.get_context("spawn") + ) as pool: child_path = list(pool.map(_child_sys_path, [None], chunksize=1))[0] assert set(expected) <= set(child_path) diff --git a/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/test_hkp_pack_parallel.py b/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/test_hkp_pack_parallel.py index f9087cf149ca..e7476cf2bf93 100644 --- a/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/test_hkp_pack_parallel.py +++ b/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/test_hkp_pack_parallel.py @@ -10,6 +10,7 @@ import concurrent.futures import itertools +import multiprocessing import os import re import sys @@ -25,7 +26,6 @@ from pack_helpers import ( GENERIC_TARGETS, _child_sys_path, - _conftest_only_paths, _kdp, _silent, _ukd, @@ -803,8 +803,8 @@ def test_variant_key_for_uses_module_globals(monkeypatch): @pytest.mark.quick -def test_worker_inherits_parent_sys_path(): - """A pool worker starts with the parent's `sys.path`, conftest inserts and all. +def test_worker_inherits_parent_sys_path(tmp_path, monkeypatch): + """A spawned pool worker starts with the parent's `sys.path`, conftest inserts and all. CPython propagates `sys.path` to children under both `spawn` and `forkserver`, so a worker can import `hkp_pack`, and any producer module a @@ -817,24 +817,22 @@ def test_worker_inherits_parent_sys_path(): pool must be built after parent-side path setup, is documented at the construction site. - The path watched is the rocm_kpack directory conftest inserts from - HIPKERNELPROVIDER_ROCM_KPACK_DIR, which no `PYTHONPATH` carries. The hkp_pack - package root conftest also inserts is exported by the ctest environment, so a - child that re-read `PYTHONPATH` would have it too. + The pool is pinned to `spawn`: under `fork`, the Linux default up to Python 3.13, + the child inherits `sys.path` with the rest of the parent's memory and the check + would prove nothing. + The path watched is one the test prepends itself, in a directory no `PYTHONPATH` + carries, so it reaches the child only if the interpreter propagates it. """ - kpack_dir = os.environ.get("HIPKERNELPROVIDER_ROCM_KPACK_DIR") - if not kpack_dir: - pytest.skip( - "HIPKERNELPROVIDER_ROCM_KPACK_DIR is unset, so conftest inserts no " - "path that PYTHONPATH does not already carry" - ) - expected = _conftest_only_paths([kpack_dir]) - assert expected, "the premise: conftest inserted it and PYTHONPATH lacks it" + probe = tmp_path / "parent-only-path" + probe.mkdir() + monkeypatch.syspath_prepend(str(probe)) - with concurrent.futures.ProcessPoolExecutor(max_workers=1) as pool: + with concurrent.futures.ProcessPoolExecutor( + max_workers=1, mp_context=multiprocessing.get_context("spawn") + ) as pool: child_path = list(pool.map(_child_sys_path, [None], chunksize=1))[0] - assert set(expected) <= set(child_path) + assert str(probe) in child_path @pytest.mark.quick diff --git a/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/test_specialization_agreement.py b/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/test_specialization_agreement.py index 4bd950a73a93..b36e5f654158 100644 --- a/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/test_specialization_agreement.py +++ b/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/test_specialization_agreement.py @@ -31,6 +31,10 @@ from hkp_pack.kpack_resolver import load_kpack from pack_helpers import GENERIC_TARGETS, GENERIC_TARGETS_JSON, write_shipped +# quick: no hipcc/comgr (the marker's definition); TestRealArchiveReaders needs only +# rocm_kpack, as other quick files do. +pytestmark = pytest.mark.quick + ARCH = "gfx942" PAYLOAD = b"\x7fELF-not-really-a-code-object" PAYLOAD_SHA = hashlib.sha256(PAYLOAD).hexdigest() diff --git a/projects/hipdnn/CMakeLists.txt b/projects/hipdnn/CMakeLists.txt index 51d2c1ad463a..07db20a821eb 100644 --- a/projects/hipdnn/CMakeLists.txt +++ b/projects/hipdnn/CMakeLists.txt @@ -214,6 +214,9 @@ if(DEFINED HIP_DNN_SKIP_TESTS) message(DEPRECATION "HIP_DNN_SKIP_TESTS has been renamed to HIPDNN_SKIP_TESTS. Please update your build scripts to use -DHIPDNN_SKIP_TESTS instead.") endif() option(HIPDNN_SKIP_TESTS "Skips building all tests" OFF) +option(HIPDNN_ENABLE_TOOLING_TESTS + "Register the DescriptorGenerator/IngestorGenerator pytest suites as ctest tests (needs pytest, PyYAML and Jinja2 importable by Python3_EXECUTABLE)" + OFF) option(HIPDNN_ENABLE_COVERAGE "Build with code coverage flags" OFF) option(BUILD_ADDRESS_SANITIZER "Build with Address Sanitizer enabled" OFF) diff --git a/projects/hipdnn/test_categories.yaml b/projects/hipdnn/test_categories.yaml index 7bf6f9396b4a..5bd3dec1c083 100644 --- a/projects/hipdnn/test_categories.yaml +++ b/projects/hipdnn/test_categories.yaml @@ -45,6 +45,10 @@ test_categories: - "hipdnn_frontend_tests" - "hipdnn_test_sdk_tests" - "hipdnn_plugin_sdk_tests" + # Build tree only, registered only with HIPDNN_ENABLE_TOOLING_TESTS=ON; see + # tools/CMakeLists.txt. + - "hipdnn_descriptor_generator_pytest" + - "hipdnn_ingestor_generator_pytest" # Build tree only; see plugin_sdk/tests/CMakeLists.txt. - "hipdnn_gpu_generic_targets_.*" diff --git a/projects/hipdnn/tools/CMakeLists.txt b/projects/hipdnn/tools/CMakeLists.txt index 064292a5efd5..7be479c91af4 100644 --- a/projects/hipdnn/tools/CMakeLists.txt +++ b/projects/hipdnn/tools/CMakeLists.txt @@ -128,6 +128,52 @@ endif() _apply_hipdnn_test_category_labels(hipdnn_list_engines_smoke) endif() +# DescriptorGenerator pytest suite, then the IngestorGenerator one below. Build tree only, like +# the smoke tests above. Both run under the hipdnn_strict_skips plugin: ctest reports a suite +# whose cases all skip as a pass, so a missing compiler or binary would otherwise go unnoticed. +if(HIPDNN_ENABLE_TOOLING_TESTS AND HIPDNN_SKIP_TESTS) + message(STATUS "hipDNN: HIPDNN_ENABLE_TOOLING_TESTS is ignored because HIPDNN_SKIP_TESTS is ON") +endif() +if(HIPDNN_ENABLE_TOOLING_TESTS AND NOT HIPDNN_SKIP_TESTS) + if(NOT Python3_FOUND) + message(FATAL_ERROR "hipDNN: HIPDNN_ENABLE_TOOLING_TESTS=ON needs a Python3 interpreter") + endif() + if(NOT HIPDNN_ENABLE_KERNEL_INGESTOR) + message( + FATAL_ERROR + "hipDNN: HIPDNN_ENABLE_TOOLING_TESTS=ON also needs HIPDNN_ENABLE_KERNEL_INGESTOR=ON: " + "the IngestorGenerator suite runs the hipdnn_validate_descriptors tool that option " + "builds. Enable it, or configure with -DHIPDNN_ENABLE_TOOLING_TESTS=OFF." + ) + endif() + execute_process( + COMMAND "${Python3_EXECUTABLE}" -c "import pytest, yaml, jinja2" + RESULT_VARIABLE _hipdnn_tooling_imports + OUTPUT_QUIET ERROR_QUIET + ) + if(NOT _hipdnn_tooling_imports EQUAL 0) + message( + FATAL_ERROR + "hipDNN: pytest, PyYAML and Jinja2 are not all importable by " + "${Python3_EXECUTABLE}, so the tooling tests cannot run. Install " + "tools/IngestorGenerator/requirements.txt for that interpreter, or configure " + "with -DHIPDNN_ENABLE_TOOLING_TESTS=OFF." + ) + endif() + set(HIPDNN_STRICT_SKIPS_PLUGIN_DIR "${CMAKE_CURRENT_SOURCE_DIR}/pytest_plugins") + add_test( + NAME hipdnn_descriptor_generator_pytest + COMMAND "${Python3_EXECUTABLE}" -m pytest -p hipdnn_strict_skips + "${CMAKE_CURRENT_SOURCE_DIR}/DescriptorGenerator/tests" -v + ) + set_tests_properties( + hipdnn_descriptor_generator_pytest + PROPERTIES ENVIRONMENT_MODIFICATION + "PYTHONPATH=path_list_prepend:${HIPDNN_STRICT_SKIPS_PLUGIN_DIR}" + ) + _apply_hipdnn_test_category_labels(hipdnn_descriptor_generator_pytest) +endif() + if(HIPDNN_ENABLE_KERNEL_INGESTOR) # ValidateDescriptors.cpp transitively includes through # DescriptorLoader.hpp, so it needs the same HIP wiring every ingestor test target @@ -353,5 +399,94 @@ endforeach() endif() _apply_hipdnn_test_category_labels(${HIPDNN_VALIDATE_DESCRIPTORS_TESTS}) + + # IngestorGenerator pytest suite. It is registered here, after the validator target: + # tests/test_round_trip.py runs that binary (HIPDNN_VALIDATE_DESCRIPTORS) and + # tests/test_native_stub.py needs the include list written below. + if(HIPDNN_ENABLE_TOOLING_TESTS) + # TestAgainstTheRealBuild gates a build packed with the gfx950 attention engine; the + # lanes pack other descriptors, so it would fail there. + add_test( + NAME hipdnn_ingestor_generator_pytest + COMMAND + "${Python3_EXECUTABLE}" -m pytest -p hipdnn_strict_skips + "${CMAKE_CURRENT_SOURCE_DIR}/IngestorGenerator/tests" -v + --deselect=tests/test_coverage_gate.py::TestAgainstTheRealBuild + ) + + set(_hipdnn_ingestor_pytest_env + "HIPDNN_VALIDATE_DESCRIPTORS=$" + ) + + # Include directories for test_native_stub.py, one per line. TARGET_PROPERTY does not + # follow ALIAS targets, so resolve to the real target. + set(_hipdnn_stub_include_exprs "") + foreach(_stub_dep flatbuffers::flatbuffers hip::host hip::amdhip64) + if(TARGET ${_stub_dep}) + get_target_property(_stub_real ${_stub_dep} ALIASED_TARGET) + if(NOT _stub_real) + set(_stub_real ${_stub_dep}) + endif() + string( + APPEND + _hipdnn_stub_include_exprs + "$,\n>\n" + ) + endif() + endforeach() + file( + GENERATE + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/ingestor_stub_include_dirs.txt" + CONTENT "${_hipdnn_stub_include_exprs}" + ) + list( + APPEND + _hipdnn_ingestor_pytest_env + "HIPDNN_INGESTOR_STUB_INCLUDE_FILE=${CMAKE_CURRENT_BINARY_DIR}/ingestor_stub_include_dirs.txt" + ) + + # test_packs_formatting.py takes the formatter from HIPDNN_CLANG_FORMAT, else PATH. + # The lanes install clang-format into the interpreter's own environment, whose + # scripts directory is not on PATH, so name it here. + get_filename_component(_hipdnn_py_dir "${Python3_EXECUTABLE}" DIRECTORY) + find_program( + HIPDNN_INGESTOR_CLANG_FORMAT + NAMES clang-format + HINTS "${_hipdnn_py_dir}" + NO_DEFAULT_PATH + ) + if(HIPDNN_INGESTOR_CLANG_FORMAT) + list(APPEND _hipdnn_ingestor_pytest_env + "HIPDNN_CLANG_FORMAT=${HIPDNN_INGESTOR_CLANG_FORMAT}" + ) + endif() + + if(DEFINED TEST_ENVIRONMENT) + list(APPEND _hipdnn_ingestor_pytest_env ${TEST_ENVIRONMENT}) + endif() + set_tests_properties( + hipdnn_ingestor_generator_pytest PROPERTIES ENVIRONMENT + "${_hipdnn_ingestor_pytest_env}" + ) + + # PYTHONPATH makes the plugin named by -p importable. + set(_hipdnn_ingestor_pytest_env_mod + "PYTHONPATH=path_list_prepend:${HIPDNN_STRICT_SKIPS_PLUGIN_DIR}" + ) + if(DEFINED TEST_ENVIRONMENT_MODIFICATION) + list(APPEND _hipdnn_ingestor_pytest_env_mod ${TEST_ENVIRONMENT_MODIFICATION}) + endif() + # Give test_native_stub.py the compiler that builds this tree; its lib/llvm/bin is not + # on the lanes' PATH. + get_filename_component(_hipdnn_stub_cxx_dir "${CMAKE_CXX_COMPILER}" DIRECTORY) + list(APPEND _hipdnn_ingestor_pytest_env_mod + "PATH=path_list_prepend:${_hipdnn_stub_cxx_dir}") + set_tests_properties( + hipdnn_ingestor_generator_pytest PROPERTIES ENVIRONMENT_MODIFICATION + "${_hipdnn_ingestor_pytest_env_mod}" + ) + + _apply_hipdnn_test_category_labels(hipdnn_ingestor_generator_pytest) + endif() endif() endif() diff --git a/projects/hipdnn/tools/DescriptorGenerator/codegen/generator.py b/projects/hipdnn/tools/DescriptorGenerator/codegen/generator.py index d1861511cd25..c934223a5c29 100644 --- a/projects/hipdnn/tools/DescriptorGenerator/codegen/generator.py +++ b/projects/hipdnn/tools/DescriptorGenerator/codegen/generator.py @@ -231,7 +231,7 @@ def render_backend(self, config: OperationConfig, output_dir: Path) -> list[str] out_path.parent.mkdir(parents=True, exist_ok=True) content = self._render_template(template_name, config) out_path.write_text(content) - written.append(str(rel_path)) + written.append(rel_path.as_posix()) fragments_dir = output_dir / "fragments" fragments_dir.mkdir(parents=True, exist_ok=True) @@ -277,7 +277,7 @@ def render_frontend(self, config: OperationConfig, output_dir: Path) -> list[str out_path.parent.mkdir(parents=True, exist_ok=True) content = self._render_template(template_name, config) out_path.write_text(content) - written.append(str(rel_path)) + written.append(rel_path.as_posix()) test_templates = { "test_attributes.cpp.j2": Path("frontend/tests") @@ -295,7 +295,7 @@ def render_frontend(self, config: OperationConfig, output_dir: Path) -> list[str out_path.parent.mkdir(parents=True, exist_ok=True) content = self._render_template(template_name, config) out_path.write_text(content) - written.append(str(rel_path)) + written.append(rel_path.as_posix()) # Frontend fragment templates fragment_templates = { @@ -375,7 +375,7 @@ def _render_constants(self, config: OperationConfig, output_dir: Path) -> list[s out_path.parent.mkdir(parents=True, exist_ok=True) content = self._render_template("constants.hpp.j2", config) out_path.write_text(content) - return [str(rel_path)] + return [rel_path.as_posix()] def _render_template(self, template_name: str, config: OperationConfig) -> str: try: diff --git a/projects/hipdnn/tools/DescriptorGenerator/tests/test_generate.py b/projects/hipdnn/tools/DescriptorGenerator/tests/test_generate.py index 6ef65225cc96..411a69a18b81 100644 --- a/projects/hipdnn/tools/DescriptorGenerator/tests/test_generate.py +++ b/projects/hipdnn/tools/DescriptorGenerator/tests/test_generate.py @@ -8,6 +8,7 @@ """ import subprocess +import sys from pathlib import Path import pytest @@ -328,11 +329,8 @@ def generate_script(self): @pytest.fixture def python_exe(self): - """Path to the .venv Python executable.""" - exe = Path(__file__).parent.parent / ".venv" / "bin" / "python" - if not exe.exists(): - pytest.skip("No .venv/bin/python found; CLI tests require a local venv") - return exe + """The interpreter running pytest; it already has PyYAML/Jinja2.""" + return Path(sys.executable) def test_missing_config_file_exits(self, generate_script, python_exe, tmp_path): """--config pointing to a nonexistent file exits with code 1.""" diff --git a/projects/hipdnn/tools/IngestorGenerator/README.md b/projects/hipdnn/tools/IngestorGenerator/README.md index a1ea7df073f0..0b7d87527533 100644 --- a/projects/hipdnn/tools/IngestorGenerator/README.md +++ b/projects/hipdnn/tools/IngestorGenerator/README.md @@ -792,7 +792,19 @@ matcher-test stubs with `g++`/`clang++`, plus an intentionally broken source con plugin/data/flatbuffers SDK and provider sources beside this checkout plus `flatbuffers/array.h` under `/opt/rocm/include`, and generates stand-ins for the CMake-configured `version.h`/`CacheRootDefaults.h` headers. Missing prerequisites skip -explicitly. +explicitly, so a checkout with no compiler or ROCm headers still runs green. Setting +`HIPDNN_INGESTOR_STUB_INCLUDE_FILE` to a file of include directories (one per line, +containing `flatbuffers/array.h` and `hip/hip_runtime.h`) replaces the +`/opt/rocm/include` probe, and a bad value then fails rather than skips. + +Under `HIPDNN_ENABLE_TOOLING_TESTS` the hipDNN build runs this suite with the +`tools/pytest_plugins/hipdnn_strict_skips.py` plugin (`-p hipdnn_strict_skips`), which turns +every skip into a failure, so a missing compiler, validator binary, `hkp_pack` or +`rocm_kpack` fails CI instead of passing by not running. A skip that is correct by design +(an author-supplied input such as `HIPDNN_INGESTOR_SHAPES` or an authoring profile, or the +POSIX-only sweep tests on Windows) carries `@pytest.mark.allow_skip(reason="...")` beside the +skip. Run by hand, the plugin is not loaded: `python -m pytest -p hipdnn_strict_skips` (with +`tools/pytest_plugins` on `PYTHONPATH`) reproduces the CI behavior. ### Fragment contracts (`tests/test_fragment_contracts.py`) diff --git a/projects/hipdnn/tools/IngestorGenerator/pyproject.toml b/projects/hipdnn/tools/IngestorGenerator/pyproject.toml index 4964e97e02b5..2c6234b931d1 100644 --- a/projects/hipdnn/tools/IngestorGenerator/pyproject.toml +++ b/projects/hipdnn/tools/IngestorGenerator/pyproject.toml @@ -6,6 +6,7 @@ python_functions = "test_*" markers = [ "round_trip: exercises hipdnn_validate_descriptors; skipped unless HIPDNN_VALIDATE_DESCRIPTORS is set.", "needs_rocm_kpack: builds a real rocm-kpack archive; skipped unless rocm_kpack is importable or HIPKERNELPROVIDER_ROCM_KPACK_DIR is set.", + "allow_skip(reason): may skip when the hipDNN build runs the suite with tools/pytest_plugins/hipdnn_strict_skips.py, which fails every other skip.", ] [tool.coverage.run] diff --git a/projects/hipdnn/tools/IngestorGenerator/templates/native.cpp.j2 b/projects/hipdnn/tools/IngestorGenerator/templates/native.cpp.j2 index 2ea82e8b51be..e26ed77b8af1 100644 --- a/projects/hipdnn/tools/IngestorGenerator/templates/native.cpp.j2 +++ b/projects/hipdnn/tools/IngestorGenerator/templates/native.cpp.j2 @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include "core/Handle.hpp" diff --git a/projects/hipdnn/tools/IngestorGenerator/templates/test_packs.cpp.j2 b/projects/hipdnn/tools/IngestorGenerator/templates/test_packs.cpp.j2 index 238a7f353f8a..ccb47e9ce3c9 100644 --- a/projects/hipdnn/tools/IngestorGenerator/templates/test_packs.cpp.j2 +++ b/projects/hipdnn/tools/IngestorGenerator/templates/test_packs.cpp.j2 @@ -18,7 +18,7 @@ #include #include #include -#include +#include {% if config.engine.census_name_symbol %} #include "engines/kernel_ingestor_engine/IngestorPacks.hpp" diff --git a/projects/hipdnn/tools/IngestorGenerator/tests/test_coverage_gate.py b/projects/hipdnn/tools/IngestorGenerator/tests/test_coverage_gate.py index b237e6f62f4e..de975003e34a 100644 --- a/projects/hipdnn/tools/IngestorGenerator/tests/test_coverage_gate.py +++ b/projects/hipdnn/tools/IngestorGenerator/tests/test_coverage_gate.py @@ -373,6 +373,13 @@ def test_serves_is_always_reported_as_owed_never_inferred(self, tmp_path): "reports another engine's work as this engine's" ) + @pytest.mark.skipif( + os.name != "posix", + reason="the stand-in validator is a shebang script, which Windows cannot execute", + ) + @pytest.mark.allow_skip( + reason="the stand-in validator is a shebang script, which Windows cannot execute" + ) def test_a_serving_threshold_this_tool_cannot_check_fails_the_gate(self, tmp_path): """Rungs 1 and 2 green, rung 3 not run: asking for served graphs must not pass on that evidence.""" diff --git a/projects/hipdnn/tools/IngestorGenerator/tests/test_inventory.py b/projects/hipdnn/tools/IngestorGenerator/tests/test_inventory.py index 000a5190d9c3..7bf3e6ec3e3f 100644 --- a/projects/hipdnn/tools/IngestorGenerator/tests/test_inventory.py +++ b/projects/hipdnn/tools/IngestorGenerator/tests/test_inventory.py @@ -13,7 +13,6 @@ import json import os -import shutil import subprocess import sys from pathlib import Path @@ -106,11 +105,12 @@ def test_a_kdp_without_kernel_descriptors_fails_instead_of_counting_zero(tmp_pat @pytest.mark.skipif( - os.name == "nt" - or shutil.which("bash") is None - or shutil.which("sha256sum") is None, + os.name == "nt", reason="the RUNBOOK recipe needs bash with GNU find/sort/sha256sum", ) +@pytest.mark.allow_skip( + reason="the RUNBOOK recipe needs bash with GNU find/sort/sha256sum, which Windows lacks" +) def test_catalog_digest_matches_the_runbook_recipe(tmp_path): bundle = _tree(tmp_path) recipe = ( diff --git a/projects/hipdnn/tools/IngestorGenerator/tests/test_native_stub.py b/projects/hipdnn/tools/IngestorGenerator/tests/test_native_stub.py index f2bbaf37ce76..9a67223dcb8c 100644 --- a/projects/hipdnn/tools/IngestorGenerator/tests/test_native_stub.py +++ b/projects/hipdnn/tools/IngestorGenerator/tests/test_native_stub.py @@ -5,6 +5,7 @@ agent or a human fills in to make an engine serve real graphs (see RUNBOOK.md).""" import json +import os import re import shutil import subprocess @@ -47,15 +48,38 @@ def compile_env(): "this checkout -- cannot attempt a real compile" ) - # Vendored flatbuffers headers: prefer an installed ROCm's, since this - # repo does not vendor flatbuffers itself. - fb_vendor = None - for candidate in (Path("/opt/rocm/include"),): - if (candidate / "flatbuffers" / "array.h").is_file(): - fb_vendor = candidate - break - if fb_vendor is None: - pytest.skip("no flatbuffers/array.h found (checked /opt/rocm/include)") + # HIPDNN_INGESTOR_STUB_INCLUDE_FILE (written by tools/CMakeLists.txt) lists the flatbuffers + # and HIP include dirs, one per line; a bad value fails rather than skips. Unset, probe an + # installed ROCm and skip when it is absent. + raw = os.environ.get("HIPDNN_INGESTOR_STUB_INCLUDE_FILE") + if raw is not None: + include_file = Path(raw) + if not include_file.is_file(): + pytest.fail( + f"HIPDNN_INGESTOR_STUB_INCLUDE_FILE is set to {raw!r}, " + "which is not an existing file" + ) + stub_includes = [ + Path(line.strip()) + for line in include_file.read_text().splitlines() + if line.strip() + ] + for header in ("flatbuffers/array.h", "hip/hip_runtime.h"): + if not any((d / header).is_file() for d in stub_includes): + pytest.fail( + f"HIPDNN_INGESTOR_STUB_INCLUDE_FILE {raw!r} lists no include " + f"directory containing {header}" + ) + else: + # Vendored flatbuffers headers: prefer an installed ROCm's, since this + # repo does not vendor flatbuffers itself. + stub_includes = [] + for candidate in (Path("/opt/rocm/include"),): + if (candidate / "flatbuffers" / "array.h").is_file(): + stub_includes.append(candidate) + break + if not stub_includes: + pytest.skip("no flatbuffers/array.h found (checked /opt/rocm/include)") gen_dir = Path(tempfile.mkdtemp(prefix="ingestor_gen_include_")) configure_targets = { @@ -164,7 +188,7 @@ def configure(text: str, subs: dict, out: Path) -> None: data_sdk, flatbuffers_sdk, provider_src, - fb_vendor, + *stub_includes, ], } diff --git a/projects/hipdnn/tools/IngestorGenerator/tests/test_strict_skips_plugin.py b/projects/hipdnn/tools/IngestorGenerator/tests/test_strict_skips_plugin.py new file mode 100644 index 000000000000..1366081e5dcb --- /dev/null +++ b/projects/hipdnn/tools/IngestorGenerator/tests/test_strict_skips_plugin.py @@ -0,0 +1,115 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""The strict-skips plugin is what keeps CI from passing by not running a test. Each case +runs a throwaway suite in a child pytest, once without the plugin and once with it.""" + +import os +import re +import subprocess +import sys +from pathlib import Path + +PLUGIN_DIR = Path(__file__).resolve().parents[2] / "pytest_plugins" + +SUITE = """ +import pytest + +def test_passes(): + pass + +def test_skips_for_a_missing_prerequisite(): + pytest.skip("no compiler") + +@pytest.mark.skipif(True, reason="no such platform") +def test_skipif_for_a_missing_prerequisite(): + pass + +@pytest.fixture +def fixture_that_skips(): + pytest.skip("no binary") + +def test_skip_raised_in_a_fixture(fixture_that_skips): + pass + +@pytest.mark.xfail(reason="known bug") +def test_xfail(): + assert False + +@pytest.mark.allow_skip(reason="author supplies the input") +def test_allowed_skip(): + pytest.skip("no input") + +@pytest.mark.allow_skip(reason="no such platform") +class TestAllowedClass: + @pytest.mark.skipif(True, reason="no such platform") + def test_member(self): + pass + +@pytest.mark.allow_skip +def test_allow_skip_without_a_reason(): + pytest.skip("no input") +""" + + +def _run(tmp_path, *extra): + (tmp_path / "test_suite.py").write_text(SUITE) + env = dict(os.environ, PYTHONPATH=str(PLUGIN_DIR)) + result = subprocess.run( + [sys.executable, "-m", "pytest", "-v", "-p", "no:cacheprovider", *extra], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + return { + match.group(1): match.group(2) + for match in re.finditer( + r"^test_suite\.py::(\S+) (PASSED|SKIPPED|FAILED|ERROR|XFAIL)", + result.stdout, + flags=re.MULTILINE, + ) + }, result.stdout + + +def test_without_the_plugin_every_skip_stays_a_skip(tmp_path): + outcomes, out = _run(tmp_path) + assert "FAILED" not in outcomes.values() and "ERROR" not in outcomes.values(), out + assert outcomes["test_skips_for_a_missing_prerequisite"] == "SKIPPED" + + +def test_with_the_plugin_a_skip_fails_unless_marked_allowed(tmp_path): + outcomes, out = _run(tmp_path, "-p", "hipdnn_strict_skips") + assert outcomes == { + "test_passes": "PASSED", + "test_skips_for_a_missing_prerequisite": "FAILED", + "test_skipif_for_a_missing_prerequisite": "ERROR", + "test_skip_raised_in_a_fixture": "ERROR", + "test_xfail": "FAILED", + "test_allowed_skip": "SKIPPED", + "TestAllowedClass::test_member": "SKIPPED", + "test_allow_skip_without_a_reason": "FAILED", + }, out + assert "skipped in a strict run: no compiler" in out + + +def test_a_module_level_skip_fails_the_collection_under_the_plugin(tmp_path): + (tmp_path / "test_ok.py").write_text("def test_ok():\n pass\n") + (tmp_path / "test_modskip.py").write_text( + "import pytest\npytest.importorskip('no_such_module_xyz')\n" + ) + env = dict(os.environ, PYTHONPATH=str(PLUGIN_DIR)) + + def run(*extra): + return subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", *extra], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert run().returncode == 0 + strict = run("-p", "hipdnn_strict_skips") + assert strict.returncode != 0, strict.stdout + assert "skipped while collecting test_modskip.py in a strict run" in strict.stdout diff --git a/projects/hipdnn/tools/IngestorGenerator/tests/test_sweep_tools.py b/projects/hipdnn/tools/IngestorGenerator/tests/test_sweep_tools.py index 9826c10b404b..4203aa3bd3ac 100644 --- a/projects/hipdnn/tools/IngestorGenerator/tests/test_sweep_tools.py +++ b/projects/hipdnn/tools/IngestorGenerator/tests/test_sweep_tools.py @@ -201,24 +201,29 @@ def value(flag): #: Marks a case that drives a sweep far enough to execute a staged fixture. Narrower #: than the file: staging a fixture and refusing a config work anywhere, so the #: refusal, device-probe and field-audit cases stay live on every platform. -_needs_posix_exec = pytest.mark.skipif( - os.name != "posix", - reason=( - "driving a sweep to a result needs POSIX execution semantics: sweep.py " - "selects the benchmark interpreter from its shebang and runs the benchmark " - "directly. sweep.py targets an allocated device host." - ), +def _skip_off_posix(reason: str): + """Skip off POSIX, and stay allowed to skip in a strict run: the platform cannot run + the case, which no build can fix.""" + + def mark(obj): + obj = pytest.mark.skipif(os.name != "posix", reason=reason)(obj) + return pytest.mark.allow_skip(reason=reason)(obj) + + return mark + + +_needs_posix_exec = _skip_off_posix( + "driving a sweep to a result needs POSIX execution semantics: sweep.py " + "selects the benchmark interpreter from its shebang and runs the benchmark " + "directly. sweep.py targets an allocated device host." ) #: Marks a case that withdraws write access to drive an operational error. `chmod` #: does not deny directory writes on Windows, so the condition never holds there. -_needs_posix_permissions = pytest.mark.skipif( - os.name != "posix", - reason=( - "making a directory unwritable needs POSIX permission semantics: chmod does " - "not withdraw directory write access on Windows" - ), +_needs_posix_permissions = _skip_off_posix( + "making a directory unwritable needs POSIX permission semantics: chmod does " + "not withdraw directory write access on Windows" ) diff --git a/projects/hipdnn/tools/ai/skills/hipdnn-superbuild-test/SKILL.md b/projects/hipdnn/tools/ai/skills/hipdnn-superbuild-test/SKILL.md index 6e98c77f1c54..bcfe40ee8090 100644 --- a/projects/hipdnn/tools/ai/skills/hipdnn-superbuild-test/SKILL.md +++ b/projects/hipdnn/tools/ai/skills/hipdnn-superbuild-test/SKILL.md @@ -131,14 +131,11 @@ architecture explicitly named under the production descriptor root that `GPU_TAR does not include (automatic; none when the build targets them all), `hkp-probe-`, one per named `hkp_add_packaging_probe()` line in `probes.cmake`, and `hkp-probe-tools`. A full ctest run includes them; to run them alone, select with the anchored `-R '^hkp-probe-'` and `--no-tests=error`. -`/hkp-probes/manifest.txt` lists the entries that must run. Check a ctest junit -(`--output-junit`) against it with `.github/scripts/check_probe_junit.py --junit ---manifest `, which fails on any listed entry missing, not run, failed or -errored and ignores the other testcases. - -In the superbuild CI lanes the probes run once, inside the "Run tests" ctest, which writes -`build/ctest-junit.xml`; the "Check packaging probes ran" step then runs only the junit -check, whenever "Run tests" ran, pass or fail. A pass does not cover device behavior, +`-DHIPKERNELPROVIDER_MIN_PACKAGING_PROBES=N` fails configure when fewer than N probes are +declared (default 0). + +In the superbuild CI lanes the probes run once, inside the "Run tests" ctest, with N=1. +A pass does not cover device behavior, install staging, UKDs the probe does not pack, or standalone-UKD references. ## Ingestor proof boundaries diff --git a/projects/hipdnn/tools/pytest_plugins/hipdnn_strict_skips.py b/projects/hipdnn/tools/pytest_plugins/hipdnn_strict_skips.py new file mode 100644 index 000000000000..93c1b0726efd --- /dev/null +++ b/projects/hipdnn/tools/pytest_plugins/hipdnn_strict_skips.py @@ -0,0 +1,82 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""pytest plugin that turns a skipped test into a failure, so a build that should run a suite +cannot pass by silently not running part of it. + +ctest reports a pytest run as passed when every case skips, and a test that needs a compiler, +a built binary or an installed package skips without a word when it is missing. The hipDNN +build runs the DescriptorGenerator and IngestorGenerator suites with this plugin +(``-p hipdnn_strict_skips``, see tools/CMakeLists.txt). Run by hand it is not loaded, so those +tests skip as usual. + +A skip that is correct by design (an author-supplied input, a platform that cannot run the +test) opts out with ``@pytest.mark.allow_skip(reason="...")`` on the test or its class; the +reason is required. A skip raised while a module is imported (``pytest.importorskip(...)``, +``pytest.skip(allow_module_level=True)``) has no test to carry the marker, so it fails the +collection. +""" + +import pytest + +MARKER = "allow_skip" + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + f"{MARKER}(reason): this test may skip under hipdnn_strict_skips, for the stated " + "reason (an author-supplied input, or a platform that cannot run it).", + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + # xfail also reports as skipped, with `wasxfail` set; a strict run has no use for it. + if not report.skipped: + return + + if isinstance(report.longrepr, tuple): + reason = report.longrepr[2].removeprefix("Skipped: ") + else: + reason = getattr(report, "wasxfail", None) or str(report.longrepr) + + marker = item.get_closest_marker(MARKER) + if marker is not None: + if marker.kwargs.get("reason"): + return + report.outcome = "failed" + report.longrepr = ( + f"@pytest.mark.{MARKER} needs a reason: " + f'@pytest.mark.{MARKER}(reason="...") (the test skipped: {reason})' + ) + return + + report.outcome = "failed" + report.longrepr = ( + f"skipped in a strict run: {reason}\n" + "Provide the missing prerequisite. If the skip is by design, mark the test " + f'@pytest.mark.{MARKER}(reason="...").' + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_make_collect_report(collector): + outcome = yield + report = outcome.get_result() + if not report.skipped: + return + + reason = ( + report.longrepr[2].removeprefix("Skipped: ") + if isinstance(report.longrepr, tuple) + else str(report.longrepr) + ) + report.outcome = "failed" + report.longrepr = ( + f"skipped while collecting {collector.nodeid} in a strict run: {reason}\n" + "Provide the missing prerequisite. A module-level skip cannot be marked " + f"@pytest.mark.{MARKER}; skip inside the tests that need it instead." + )