From e5fa4fd8bc093d29e6d860e8ca886e91ca37ed2c Mon Sep 17 00:00:00 2001 From: Adam Dickin Date: Thu, 29 Oct 2026 08:26:03 -0600 Subject: [PATCH 1/4] test(integration-tests): exclude quick-tier SDPA GpuRef golden-data cases from ffm-quick The GPU SDPA reference recomputes QK^T per output element. On the FFM simulator the smallest quick SdpaFwd bundle took 329 s and the 2x32x2048x128 bundle had not finished after 40 minutes. referenceShapeIsAffordable() gates only the CPU reference, assuming the GPU reference is fast; that does not hold on FFM. Real-GPU lanes still run these cases and the CpuRef lane still validates the same bundles. --- dnn-providers/integration-tests/test_categories.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dnn-providers/integration-tests/test_categories.yaml b/dnn-providers/integration-tests/test_categories.yaml index d104f18681c8..c8a8a0d495b7 100644 --- a/dnn-providers/integration-tests/test_categories.yaml +++ b/dnn-providers/integration-tests/test_categories.yaml @@ -86,7 +86,7 @@ test_categories: - "slow" ffm-quick: - description: "Fast-feedback mechanism (FFM) quick tier - currently mirrors the quick tier" + description: "Fast-feedback mechanism (FFM) quick tier - mirrors the quick tier minus the quick-tier SDPA golden-data GPU-reference cases" test_patterns: - "*" exclude: @@ -97,6 +97,15 @@ test_categories: - "comprehensive_*" - "Full*" - "full_*" + # The GPU SDPA reference recomputes QK^T per output element, so cost tracks + # B*H*Sq*Skv*D. On the FFM simulator even the smallest quick-tier bundle + # (1x8x256x128, 36 ms on MI300X) took 329 s, hd192 took 484 s, and the + # 2x32x2048x128 bundle had not finished after 40 minutes. + # referenceShapeIsAffordable() gates only the CPU reference, on the + # assumption that the GPU reference runs these shapes in milliseconds; + # that holds on hardware, not on FFM. Real-GPU lanes still run these + # cases and the CpuRef lane still validates the same bundles on the host. + - "quick_SdpaFwd_*_GpuRef.*" labels: - "ffm-quick" - "ffm-full" From 254e13347a8eafb250a7546e9e15dcac008ff0f3 Mon Sep 17 00:00:00 2001 From: Adam Dickin Date: Thu, 29 Oct 2026 08:26:13 -0600 Subject: [PATCH 2/4] test(hipdnn): exclude the stall-gate test from ffm-quick via a gtest-filter suite StallGateExcludesHostSubmissionDelay asserts that a stalled start event leaves a 20 ms host delay out of the measured span. The FFM simulator does not honour the stall gate (112 ms stalled vs < 5 ms expected), so the test cannot pass there. test_categories.yaml labels whole CTest entries by name and hipdnn_backend_tests is one entry, so it could not drop one gtest. add_hipdnn_test() takes a GTEST_CATEGORIES flag that also generates __suite entries from the new test_categories_gtest.yaml, in the build tree and in the installed CTestTestfile.cmake. hipdnn_backend_tests opts in for ffm-quick. The plain entry must not keep the ffm-quick label, or it still runs the excluded test. parse_ctest_categories.py gains a per-category exclude_tests key: matching tests receive none of that category's labels. ctest -L is a regex search, so hipdnn-quick-check (-L quick) also selected every ffm-quick test and would run hipdnn_backend_tests twice. The check targets now anchor the label, with VERBATIM so the anchors survive cmd.exe, where ^ is the escape character. --- projects/hipdnn/backend/tests/CMakeLists.txt | 2 +- projects/hipdnn/cmake/Tests.cmake | 71 ++++++++- projects/hipdnn/docs/TESTING.md | 2 + projects/hipdnn/test_categories.yaml | 7 +- projects/hipdnn/test_categories_gtest.yaml | 38 +++++ shared/ctest/parse_ctest_categories.py | 72 +++++++-- shared/ctest/tests/CMakeLists.txt | 8 + .../tests/test_parse_ctest_categories.py | 147 ++++++++++++++++++ 8 files changed, 331 insertions(+), 16 deletions(-) create mode 100644 projects/hipdnn/test_categories_gtest.yaml create mode 100644 shared/ctest/tests/test_parse_ctest_categories.py diff --git a/projects/hipdnn/backend/tests/CMakeLists.txt b/projects/hipdnn/backend/tests/CMakeLists.txt index aec6fe686135..1ee8888b12fb 100644 --- a/projects/hipdnn/backend/tests/CMakeLists.txt +++ b/projects/hipdnn/backend/tests/CMakeLists.txt @@ -322,4 +322,4 @@ add_dependencies(hipdnn_backend_tests ${HIPDNN_TEST_MALFORMED_VERSION_PLUGIN_TARGET} ) -add_hipdnn_test(hipdnn_backend_tests ${CMAKE_CURRENT_BINARY_DIR}) +add_hipdnn_test(hipdnn_backend_tests ${CMAKE_CURRENT_BINARY_DIR} GTEST_CATEGORIES) diff --git a/projects/hipdnn/cmake/Tests.cmake b/projects/hipdnn/cmake/Tests.cmake index 0e5a662ea660..d036cc141c8b 100644 --- a/projects/hipdnn/cmake/Tests.cmake +++ b/projects/hipdnn/cmake/Tests.cmake @@ -25,8 +25,11 @@ findandcheckllvmsymbolizer() # YAML-driven CTest categorisation. Build-tree tests get labels as they are # registered; install_hipdnn_ctest_files() bakes the same labels into the -# installed CTestTestfile.cmake. +# installed CTestTestfile.cmake. test_categories.yaml labels whole CTest entries by +# name. A target that must run a gtest-filtered subset (add_hipdnn_test(... +# GTEST_CATEGORIES)) also gets suites generated from test_categories_gtest.yaml. set(_HIPDNN_TEST_CATEGORIES_YAML "${PROJECT_SOURCE_DIR}/test_categories.yaml") +set(_HIPDNN_GTEST_CATEGORIES_YAML "${PROJECT_SOURCE_DIR}/test_categories_gtest.yaml") set(_HIPDNN_SHARED_CTEST "${ROCM_LIBRARIES_ROOT}/shared/ctest/TestCategories.cmake") if(EXISTS "${_HIPDNN_SHARED_CTEST}" AND EXISTS "${_HIPDNN_TEST_CATEGORIES_YAML}") include("${_HIPDNN_SHARED_CTEST}") @@ -176,9 +179,10 @@ function(_add_check_target_internal PREFIX_NAME TARGET_NAME LABEL VERBOSE COMMEN # Build the ctest command set(CTEST_CMD ${CMAKE_COMMAND} -E env ${CTEST_ENV} ${CMAKE_CTEST_COMMAND}) - # Add label filter if specified + # Add label filter if specified. Anchored: -L is a regex search, so a bare `quick` would + # also select every test labelled `ffm-quick`. if(NOT "${LABEL}" STREQUAL "") - list(APPEND CTEST_CMD -L "${LABEL}") + list(APPEND CTEST_CMD -L "^${LABEL}$") endif() # Always add --output-on-failure @@ -194,7 +198,10 @@ function(_add_check_target_internal PREFIX_NAME TARGET_NAME LABEL VERBOSE COMMEN # Create the target with prefix set(FULL_TARGET_NAME "${PREFIX_NAME}-${TARGET_NAME}") - add_custom_target(${FULL_TARGET_NAME} COMMAND ${CTEST_CMD} COMMENT "${COMMENT}" USES_TERMINAL) + # VERBATIM: the anchors in -L "^label$" are shell-special (cmd.exe treats ^ as its escape + # character), so the arguments must be quoted for the host shell. + add_custom_target(${FULL_TARGET_NAME} COMMAND ${CTEST_CMD} COMMENT "${COMMENT}" USES_TERMINAL + VERBATIM) add_dependencies(${FULL_TARGET_NAME} ${PREFIX_NAME}-validate_test_names) message(VERBOSE "Created ${FULL_TARGET_NAME} target") endfunction() # _add_check_target_internal @@ -292,6 +299,43 @@ function(finalize_test_targets prefix_name) # cmake-format: on endfunction() # finalize_test_targets +# Generates the __suite entries of a GTEST_CATEGORIES target from +# test_categories_gtest.yaml (see add_hipdnn_test). +function(_hipdnn_add_gtest_category_suites TARGET WORKING_DIR) + if(NOT COMMAND apply_test_category_labels OR NOT EXISTS "${_HIPDNN_GTEST_CATEGORIES_YAML}") + message(FATAL_ERROR + "add_hipdnn_test(${TARGET} GTEST_CATEGORIES) needs shared/ctest and " + "${_HIPDNN_GTEST_CATEGORIES_YAML}" + ) + endif() + + # Same environment and fixtures as the plain entry. The installed-tree lines that the parser + # writes into the fragment carry the same ENVIRONMENT, which only differs from the plain + # installed entry in sanitizer builds. + set(_category_args "") + if(DEFINED TEST_ENVIRONMENT) + list(APPEND _category_args ENVIRONMENT ${TEST_ENVIRONMENT}) + endif() + if(DEFINED TEST_ENVIRONMENT_MODIFICATION) + list(APPEND _category_args ENVIRONMENT_MODIFICATION ${TEST_ENVIRONMENT_MODIFICATION}) + endif() + if(DEFINED HIPDNN_TEST_MIOPEN_CACHE_DIR) + list(APPEND _category_args FIXTURES_REQUIRED hipdnn_clear_miopen_test_cache) + endif() + + # The parser appends the installed-tree add_test() lines to this fragment; + # install_hipdnn_ctest_files() copies it into the installed CTestTestfile.cmake. + set(_fragment "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_gtest_categories.ctest") + file(WRITE "${_fragment}" "") + set_property(GLOBAL APPEND PROPERTY HIPDNN_GTEST_CATEGORY_FRAGMENTS "${_fragment}") + + apply_test_category_labels( + ${TARGET} "${_HIPDNN_GTEST_CATEGORIES_YAML}" "${WORKING_DIR}" + INSTALL_TEST_FILE "${_fragment}" + ${_category_args} + ) +endfunction() # _hipdnn_add_gtest_category_suites + # ~~~ # Records, configures, and registers a hipDNN gtest-based CTest test target. Assumes that the # test target is a gtest executable, setting up: @@ -304,8 +348,14 @@ endfunction() # finalize_test_targets # Parameters: # TARGET - Name of the test executable target (must already exist) # WORKING_DIR - Working directory for test execution +# GTEST_CATEGORIES - Optional flag. Also generate __suite entries, each running +# the binary with the --gtest_filter that test_categories_gtest.yaml defines for that +# category, labelled as that file says. The target must then be listed under +# `exclude_tests` for the same categories in test_categories.yaml, or its plain entry would +# carry those labels too and run the tests the filter excludes. # ~~~ function(add_hipdnn_test TARGET WORKING_DIR) + cmake_parse_arguments(ARG "GTEST_CATEGORIES" "" "" ${ARGN}) # ROCm ships its own googletest, which hip::host puts on the include path. Ahead of the # fetched googletest it compiles against ROCm's headers but links the fetched archive. # Pinned per target, for both gtest and gmock, so library sources never see googletest. @@ -381,6 +431,10 @@ function(add_hipdnn_test TARGET WORKING_DIR) if(DEFINED HIPDNN_TEST_MIOPEN_CACHE_DIR) set_tests_properties(${TARGET} PROPERTIES FIXTURES_REQUIRED hipdnn_clear_miopen_test_cache) endif() + + if(ARG_GTEST_CATEGORIES) + _hipdnn_add_gtest_category_suites(${TARGET} "${WORKING_DIR}") + endif() endfunction() # add_hipdnn_test # Install CTest configuration files for direct test execution This should be called once at the end @@ -431,6 +485,15 @@ function(install_hipdnn_ctest_files) ) endif() + # Suites generated from test_categories_gtest.yaml. Appended after the label pass on + # purpose: that pass would label them through test_categories.yaml's `.*` patterns + # (quick, standard, ...), and their only labels are the ones the gtest yaml gives them. + get_property(gtest_category_fragments GLOBAL PROPERTY HIPDNN_GTEST_CATEGORY_FRAGMENTS) + foreach(_fragment IN LISTS gtest_category_fragments) + file(READ "${_fragment}" _fragment_contents) + file(APPEND "${INSTALLED_CTEST_FILE}" "\n${_fragment_contents}") + endforeach() + # Install the generated CTestTestfile.cmake to HIPDNN_CTEST_FILE_INSTALL_PATH install(FILES "${INSTALLED_CTEST_FILE}" DESTINATION ${HIPDNN_CTEST_FILE_INSTALL_PATH} RENAME CTestTestfile.cmake diff --git a/projects/hipdnn/docs/TESTING.md b/projects/hipdnn/docs/TESTING.md index d92d6b444158..4e3d272ed7a6 100644 --- a/projects/hipdnn/docs/TESTING.md +++ b/projects/hipdnn/docs/TESTING.md @@ -48,6 +48,8 @@ cmake --build build --target hipdnn-standard-check `hipdnn--check` and `hipdnn--check-verbose` exist for each category in [`test_categories.yaml`](../test_categories.yaml): the tiers `quick`, `standard`, `comprehensive` and `full`, plus `ffm-quick`, `unit` and `integration`. They build and run the hipDNN core tests only; `hipdnn-check` runs all core tests. Provider and integration-tests targets are listed in [Building hipDNN: Superbuild Targets](./Building.md#superbuild-targets). +`test_categories.yaml` labels whole CTest entries by name, so it cannot leave one gtest out of a binary. A target registered with `add_hipdnn_test( GTEST_CATEGORIES)` also gets `__suite` entries generated from [`test_categories_gtest.yaml`](../test_categories_gtest.yaml). Each runs the binary with that category's `--gtest_filter` and carries exactly the labels the file lists. Today only `hipdnn_backend_tests` opts in, and only for `ffm-quick`, where the file excludes tests the FFM simulator cannot pass. List such a target under `exclude_tests` for the same category in `test_categories.yaml`, so the plain entry does not carry the label and run the excluded tests. The `hipdnn--check` targets match their label exactly (`ctest -L '^$'`), so `hipdnn-quick-check` does not also run the `ffm-quick` suite. + **One provider.** Build the provider's preset (for example `miopen-provider`), then run the shared integration suite against it: ```bash diff --git a/projects/hipdnn/test_categories.yaml b/projects/hipdnn/test_categories.yaml index e4d25f391c25..9150692bb70d 100644 --- a/projects/hipdnn/test_categories.yaml +++ b/projects/hipdnn/test_categories.yaml @@ -28,9 +28,14 @@ test_categories: test_patterns: ffm-quick: - description: "Fast-feedback mechanism (FFM) quick tier; currently mirrors the quick tier" + description: "Fast-feedback mechanism (FFM) quick tier; mirrors the quick tier" test_patterns: - ".*" + # Targets whose ffm-quick suite is generated from test_categories_gtest.yaml (so + # it can exclude individual gtests) carry the ffm labels on that suite only. + # Labelling the plain entry too would run the excluded tests under -L ffm-quick. + exclude_tests: + - "hipdnn_backend_tests" labels: - "ffm-full" diff --git a/projects/hipdnn/test_categories_gtest.yaml b/projects/hipdnn/test_categories_gtest.yaml new file mode 100644 index 000000000000..1494ded9e8b9 --- /dev/null +++ b/projects/hipdnn/test_categories_gtest.yaml @@ -0,0 +1,38 @@ +# hipDNN gtest-filter test categories. +# +# test_categories.yaml labels whole CTest entries by name; it cannot drop a single +# gtest from a binary. This file is the gtest-schema counterpart (shared/ctest/ +# parse_test_categories.py): each category becomes a CTest suite named +# __suite that runs the binary with a --gtest_filter built from +# test_patterns/exclude. Labels are exactly the ones listed here; tiers do not +# cascade. +# +# It applies to every gtest target that opts in with +# add_hipdnn_test(... GTEST_CATEGORIES) (cmake/Tests.cmake), so one exclude list +# serves all of them. A pattern that matches no test in a binary is harmless, but +# qualify excludes as Suite.Test so they cannot match anything unintended. +# +# A target that opts in carries the labels of the categories below on these suites +# only, so test_categories.yaml must list it under `exclude_tests` for the same +# categories. Today that is ffm-quick. + +test_categories: + ffm-quick: + description: "Fast-feedback mechanism (FFM) quick tier - mirrors the quick tier minus tests that cannot pass on the simulator" + test_patterns: + - "*" + exclude: + - "*DISABLED*" + # Asserts that a stalled start event excludes a 20 ms host delay from the + # measured span. FFM does not honour the stall gate: stalled timing reported + # 100-112 ms against an expected < 5 ms. + - "TestGpuProfilingControlDescriptor.StallGateExcludesHostSubmissionDelay" + labels: + - "ffm-quick" + - "ffm-full" + +execution_settings: + default_timeout: 600 + timeout_multiplier: 1 + category_timeouts: + ffm-quick: 7200 # 2 hours; matches the integration-tests ffm-quick tier diff --git a/shared/ctest/parse_ctest_categories.py b/shared/ctest/parse_ctest_categories.py index 1c39c1ecd88f..83cc169cf95b 100644 --- a/shared/ctest/parse_ctest_categories.py +++ b/shared/ctest/parse_ctest_categories.py @@ -120,7 +120,40 @@ def tier_labels(category_name): return TIER_ORDER[idx:] -def _emit_label_block(lines, test_names, labels_str, comment, explicit_tests=None): +def _is_skipped(test_name, skip_patterns): + """True if test_name matches an entry of a category's ``exclude_tests``. + + Entries follow the rules of ``test_patterns``: a name with no regex + metacharacters is an exact match, anything else is a full-match regex. + """ + for pattern in skip_patterns: + if is_regex_pattern(pattern): + if re.fullmatch(pattern, test_name): + return True + elif pattern == test_name: + return True + return False + + +def _cmake_skip_condition(skip_patterns): + """Extra ``if()`` conjuncts that are false for tests named by skip_patterns. + + Refers to ``_test``, the loop variable of the surrounding label block. Regex + entries are evaluated by CMake, so they must stay within CMake's regex dialect + (no lookahead), like every other regex pattern in these files. + """ + condition = "" + for pattern in skip_patterns: + if is_regex_pattern(pattern): + condition += f' AND NOT _test MATCHES "^{pattern}$"' + else: + condition += f' AND NOT _test STREQUAL "{pattern}"' + return condition + + +def _emit_label_block( + lines, test_names, labels_str, comment, explicit_tests=None, skip=None +): """Append CMake code to apply labels to a list of test names/patterns. When explicit_tests is provided (a list of test names known at parse @@ -134,8 +167,13 @@ def _emit_label_block(lines, test_names, labels_str, comment, explicit_tests=Non for the build tree because CMake configure time materialises set_property(TEST ...) into explicit set_tests_properties lines in the autogenerated CTestTestfile.cmake files. + + Tests matching ``skip`` (a category's ``exclude_tests``) get no label. """ - exact = [t for t in test_names if not is_regex_pattern(t)] + skip = skip or [] + exact = [ + t for t in test_names if not is_regex_pattern(t) and not _is_skipped(t, skip) + ] regex = [t for t in test_names if is_regex_pattern(t)] if exact: @@ -160,7 +198,11 @@ def _emit_label_block(lines, test_names, labels_str, comment, explicit_tests=Non for pattern in regex: pat_re = re.compile(f"^{pattern}$") for t in explicit_tests: - if pat_re.match(t) and t not in matched: + if ( + pat_re.match(t) + and t not in matched + and not _is_skipped(t, skip) + ): matched.append(t) if matched: lines.append(f"# {comment} (regex match, expanded)") @@ -178,13 +220,14 @@ def _emit_label_block(lines, test_names, labels_str, comment, explicit_tests=Non else: # Build-tree path: rely on CMake configure-time directory # property + later materialisation to set_tests_properties. + skip_condition = _cmake_skip_condition(skip) lines.append(f"# {comment} (regex match)") lines.append( "get_property(_all_tests DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY TESTS)" ) lines.append("foreach(_test ${_all_tests})") for pattern in regex: - lines.append(f' if(_test MATCHES "^{pattern}$")') + lines.append(f' if(_test MATCHES "^{pattern}$"{skip_condition})') lines.append( f' set_property(TEST ${{_test}} APPEND PROPERTY LABELS "{labels_str}")' ) @@ -193,11 +236,12 @@ def _emit_label_block(lines, test_names, labels_str, comment, explicit_tests=Non lines.append("") -def _emit_label_match_block(lines, test_labels, labels_str, comment): +def _emit_label_match_block(lines, test_labels, labels_str, comment, skip=None): """Append CMake code to apply labels to tests that already carry any of - the given existing labels.""" + the given existing labels. Tests matching ``skip`` get no label.""" if not test_labels: return + skip_condition = _cmake_skip_condition(skip or []) lines.append(f"# {comment} (match by existing label)") lines.append( "get_property(_all_tests DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY TESTS)" @@ -205,7 +249,7 @@ def _emit_label_match_block(lines, test_labels, labels_str, comment): lines.append("foreach(_test ${_all_tests})") lines.append(" get_property(_test_labels TEST ${_test} PROPERTY LABELS)") for label in test_labels: - lines.append(f' if("{label}" IN_LIST _test_labels)') + lines.append(f' if("{label}" IN_LIST _test_labels{skip_condition})') lines.append( f' set_property(TEST ${{_test}} APPEND PROPERTY LABELS "{labels_str}")' ) @@ -245,15 +289,17 @@ def match_pattern_to_tests(pattern): test_label_matches = config.get("test_labels") or [] excludes = config.get("exclude") or [] custom_labels = config.get("labels") or [] + skip = config.get("exclude_tests") or [] all_labels = list(dict.fromkeys(tier_labels(category) + custom_labels)) for pattern in test_patterns: for t in match_pattern_to_tests(pattern): - append_labels(t, all_labels) + if not _is_skipped(t, skip): + append_labels(t, all_labels) for existing_label in test_label_matches: for t in test_names: - if existing_label in labels[t]: + if existing_label in labels[t] and not _is_skipped(t, skip): append_labels(t, all_labels) exclude_label = f"{category}_exclude" @@ -318,6 +364,7 @@ def generate_cmake(categories, explicit_tests=None): test_labels = config.get("test_labels") or [] excludes = config.get("exclude") or [] custom_labels = config.get("labels") or [] + skip = config.get("exclude_tests") or [] # Combine the tier labels and custom labels, and remove duplicates all_labels = list(dict.fromkeys(tier_labels(category) + custom_labels)) @@ -329,11 +376,16 @@ def generate_cmake(categories, explicit_tests=None): labels_str, f"Category: {category} - test_patterns", explicit_tests=explicit_tests, + skip=skip, ) if test_labels: _emit_label_match_block( - lines, test_labels, labels_str, f"Category: {category} - test_labels" + lines, + test_labels, + labels_str, + f"Category: {category} - test_labels", + skip=skip, ) if excludes: diff --git a/shared/ctest/tests/CMakeLists.txt b/shared/ctest/tests/CMakeLists.txt index 2b1f9f584fc1..61122c22b634 100644 --- a/shared/ctest/tests/CMakeLists.txt +++ b/shared/ctest/tests/CMakeLists.txt @@ -21,4 +21,12 @@ if(Python3_FOUND) -s "${CMAKE_CURRENT_LIST_DIR}" -p "test_parse_test_categories.py" ) + + add_test( + NAME shared_ctest_parse_ctest_categories_unit + COMMAND + ${Python3_EXECUTABLE} -m unittest discover -v + -s "${CMAKE_CURRENT_LIST_DIR}" + -p "test_parse_ctest_categories.py" + ) endif() diff --git a/shared/ctest/tests/test_parse_ctest_categories.py b/shared/ctest/tests/test_parse_ctest_categories.py new file mode 100644 index 000000000000..ebfab9dabad6 --- /dev/null +++ b/shared/ctest/tests/test_parse_ctest_categories.py @@ -0,0 +1,147 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +import json +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +_TESTS_DIR = Path(__file__).resolve().parent +_CTEST_DIR = _TESTS_DIR.parent + +sys.path.insert(0, str(_CTEST_DIR)) +import parse_ctest_categories as pcc # noqa: E402 + +# `.*` is the shape that needs exclude_tests: every test is a candidate and one +# entry must be left out. The other categories show that skipping is per category. +CATEGORIES = { + "quick": {"test_patterns": [".*"]}, + "ffm-quick": { + "test_patterns": [".*"], + "exclude_tests": ["plain_binary"], + "labels": ["ffm-full"], + }, + "unit": {"test_patterns": ["plain_binary", "other_binary"]}, +} + + +class TestExcludeTestsInstallLabels(unittest.TestCase): + def test_skipped_test_keeps_every_other_category_label(self): + labels = pcc._compute_install_labels( + CATEGORIES, {}, ["plain_binary", "other_binary"] + ) + + self.assertEqual( + labels["plain_binary"], + ["quick", "standard", "comprehensive", "full", "unit"], + ) + self.assertEqual( + labels["other_binary"], + [ + "quick", + "standard", + "comprehensive", + "full", + "ffm-quick", + "ffm-full", + "unit", + ], + ) + + def test_regex_entry_skips_every_match(self): + categories = { + "ffm-quick": { + "test_patterns": [".*"], + "exclude_tests": ["skipped_.*"], + } + } + + labels = pcc._compute_install_labels( + categories, {}, ["skipped_one", "skipped_two", "kept"] + ) + + self.assertEqual(labels["skipped_one"], []) + self.assertEqual(labels["skipped_two"], []) + self.assertEqual(labels["kept"], ["ffm-quick"]) + + +class TestExcludeTestsBuildTreeLabels(unittest.TestCase): + def test_skipped_test_is_unlabelled_in_the_build_tree(self): + cmake = shutil.which("cmake") + ctest = shutil.which("ctest") + if not cmake or not ctest: + self.skipTest("CMake and CTest are required for category integration") + + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) + build = source / "build" + (source / "categories.yaml").write_text( + "test_categories:\n" + " ffm-quick:\n" + " test_patterns:\n" + " - '.*'\n" + " exclude_tests:\n" + " - 'plain_binary'\n", + encoding="utf-8", + ) + module = (_CTEST_DIR / "TestCategories.cmake").as_posix() + # No EXPLICIT_TESTS: this is the get_property(DIRECTORY ... TESTS) loop, + # whose regex match is evaluated by CMake rather than Python. + (source / "CMakeLists.txt").write_text( + f"""cmake_minimum_required(VERSION 3.20) +project(ExcludeTests NONE) +enable_testing() +include("{module}") +foreach(name plain_binary other_binary) + add_test(NAME ${{name}} COMMAND "${{CMAKE_COMMAND}}" -E true) +endforeach() +apply_ctest_category_labels("${{CMAKE_CURRENT_SOURCE_DIR}}/categories.yaml") +""", + encoding="utf-8", + ) + configured = subprocess.run( + [ + cmake, + "-S", + str(source), + "-B", + str(build), + f"-DPython3_EXECUTABLE={sys.executable}", + f"-DROCM_LIBRARIES_ROOT={_CTEST_DIR.parent.parent}", + ], + capture_output=True, + cwd=source, + text=True, + check=False, + ) + self.assertEqual( + configured.returncode, 0, configured.stdout + configured.stderr + ) + # A multi-config generator (Visual Studio) lists no tests without -C. + selected = subprocess.run( + [ + ctest, + "--test-dir", + str(build), + "-C", + "Debug", + "--show-only=json-v1", + "-L", + "^ffm-quick$", + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(selected.returncode, 0, selected.stderr) + self.assertEqual( + {test["name"] for test in json.loads(selected.stdout)["tests"]}, + {"other_binary"}, + ) + + +if __name__ == "__main__": + unittest.main() From cdce6bb6014d68d88d74322e637288c44319c631 Mon Sep 17 00:00:00 2001 From: Adam Dickin Date: Thu, 29 Oct 2026 09:01:29 -0600 Subject: [PATCH 3/4] test(miopen-provider): make ffm-quick select the quick-tier bundles ffm-quick named six sweep-case ids, one per op. Those ids are renamed whenever a sweep is regenerated, and the FFM run only exercised three of them; the other three skipped. Select quick_* and Smoke/* instead, so FFM gets the same bundle coverage as the quick tier without a list to maintain. Drop the doc passages that described the exact-id list. --- .../integration-tests/docs/adding-tests.md | 3 +-- .../integration-tests/docs/file-formats.md | 5 +--- .../integration-tests/docs/running-tests.md | 2 +- .../test_categories_integration.yaml | 25 ++++++++----------- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/dnn-providers/integration-tests/docs/adding-tests.md b/dnn-providers/integration-tests/docs/adding-tests.md index b04b830d276c..dec9e60ec261 100644 --- a/dnn-providers/integration-tests/docs/adding-tests.md +++ b/dnn-providers/integration-tests/docs/adding-tests.md @@ -649,5 +649,4 @@ the harness (`hipdnn_integration_tests_unit_tests`). - [ ] `python dnn-providers/integration-tests/scripts/verify_support_claims.py` passes when you run it by hand. - [ ] If you regenerated or renamed sweep cases, no sidecar names a case that no - longer exists. The `ffm-quick` ids of miopen-provider still exist - (`ctest --test-dir build/dnn-providers/miopen-provider -L ffm-quick -N`). + longer exists. diff --git a/dnn-providers/integration-tests/docs/file-formats.md b/dnn-providers/integration-tests/docs/file-formats.md index 5ed279602cbd..a06bd3f5c4ef 100644 --- a/dnn-providers/integration-tests/docs/file-formats.md +++ b/dnn-providers/integration-tests/docs/file-formats.md @@ -207,8 +207,7 @@ suffix only when the readable part is not already unique. **Do not write cases by hand.** Generate them with `import_graph.py` (see [Adding Tests](adding-tests.md#add-a-bundle)). You can edit a field of an existing case, for example to add its `golden` pointer. Do not rename ids without a -reason, because support-claim sidecars and the `ffm-quick` tier of -miopen-provider name cases by id. +reason, because support-claim sidecars name cases by id. #### `tensor_patches` @@ -545,8 +544,6 @@ execution_settings: `DISABLED` and is never launched. Variants are generated for each `ex_gpu_*` label declared in the file, not for the arch that you build. To select one, use `ctest -L ex_gpu_`. -- miopen-provider's `ffm-quick` lists exact test names (sweep case ids). - Regenerating a sweep can rename those ids and silently shrink that tier. ## Where each format is enforced diff --git a/dnn-providers/integration-tests/docs/running-tests.md b/dnn-providers/integration-tests/docs/running-tests.md index b7a8f32cf481..f4288ceda39f 100644 --- a/dnn-providers/integration-tests/docs/running-tests.md +++ b/dnn-providers/integration-tests/docs/running-tests.md @@ -313,7 +313,7 @@ Each lane YAML also defines an `ffm-quick` category (labels `ffm-quick`, `ffm-full`). This is the fast-feedback tier. It is not a superset of `quick`. It differs for each lane: -- miopen lists one sweep case per op by exact name. +- miopen selects `quick_*`, `Smoke/*` and `Quick/*`, the same as its quick tier. - HIP_MLOPS selects `quick_*`, `Smoke/*` and `Quick/*`. - ASM_SDPA selects the `*Sdpa*` subset of those. - hipBLASLt selects only `Smoke/*`, `Quick/*` and `TestCpu*GoldenReference*`, diff --git a/dnn-providers/miopen-provider/test_categories_integration.yaml b/dnn-providers/miopen-provider/test_categories_integration.yaml index d65955adb101..fe4983ff510a 100644 --- a/dnn-providers/miopen-provider/test_categories_integration.yaml +++ b/dnn-providers/miopen-provider/test_categories_integration.yaml @@ -91,22 +91,17 @@ test_categories: - "all" ffm-quick: - description: "Fast-feedback mechanism (FFM) quick tier: one bundle case per operation for rapid signal" + description: "Fast-feedback mechanism (FFM) quick tier: same coverage as the quick tier (quick-tier bundle sweeps plus always-built C++ tests), with a 2 hour ctest timeout" test_patterns: - # One case per op, named exactly. These are sweep-case ids taken from - # integration-test-bundles/quick//Default/sweep.json (and the SDPA - # single-graph bundle path); regenerating a sweep can rename them, in - # which case this tier silently shrinks. Verify with: - # ctest -L ffm-quick -N - - "quick_ConvolutionFwd_Default.1_16_16_16_bfp16_nchw_dil1x1_postpad0x0_prepad0x0" - - "quick_BatchnormInference_Default.1_3_224_fp32_ncl" - - "quick_Pointwise_Default.2_4_8_8_fp16_nchw" - - "quick_Reduction_Default.1_16_8_8_bfp16_nchw_modeadd" - - "quick_RMSNorm_Default.2_3_4_4_bfp16_nchw_bcd247" - # Layernorm had no entry here at all, so the op had no fast-feedback - # signal; this is the smallest quick-tier layernorm bundle case. - - "quick_Layernorm_Default.2_2_3_2_bfp16_nhwc_normalized_dim_count1_6d6330" - - "quick_SdpaFwd_bhsd_bf16_hd128_nomask_batch_Small.Small" + # Deliberately identical to the quick tier above. This tier used to name + # six sweep-case ids explicitly (one per op); those ids are silently + # renamed whenever a sweep is regenerated, and the FFM run only ever + # exercised three of them (the other three skipped). Mirroring quick + # removes that maintenance trap and gives FFM the same bundle coverage + # every other tier gets. + - "quick_*" + - "Smoke/*" + - "Quick/*" exclude: *common_excludes labels: - "ffm-quick" From 159725d3545a4e10422693b3e48d166c2f6b557b Mon Sep 17 00:00:00 2001 From: Adam Dickin Date: Thu, 29 Oct 2026 12:40:41 -0600 Subject: [PATCH 4/4] fix(hipdnn): configure without shared/ctest when a target opts into gtest categories add_hipdnn_test(... GTEST_CATEGORIES) ended configure with FATAL_ERROR when shared/ctest was absent. The hipDNN Superbuild CI sparse checkout, and the standalone projects/hipdnn checkout that Building.md documents, have no shared/, and the rest of Tests.cmake already treats that as 'skip CTest categories'. Skip the gtest-filter suites the same way. A missing test_categories_gtest.yaml next to a present shared/ctest is a warning, because it drops the target out of the categories that file defines. --- projects/hipdnn/cmake/Tests.cmake | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/projects/hipdnn/cmake/Tests.cmake b/projects/hipdnn/cmake/Tests.cmake index d036cc141c8b..b0a196b694c6 100644 --- a/projects/hipdnn/cmake/Tests.cmake +++ b/projects/hipdnn/cmake/Tests.cmake @@ -301,12 +301,22 @@ endfunction() # finalize_test_targets # Generates the __suite entries of a GTEST_CATEGORIES target from # test_categories_gtest.yaml (see add_hipdnn_test). +# +# Without shared/ctest (a sparse or standalone checkout of projects/hipdnn) no test gets category +# labels at all, so there is nothing to pair these suites with: skip them, as the label pass above +# skips. A missing yaml next to a present shared/ctest would instead drop the target out of every +# category the yaml defines, so that case is a warning. function(_hipdnn_add_gtest_category_suites TARGET WORKING_DIR) - if(NOT COMMAND apply_test_category_labels OR NOT EXISTS "${_HIPDNN_GTEST_CATEGORIES_YAML}") - message(FATAL_ERROR - "add_hipdnn_test(${TARGET} GTEST_CATEGORIES) needs shared/ctest and " - "${_HIPDNN_GTEST_CATEGORIES_YAML}" + if(NOT COMMAND apply_test_category_labels) + message(STATUS "hipDNN: shared/ctest not available; skipping the gtest-filter suites of ${TARGET}") + return() + endif() + if(NOT EXISTS "${_HIPDNN_GTEST_CATEGORIES_YAML}") + message(WARNING + "hipDNN: ${_HIPDNN_GTEST_CATEGORIES_YAML} not found; ${TARGET} gets no gtest-filter " + "suites and is not selected by the categories that file defines" ) + return() endif() # Same environment and fixtures as the plain entry. The installed-tree lines that the parser