EwanC · Oct 27, 2026 · Oct 28, 2026 · Nov 3, 2026
@@ -359,6 +359,33 @@ clang_tidy_check(${INTEGRATION_TESTS_EXE})
 # (see dnn-providers/miopen-provider) which runs this binary against a specific
 # engine and selects tests via that provider's test_categories_integration.yaml.
 
+# The one exception needs no plugin: --benchmark-reference with a mode that never
+# runs a reference must be refused before any engine loads. PASS_REGULAR_EXPRESSION
+# replaces the exit-code check, and the refusal message is printed only on that
+# exit. A device-less runner passes, as the binary skips itself before parsing, so
+# the entry is also staged into the installed CTest file below, which the GPU jobs
+# run.
+set(HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_TEST hipdnn_integration_tests_benchmark_reference_refusal)
+set(HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_ARGS --benchmark-reference --skip-graph-validation)
+set(HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_PASS
+    "--benchmark-reference cannot be combined with --skip-graph-validation|No HIP devices available"
+)
+add_test(
+    NAME ${HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_TEST}
+    COMMAND ${INTEGRATION_TESTS_EXE} ${HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_ARGS}
+)
+set_tests_properties(${HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_TEST} PROPERTIES
+    PASS_REGULAR_EXPRESSION "${HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_PASS}"
+)
+if(TEST_ENVIRONMENT)
+    set_tests_properties(${HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_TEST} PROPERTIES
+        ENVIRONMENT "${TEST_ENVIRONMENT}")
+endif()
+if(TEST_ENVIRONMENT_MODIFICATION)
+    set_tests_properties(${HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_TEST} PROPERTIES
+        ENVIRONMENT_MODIFICATION "${TEST_ENVIRONMENT_MODIFICATION}")
+endif()
+
 set(HIPDNN_IT_GTEST_CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/test_categories.yaml")
 set(HIPDNN_IT_CTEST_CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/test_categories_external.yaml")
 set(HIPDNN_IT_YAML_CATEGORIZATION_ENABLED FALSE)
@@ -401,6 +428,7 @@ if(EXISTS "${HIPDNN_IT_GTEST_CATEGORIES_YAML}"
     if(HIPDNN_IT_GOLDEN_SDPA_FWD_PAGED_GENERATOR_TEST)
         _apply_provider_ctest_category_labels(${HIPDNN_IT_GOLDEN_SDPA_FWD_PAGED_GENERATOR_TEST})
     endif()
+    _apply_provider_ctest_category_labels(${HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_TEST})
 else()
     message(STATUS "hipdnn-integration-tests: shared/ctest or test_categories.yaml not found, using legacy CTest labels")
 endif()
@@ -443,6 +471,31 @@ if(HIPDNN_IT_YAML_CATEGORIZATION_ENABLED)
         )
     endforeach()
 
+    # The refusal check, next to the GTest suites. It is the only pre-registered check
+    # staged here: the others run python scripts from the source tree. Labelled from
+    # test_categories_external.yaml, which matches it by exact name only.
+    set(_refusal_install_cmd
+        "add_test(\"${HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_TEST}\" \"../hipdnn_integration_tests${CMAKE_EXECUTABLE_SUFFIX}\""
+    )
+    foreach(_refusal_arg IN LISTS HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_ARGS)
+        string(APPEND _refusal_install_cmd " \"${_refusal_arg}\"")
+    endforeach()
+    string(APPEND _refusal_install_cmd ")\n")
+    set(_refusal_install_properties
+        "PASS_REGULAR_EXPRESSION \"${HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_PASS}\"")
+    # Unescaped, as the GTest suites above write theirs: ctest splits the quoted list.
+    if(TEST_ENVIRONMENT)
+        string(APPEND _refusal_install_properties " ENVIRONMENT \"${TEST_ENVIRONMENT}\"")
+    endif()
+    if(TEST_ENVIRONMENT_MODIFICATION)
+        string(APPEND _refusal_install_properties
+               " ENVIRONMENT_MODIFICATION \"${TEST_ENVIRONMENT_MODIFICATION}\"")
+    endif()
+    file(APPEND "${HIPDNN_IT_CTEST_INSTALL_FILE}"
+         "\n${_refusal_install_cmd}"
+         "set_tests_properties(\"${HIPDNN_IT_BENCHMARK_REFERENCE_REFUSAL_TEST}\" PROPERTIES ${_refusal_install_properties})\n")
+    apply_ctest_category_labels("${HIPDNN_IT_CTEST_CATEGORIES_YAML}" "${HIPDNN_IT_CTEST_INSTALL_FILE}")
+
     # Install the generated (category-labeled) CTestTestfile so
     # `ctest -L quick|standard|...` works from the installed tree.
     install(FILES "${HIPDNN_IT_CTEST_INSTALL_FILE}"

@@ -29,6 +29,7 @@
 #         [ENVIRONMENT <VAR=value>...]
 #         [INSTALL_ENVIRONMENT <VAR=value>...]
 #         [GTEST_FILTER <filter>...]
+#         [BENCHMARK_REFERENCE_SMOKE_FILTER <filter>]
 #     )
 #
 #   ``TARGET_NAME``
@@ -71,6 +72,19 @@
 #     If omitted, all tests run. Patterns can be specified one per line for
 #     readability.
 #
+#   ``BENCHMARK_REFERENCE_SMOKE_FILTER``
+#     Optional Google Test filter for a small set of bundle tests. When
+#     provided, also register ``<TARGET_NAME>-benchmark-reference-smoke``,
+#     which runs those tests with ``--benchmark-reference`` and passes only
+#     when the run prints a reference-benchmark summary with at least one
+#     verified measurement and no failed test. A runner without a HIP device
+#     passes, as the binary skips itself there. When ``INSTALL_SUBDIR`` is set,
+#     the install-tree entry is appended to the
+#     ``EXTERNAL_TEST_INSTALL_STAGING_<key>`` accumulator even when
+#     ``TEST_CATEGORIES_YAML`` keeps the base test out of it, so it reaches the
+#     installed CTest file only if the caller folds that accumulator in. Label
+#     it through the provider's CTest categories YAML.
+#
 #   ``TEST_CATEGORIES_YAML``
 #     Optional path to a GTest ``test_categories.yaml``. When provided, this
 #     helper also creates category-specific CTest suites for
@@ -199,6 +213,117 @@ macro(_stage_external_integration_install_test)
     endif()
 endmacro()
 
+# Stages the install-tree entry of the --benchmark-reference smoke test in the same
+# accumulator as an uncategorized base test's.
+#
+# Uses the _smoke_* variables _add_benchmark_reference_smoke() set and the install
+# paths _stage_external_integration_install_test() computed.
+# ~~~
+macro(_stage_benchmark_reference_smoke_install)
+    set(_smoke_install_cmd
+        "add_test(\"${_smoke_name}\" \"${_install_bin}\" \"--test-article\" \"${_install_plugin}\" \"--test-engine\" \"${ARG_ENGINE_NAME}\""
+    )
+    if(ARG_TEST_CONFIG)
+        string(APPEND _smoke_install_cmd " \"--test-config\" \"${_install_config}\"")
+    endif()
+    foreach(_smoke_arg IN LISTS _smoke_tail)
+        string(APPEND _smoke_install_cmd " \"${_smoke_arg}\"")
+    endforeach()
+    string(APPEND _smoke_install_cmd ")\n")
+
+    set(_smoke_install_properties
+        "LABELS \"${_smoke_labels}\" PASS_REGULAR_EXPRESSION \"${_smoke_pass}\" FAIL_REGULAR_EXPRESSION \"${_smoke_fail}\""
+    )
+    if(ARG_INSTALL_ENVIRONMENT)
+        set(_smoke_install_environment "${ARG_INSTALL_ENVIRONMENT}")
+    else()
+        set(_smoke_install_environment "${ARG_ENVIRONMENT}")
+    endif()
+    # Unescaped: ctest splits a quoted "A=1;B=2" into two entries, while "A=1\;B=2"
+    # stays one variable whose value holds the rest.
+    if(_smoke_install_environment)
+        string(APPEND _smoke_install_properties
+               " ENVIRONMENT \"${_smoke_install_environment}\"")
+    endif()
+    if(ARG_ENVIRONMENT_MODIFICATION)
+        string(APPEND _smoke_install_properties
+               " ENVIRONMENT_MODIFICATION \"${ARG_ENVIRONMENT_MODIFICATION}\"")
+    endif()
+    string(APPEND _smoke_install_cmd
+        "set_tests_properties(\"${_smoke_name}\" PROPERTIES ${_smoke_install_properties})\n"
+    )
+
+    set(_smoke_staging_key "${ARG_INSTALL_SUBDIR}")
+    if(ARG_INSTALL_STAGING_KEY)
+        set(_smoke_staging_key "${ARG_INSTALL_STAGING_KEY}")
+    endif()
+    set_property(GLOBAL APPEND_STRING
+        PROPERTY "EXTERNAL_TEST_INSTALL_STAGING_${_smoke_staging_key}"
+        "${_smoke_install_cmd}"
+    )
+endmacro()
+
+# Registers the --benchmark-reference smoke test, and stages its install-tree entry.
+#
+# PASS_REGULAR_EXPRESSION replaces the exit-code check, so FAIL_REGULAR_EXPRESSION
+# catches what a failing run prints instead: a failed test, or one of the messages
+# main() prints before returning non-zero once tests have run. CTest still fails a
+# crash on a signal; a non-zero exit after the summary that prints none of these
+# messages is not caught.
+#
+# Uses the ARG_* variables parsed by add_external_integration_test_target() and the
+# install paths _stage_external_integration_install_test() computed.
+# ~~~
+macro(_add_benchmark_reference_smoke)
+    if(ARG_BENCHMARK_REFERENCE_SMOKE_FILTER)
+        # Set here, not at file scope: a provider built in the same tree calls this
+        # from its own directory, which never includes this file. No backslashes, so
+        # the same text survives being written into the staged install-tree CTest
+        # file: [[] and []] match a literal bracket. A device-less runner passes, as
+        # the binary skips itself there.
+        set(_smoke_pass
+            "[[] REFERENCE BENCHMARK SUMMARY []] reference=[a-z]+ measured=[1-9]|No HIP devices available"
+        )
+        set(_smoke_fail "[[]  FAILED  []]|FATAL:|Fatal error:|Error: zero tests ran")
+        set(_smoke_name "${ARG_TARGET_NAME}-benchmark-reference-smoke")
+        set(_smoke_labels "integration_test;external_integration_test;${_engine_label}")
+        set(_smoke_tail
+            "--gtest_filter=${ARG_BENCHMARK_REFERENCE_SMOKE_FILTER}" "--benchmark-reference"
+        )
+
+        set(_smoke_cmd
+            $<TARGET_FILE:hipdnn_integration_tests>
+            --test-article $<TARGET_FILE:${ARG_PLUGIN_TARGET}>
+            --test-engine ${ARG_ENGINE_NAME}
+        )
+        if(ARG_TEST_CONFIG)
+            list(APPEND _smoke_cmd "--test-config" "${ARG_TEST_CONFIG}")
+        endif()
+        list(APPEND _smoke_cmd ${_smoke_tail})
+        add_test(NAME ${_smoke_name} COMMAND ${_smoke_cmd})
+        set_tests_properties(${_smoke_name} PROPERTIES
+            LABELS "${_smoke_labels}"
+            PASS_REGULAR_EXPRESSION "${_smoke_pass}"
+            FAIL_REGULAR_EXPRESSION "${_smoke_fail}"
+        )
+        if(ARG_ENVIRONMENT)
+            set_tests_properties(${_smoke_name} PROPERTIES ENVIRONMENT "${ARG_ENVIRONMENT}")
+        endif()
+        if(ARG_ENVIRONMENT_MODIFICATION)
+            set_tests_properties(${_smoke_name} PROPERTIES
+                ENVIRONMENT_MODIFICATION "${ARG_ENVIRONMENT_MODIFICATION}")
+        endif()
+        if(ARG_FIXTURES_REQUIRED)
+            set_tests_properties(${_smoke_name} PROPERTIES
+                FIXTURES_REQUIRED "${ARG_FIXTURES_REQUIRED}")
+        endif()
+
+        if(_install_bin)
+            _stage_benchmark_reference_smoke_install()
+        endif()
+    endif()
+endmacro()
+
 # Adds category-specific GTest-filtered CTest suites.
 #
 # Uses the ARG_* variables parsed by add_external_integration_test_target().
@@ -288,7 +413,7 @@ function(add_external_integration_test_target)
     cmake_parse_arguments(
         ARG
         ""
-        "TARGET_NAME;PLUGIN_TARGET;ENGINE_NAME;INSTALL_SUBDIR;TEST_CONFIG;REFERENCE_EXECUTOR;TEST_CATEGORIES_YAML;INSTALL_TEST_FILE;TEST_NAME_PREFIX;INSTALL_DESTINATION;INSTALL_STAGING_KEY"
+        "TARGET_NAME;PLUGIN_TARGET;ENGINE_NAME;INSTALL_SUBDIR;TEST_CONFIG;REFERENCE_EXECUTOR;TEST_CATEGORIES_YAML;INSTALL_TEST_FILE;TEST_NAME_PREFIX;INSTALL_DESTINATION;INSTALL_STAGING_KEY;BENCHMARK_REFERENCE_SMOKE_FILTER"
         "GTEST_FILTER;ENVIRONMENT;INSTALL_ENVIRONMENT;ENVIRONMENT_MODIFICATION;FIXTURES_REQUIRED"
         ${ARGN}
     )
@@ -357,5 +482,6 @@ function(add_external_integration_test_target)
     # can include this test in the installed CTestTestfile.cmake. Required for
     # CI flows that invoke ctest from the install tree (e.g. TheRock).
     _stage_external_integration_install_test()
+    _add_benchmark_reference_smoke()
     _add_external_integration_category_suites()
 endfunction()
@@ -8,7 +8,7 @@ developer who must run the suite, read its output, or add to it.
 | Document | Read it to… |
 |---|---|
 | [File Formats](file-formats.md) | Understand every file in the bundle tree and the provider configs: graphs, template sweeps, metadata, golden data and DVC pointers, support-claim sidecars, per-engine TOML, tier YAML. Also learn how test names are derived. |
-| [Running the Tests](running-tests.md) | Run the suite (CTest, check targets, the binary directly), time the engine with `--benchmark`, learn what the tiers and filters select, and read the output: coverage summary, support-claim summary, hard stops, and runs that look green but are not. |
+| [Running the Tests](running-tests.md) | Run the suite (CTest, check targets, the binary directly), time the engine with `--benchmark` and the reference with `--benchmark-reference`, learn what the tiers and filters select, and read the output: coverage summary, support-claim summary, hard stops, and runs that look green but are not. |
 | [Adding Tests and Updating Claims](adding-tests.md) | Add a bundle (end to end, including a new op), add golden data, update support claims when a run reports `unclaimed_support`, record an engine limitation, add an engine lane, or add a C++ test when a bundle cannot express it. |
 | [Support Claim Enforcement](support-claim-enforcement.md) | For harness maintainers: the claim verdict model, the lifecycle inside `TestBody()`, the coverage ladder, and who owns what in the harness. |
 | [Support Matrix and Feature Coverage](support-matrix.md) | Render the human-readable matrix of which engine supports which graph on which GPU and OS, validate the sidecars it is built from, see the pre-commit and CI checks around it, and measure per-op feature coverage. |
@@ -52,11 +52,14 @@ other rows do not, so they are easy to miss.
 | `Error: zero tests ran.` | **yes** | A discovery or filter misconfiguration, or, with GTest sharding, an empty shard. | [Hard stops](running-tests.md#hard-stops) |
 | `FATAL: --enforce-support-claims is active and …` | **yes** | Enforcement verified nothing. | [Hard stops](running-tests.md#hard-stops) |
 | `benchmark failed: …` | **yes** (that test failed) | Under `--benchmark`, the verified graph failed when it ran again, or the timer returned too few measurements. | [Time the engine](running-tests.md#4-time-the-engine---benchmark) |
+| `reference benchmark synchronization failed: …` | **yes** (that test failed) | Under `--benchmark-reference`, a device synchronize around a reference phase failed. The test does not fall back to another reference. | [Time the reference](running-tests.md#5-time-the-reference---benchmark-reference) |
 | `unclaimed_support` in the `SUPPORT CLAIM SUMMARY` | no | The engine accepts graphs that no sidecar claims. Record them. | [Updating support claims](adding-tests.md#updating-support-claims) |
 | `Skipped:` equal to (or close to) the total | no | Green, but little or nothing was tested. Read the skip reasons. | [Test coverage summary](running-tests.md#test-coverage-summary) |
 | `WARNING: Bundle tests are enabled but …` / `WARNING: No bundles could be loaded from …` | no | No bundles registered. Only compiled-in C++ tests ran. | [Troubleshooting](running-tests.md#troubleshooting) |
 | `No tests were found!!!` from CTest | no (exit 0) | Wrong directory, wrong label, or only disabled suites. | [Runs that look green but are not](running-tests.md#runs-that-look-green-but-are-not) |
 | `No HIP devices available; skipping …` | no (exit 0) | There is no GPU, so nothing ran. | [Runs that look green but are not](running-tests.md#runs-that-look-green-but-are-not) |
+| `[ REFERENCE BENCHMARK ] … status=declined` or `status=error` | no | Under `--benchmark-reference`, a reference was tried and did not produce a sample. The attempt has no timing; it is not a zero-cost reference. | [Time the reference](running-tests.md#5-time-the-reference---benchmark-reference) |
+| `[ REFERENCE BENCHMARK SUMMARY ] measured=0` | no | Under `--benchmark-reference`, no reference was verified, so nothing was measured. | [Time the reference](running-tests.md#5-time-the-reference---benchmark-reference) |
 | `REFERENCE EXECUTOR ERRORS` | sometimes | A reference executor threw. The test fails if this was the last oracle to try. Otherwise, the run fell back. | [Unverifiable bundles](running-tests.md#unverifiable-bundles) |
 | `verdicts` all zero while `unenforced.no_applicable_claim` > 0 | no | The harness read the sidecars, but they claim nothing for this arch and platform. Nothing was enforced here. | [Updating support claims](adding-tests.md#updating-support-claims) |
 | `WARNING ONLY -- NOT ENFORCED` summary header | no | Enforcement was off (`--enforce-support-claims=false`). Broken claims were reported but did not fail the run. | [Command-line reference](running-tests.md#command-line-reference--hipdnn_integration_tests) |