test(integration-tests): re-tier integration bundles per op and make tier cascading explicit

dnn-providers/hip-kernel-provider/ASM_SDPA_ENGINE_test_categories_integration.yaml

@@ -11,21 +11,23 @@
 #
 # Tier pattern shape mirrors miopen-provider/test_categories_integration.yaml,
 # scoped to the SDPA surface this engine actually implements:
-#   - "quick_*Sdpa*"/"standard_*Sdpa*" match the bundle suites
-#     (integration-test-bundles/{quick,standard}/SdpaFwd*), whose names mirror
-#     the on-disk tier directory; bundles are the CI driver.
+#   - "quick_*Sdpa*"/"standard_*Sdpa*"/"comprehensive_*Sdpa*" match the bundle
+#     suites (integration-test-bundles/{quick,standard,comprehensive}/Sdpa*),
+#     whose names mirror the on-disk tier directory; bundles are the CI driver.
 #   - "Smoke/*Sdpa*" (slash) matches the C++ parameterised suites
 #     (Smoke/IntegrationGpuSdpaFwdBfp16 and the ported backward cases
 #     Smoke/IntegrationGpuSdpaBwd{Bfp16,Fp16}). These no longer build by
 #     default -- see BUILD_CPP_GRAPH_TESTS in
 #     dnn-providers/integration-tests/src/integration-tests/CMakeLists.txt --
 #     but keep the pattern regardless so a graph-tests-enabled build still
 #     picks them up.
-#   - "*Sdpa*" (comprehensive/full) covers the whole SDPA surface in either
-#     authoring style, including any future full-tier SDPA bundle.
+#   - "*Sdpa*" (full) covers the whole SDPA surface in either authoring style,
+#     including the full-tier SDPA bundles no lower tier selects.
+# Tiers cascade: standard runs quick + standard, comprehensive runs quick +
+# standard + comprehensive; only "full" runs the full-tier bundles.
 # The harness's generic engine-support check (checkEngineSupportOrSkip /
 # verifyGraph) skips any case ASM_SDPA_ENGINE doesn't service, so pinning the
-# whole SDPA surface at the comprehensive/full tier is safe.
+# whole SDPA surface is safe.
 _common_excludes: &common_excludes
   - "*DISABLED*"
   # SdpaBackward bundles fail on two independent issues:
@@ -60,9 +62,15 @@ test_categories:
       - "precheckin"
 
   comprehensive:
-    description: "Nightly comprehensive ASM SDPA external integration coverage"
+    description: "Nightly comprehensive ASM SDPA external integration coverage: quick + standard + comprehensive-tier bundles"
     test_patterns:
-      - "*Sdpa*"
+      - "quick_*Sdpa*"
+      - "standard_*Sdpa*"
+      - "comprehensive_*Sdpa*"
+      - "Smoke/*Sdpa*"
+      # The always-built C++ tests that carry no tier prefix
+      # (IntegrationCpuSdpaFwdBwdChained).
+      - "Integration*Sdpa*"
     exclude: *common_excludes
     labels:
       - "comprehensive"
@@ -92,12 +100,16 @@ test_categories:
       - "ffm-quick"
       - "ffm-full"
 
+# category_timeouts are CTest kill limits, set to 2x the tier's runtime budget
+# (quick ~5 min, standard ~30 min incl. quick, comprehensive <= 60 min incl.
+# quick+standard; full has no budget) so a budget overrun is a failure to
+# investigate while ordinary run-to-run variance does not trip the timeout.
 execution_settings:
   default_timeout: 300
   timeout_multiplier: 1
   category_timeouts:
     quick: 1200
-    standard: 2700
-    comprehensive: 25200
+    standard: 3600
+    comprehensive: 7200
     full: 28800
     ffm-quick: 7200  # 2 hours

dnn-providers/hip-kernel-provider/HIP_MLOPS_ENGINE_test_categories_integration.yaml

@@ -9,16 +9,18 @@
 # or hip_kernel_provider_integration_tests names.
 #
 # Tier pattern shape mirrors miopen-provider/test_categories_integration.yaml:
-#   - "quick_*"/"standard_*" match the bundle suites (integration-test-bundles/
-#     {quick,standard}/...), whose names mirror the on-disk tier directory;
+#   - "quick_*"/"standard_*"/"comprehensive_*" match the bundle suites
+#     (integration-test-bundles/{quick,standard,comprehensive}/...), whose names
+#     mirror the on-disk tier directory;
 #     bundles are the CI driver. "Smoke/*" (slash) matches the always-built,
 #     non-graph C++ parameterised suites -- most Smoke/* graph tests only build
 #     under -DBUILD_CPP_GRAPH_TESTS=ON (see that flag in
 #     dnn-providers/integration-tests/src/integration-tests/CMakeLists.txt),
 #     but keep the pattern regardless so a graph-tests-enabled build still
 #     picks them up. GTest treats '/' and '_' as literal chars, so both
-#     pattern shapes are needed. "full_*" matches the full-tier bundles
-#     (integration-test-bundles/full/...).
+#     pattern shapes are needed. The "full" tier runs everything, including the
+#     full-tier bundles (integration-test-bundles/full/...), which no lower tier
+#     selects.
 #
 # hipdnn_integration_tests covers ops (conv, matmul, sdpa, reduction, ...) that
 # HIP_MLOPS_ENGINE doesn't implement; the harness's generic engine-support check
@@ -59,51 +61,31 @@ test_categories:
       - "precheckin"
 
   comprehensive:
-    description: "Nightly comprehensive external integration coverage"
-    # An include list, like quick and standard above, rather than "*". With "*"
-    # this tier was byte-identical to "full" below, so the on-disk bundle tiers
-    # and the C++ tier prefixes meant nothing here.
-    #
-    # Everything listed has to fit the 30-minute wall the nightly job actually
-    # dies at (ROCm/rocm-libraries#11529), which the 7200 s CTest timeout below
-    # never gets to enforce.
+    description: "Nightly comprehensive external integration coverage: quick + standard + comprehensive-tier bundles"
+    # Tiers cascade: comprehensive runs everything quick and standard run, plus
+    # the comprehensive-tier bundles. It deliberately does NOT run the full-tier
+    # bundles (full_*): "full" holds only what does not belong in a lower tier
+    # (exhaustive/stress cases, or cases redundant with lower-tier coverage of
+    # the same option axis) and has no runtime budget; "full" below runs it.
     #
     # Two namespaces are in play and both are needed. Lowercase "quick_*" /
-    # "standard_*" / "full_*" are bundle suites, named after their on-disk tier
-    # under integration-test-bundles/. Capitalised "Smoke/" / "Quick/" /
-    # "Standard/" / "Comprehensive/" are C++ gtest prefixes. "full_*" is listed
-    # and "Full/*" is not: the bundle full tier is nightly coverage this tier
-    # has always carried, while the C++ Full tier is the weekly-only set that
-    # "full" below still runs. "Comprehensive/*" matches nothing today --
-    # layernorm was the only op with that prefix and its shapes moved to Full --
-    # but it is kept so a future op that adopts the tier prefixes is picked up
-    # here, and so all three provider files keep the same shape.
-    #
-    # This replaced a list of hand-maintained carve-outs that had to name gtest
-    # parameter indices, which shifted silently whenever a shape list changed.
-    # The miopen and hipblaslt files are include-lists of the same shape; only
-    # the "full_Layernorm*" carve-out below is unique to this engine.
+    # "standard_*" / "comprehensive_*" are bundle suites, named after their
+    # on-disk tier under integration-test-bundles/. Capitalised "Smoke/" /
+    # "Quick/" / "Standard/" / "Comprehensive/" are C++ gtest prefixes; "Full/*"
+    # is not listed because the C++ Full tier is the weekly-only set that
+    # "full" below still runs.
     test_patterns:
       - "quick_*"
       - "standard_*"
-      - "full_*"
+      - "comprehensive_*"
       - "Smoke/*"
       - "Quick/*"
       - "Standard/*"
       - "Comprehensive/*"
       # The always-built C++ tests that carry no tier prefix
       # (IntegrationIsSupportedExtPerformance, IntegrationCpuSdpaFwdBwdChained).
       - "Integration*"
-    # Not *common_excludes: this tier needs one extra carve-out, and YAML
-    # cannot append to an aliased sequence, so the shared entry is repeated.
-    exclude:
-      - "*DISABLED*"
-      # The full-tier Layernorm bundles (forward and backward) are omitted
-      # because they take too much time in CI (ROCm/rocm-libraries#11529). The
-      # glob has no underscore after "Layernorm" on purpose so it also matches
-      # full_LayernormBackward_*. They are excluded by suite name rather than by
-      # dropping "full_*" wholesale so the other full-tier bundles still run.
-      - "full_Layernorm*"
+    exclude: *common_excludes
     labels:
       - "comprehensive"
       - "nightly"
@@ -132,12 +114,16 @@ test_categories:
       - "ffm-quick"
       - "ffm-full"
 
+# category_timeouts are CTest kill limits, set to 2x the tier's runtime budget
+# (quick ~5 min, standard ~30 min incl. quick, comprehensive <= 60 min incl.
+# quick+standard; full has no budget) so a budget overrun is a failure to
+# investigate while ordinary run-to-run variance does not trip the timeout.
 execution_settings:
   default_timeout: 300
   timeout_multiplier: 1
   category_timeouts:
     quick: 1200
-    standard: 2700
+    standard: 3600
     comprehensive: 7200
     full: 28800
     ffm-quick: 7200  # 2 hours

dnn-providers/hip-kernel-provider/hipkernel_Gfx950AttentionDense_test_categories_integration.yaml

@@ -4,11 +4,13 @@
 # hip-kernel-provider hipkernel:Gfx950AttentionDense external integration test categories.
 #
 # Pinned to hipkernel:Gfx950AttentionDense via --test-engine. Selects the shared sweeps
-# integration-test-bundles/{quick,standard}/SdpaFwd/Default/ by suite, not by text in
-# the case id: import_graph.py names cases in its own format, so a name pattern would
-# miss the cases it adds. Engine pinning and support claims do the rest: a claimed case
-# the engine declines fails its lane, an unclaimed one it declines skips, and an
-# unclaimed one it accepts runs and is reported as unclaimed support.
+# integration-test-bundles/{quick,standard,comprehensive,full}/SdpaFwd/Default/ by suite,
+# not by text in the case id: import_graph.py names cases in its own format, so a name
+# pattern would miss the cases it adds. Engine pinning and support claims do the rest: a
+# claimed case the engine declines fails its lane, an unclaimed one it declines skips,
+# and an unclaimed one it accepts runs and is reported as unclaimed support.
+# Tiers cascade: standard runs quick + standard, comprehensive runs quick +
+# standard + comprehensive; only "full" runs the full-tier bundles.
 
 _common_excludes: &common_excludes
   - "*DISABLED*"
@@ -37,9 +39,11 @@ test_categories:
       - "precheckin"
 
   comprehensive:
-    description: "Comprehensive gfx950 dense attention coverage"
+    description: "Comprehensive gfx950 dense attention coverage: quick + standard + comprehensive-tier bundles"
     test_patterns:
-      - "*_SdpaFwd_Default.*"
+      - "quick_SdpaFwd_Default.*"
+      - "standard_SdpaFwd_Default.*"
+      - "comprehensive_SdpaFwd_Default.*"
     exclude: *common_excludes
     labels:
       - "comprehensive"
@@ -64,12 +68,16 @@ test_categories:
       - "ffm-quick"
       - "ffm-full"
 
+# category_timeouts are CTest kill limits, set to 2x the tier's runtime budget
+# (quick ~5 min, standard ~30 min incl. quick, comprehensive <= 60 min incl.
+# quick+standard; full has no budget) so a budget overrun is a failure to
+# investigate while ordinary run-to-run variance does not trip the timeout.
 execution_settings:
   default_timeout: 300
   timeout_multiplier: 1
   category_timeouts:
     quick: 600
-    standard: 1800
+    standard: 3600
     comprehensive: 7200
     full: 14400
     ffm-quick: 1800

dnn-providers/hip-kernel-provider/src/CMakeLists.txt

@@ -398,7 +398,7 @@ if(HIPKERNELPROVIDER_ENABLE_TESTS)
             INSTALL_SUBDIR hip_kernel_provider
             ${_gfx950_shard_args}
             TEST_CONFIG    ${CMAKE_CURRENT_SOURCE_DIR}/../config/hipkernel_Gfx950AttentionDense.toml
-            GTEST_FILTER   "quick_SdpaFwd_Default.*:standard_SdpaFwd_Default.*"
+            GTEST_FILTER   "quick_SdpaFwd_Default.*:standard_SdpaFwd_Default.*:comprehensive_SdpaFwd_Default.*:full_SdpaFwd_Default.*"
             ENVIRONMENT    ${TEST_ENVIRONMENT}
             ENVIRONMENT_MODIFICATION ${TEST_ENVIRONMENT_MODIFICATION}
             ${_gfx950_attention_dense_category_args}

dnn-providers/hip-kernel-provider/src/integration_tests/CMakeLists.txt

@@ -223,7 +223,7 @@ if(HIPDNN_ENABLE_KERNEL_INGESTOR AND HIPDNN_ENABLE_SDPA
         INSTALL_SUBDIR hip_kernel_provider
         ${_gfx950_shard_args}
         REFERENCE_EXECUTOR gpu
-        GTEST_FILTER "quick_SdpaFwd_Default.*:standard_SdpaFwd_Default.*"
+        GTEST_FILTER "quick_SdpaFwd_Default.*:standard_SdpaFwd_Default.*:comprehensive_SdpaFwd_Default.*:full_SdpaFwd_Default.*"
         ENVIRONMENT_MODIFICATION ${TEST_ENVIRONMENT_MODIFICATION}
     )
 endif()

dnn-providers/hipblaslt-provider/config/HIPBLASLT_ENGINE.toml

@@ -28,7 +28,7 @@ version = 1
 # ref=4.5, threshold becomes 0.11) with ~75% margin, and covers the
 # smaller 0.03125-at-ref=2.0625 case with ~95% margin.
 [[tolerance_overrides]]
-filters = ["Smoke/IntegrationGpuMatmulBiasActivBf16.Correctness/*", "quick_MatmulPointwise_Variant3.*bfp16*swish*"]
+filters = ["Smoke/IntegrationGpuMatmulBiasActivBf16.Correctness/*", "*_MatmulPointwise_Variant3.*bfp16*swish*"]
 atol = 0.02
 rtol = 0.02
 

dnn-providers/hipblaslt-provider/test_categories_integration.yaml

@@ -57,23 +57,23 @@ test_categories:
       - "precheckin"
 
   comprehensive:
-    description: "Nightly comprehensive external integration coverage"
-    # An include list, like quick and standard above, rather than "*". With "*"
-    # this tier was byte-identical to "full" below, so the C++ tier prefixes
-    # meant nothing: a suite's Full tier ran in the nightly job as well as the
-    # weekly one.
+    description: "Nightly comprehensive external integration coverage: quick + standard + comprehensive-tier bundles"
+    # Tiers cascade: comprehensive runs everything quick and standard run, plus
+    # the comprehensive-tier bundles. It deliberately does NOT run the full-tier
+    # bundles (full_*): "full" holds only what does not belong in a lower tier
+    # (exhaustive/stress cases, or cases redundant with lower-tier coverage of
+    # the same option axis) and has no runtime budget; "full" below runs it.
     #
     # Two namespaces are in play and both are needed. Lowercase "quick_*" /
-    # "standard_*" / "full_*" are bundle suites, named after their on-disk tier
-    # under integration-test-bundles/. Capitalised "Smoke/" / "Quick/" /
-    # "Standard/" / "Comprehensive/" are C++ gtest prefixes. "full_*" is listed
-    # and "Full/*" is not: the bundle full tier is nightly coverage this tier has
-    # always carried, while the C++ Full tier is the weekly-only set that "full"
-    # below still runs.
+    # "standard_*" / "comprehensive_*" are bundle suites, named after their
+    # on-disk tier under integration-test-bundles/. Capitalised "Smoke/" /
+    # "Quick/" / "Standard/" / "Comprehensive/" are C++ gtest prefixes; "Full/*"
+    # is not listed because the C++ Full tier is the weekly-only set that
+    # "full" below still runs.
     test_patterns:
       - "quick_*"
       - "standard_*"
-      - "full_*"
+      - "comprehensive_*"
       - "Smoke/*"
       - "Quick/*"
       - "Standard/*"
@@ -110,12 +110,16 @@ test_categories:
       - "ffm-quick"
       - "ffm-full"
 
+# category_timeouts are CTest kill limits, set to 2x the tier's runtime budget
+# (quick ~5 min, standard ~30 min incl. quick, comprehensive <= 60 min incl.
+# quick+standard; full has no budget) so a budget overrun is a failure to
+# investigate while ordinary run-to-run variance does not trip the timeout.
 execution_settings:
   default_timeout: 300
   timeout_multiplier: 1
   category_timeouts:
-    quick: 300
-    standard: 2700
-    comprehensive: 25200
+    quick: 600
+    standard: 3600
+    comprehensive: 7200
     full: 28800
     ffm-quick: 7200  # 2 hours