test(integration-tests): add the SDPA backward test plan and its generated bundles

dnn-providers/hip-kernel-provider/ASM_SDPA_ENGINE_test_categories_integration.yaml

@@ -28,8 +28,23 @@
 # The harness's generic engine-support check (checkEngineSupportOrSkip /
 # verifyGraph) skips any case ASM_SDPA_ENGINE doesn't service, so pinning the
 # whole SDPA surface is safe.
+#
+# Quarantined SdpaBackward rejection probes (RFC 0022 §6; test plan sdpa-bwd.md
+# §4.2). Each carries metadata.expected_rejection: the engine must decline it,
+# and ASM_SDPA_ENGINE accepts it today, so it would fail every lane. Each entry
+# names its ticket; the pull request that adds the decline deletes it.
+#   prbu01, prbu03  O or dQ dims that differ from Q's: ALMIOPEN-3039 (#14684)
+#   prbu13          a dQ sequence stride below D (overlapping rows): ALMIOPEN-3038 (#14684)
+#   prbt04, prbt04n an O dtype that differs from Q's: ALMIOPEN-3050
+#   prbr05          dropout_scale_inv without a dropout mask: ALMIOPEN-3051
 _common_excludes: &common_excludes
   - "*DISABLED*"
+  - "*_SdpaBackward_*.prbu01_*"
+  - "*_SdpaBackward_*.prbu03_*"
+  - "*_SdpaBackward_*.prbu13_*"
+  - "*_SdpaBackward_*.prbt04_*"
+  - "*_SdpaBackward_*.prbt04n_*"
+  - "*_SdpaBackward_*.prbr05_*"
 
 test_categories:
   quick:

dnn-providers/hip-kernel-provider/config/ASM_SDPA_ENGINE.toml

@@ -23,22 +23,135 @@ version = 1
 #   rtol: relative tolerance
 #
 
-# The common suite's ToleranceResolver maps both SdpaAttributes and
-# SdpaBackwardAttributes to tol::sdpa::getToleranceFwd<T>() (tight, tuned for
-# forward), so SDPA backward gradients (dQ/dK/dV) get their own entries here.
-# Calibrated on gfx942 against the CPU reference, with O and the LSE derived from
-# Q, K and V by the bundle fill, over every SdpaBackward case and eight seeds: the
-# worst |GPU - CPU| was 7.8e-3 for bf16 and 2.0e-3 for fp16, about one bf16 and two
-# fp16 ulps of the largest gradient (|dV| up to 2.1). Each entry leaves about 3x
-# that at the worst element, and an all-zero output fails every case.
-# Filters key on the SdpaBackward bundle case ids' dtype token; the leading "_"
-# in "_fp16_" keeps it from also matching "bfp16_".
-[[tolerance_overrides]]
-filters = ["*_SdpaBackward_*.*_fp16_*"]
+# SdpaBackward defect DEF-03 (test plan sdpa-bwd.md §4.2): the ASM kernel's fp16
+# dV loses precision when Skv <= 2 and Sq >= 512 (4 to 18 ulps, growing with Sq;
+# bf16 is bit-exact). The fp16 probes stay in their levels: every ASM run reports
+# them as skipped, and any other engine runs them. Delete this entry when the
+# kernel is fixed or the engine declines these shapes.
+[[test_skips]]
+filters = [
+    "*_SdpaBackward_*.def03a_*_fp16_*",
+    "*_SdpaBackward_*.def03d_*_fp16_*",
+    "*_SdpaBackward_*.x07_*_fp16_*",
+]
+reason = "DEF-03: fp16 dV loses precision at Skv <= 2 and Sq >= 512 (AITER kernel); tracked as ALMIOPEN-3046"
+
+# Independent PyTorch golden sweeps retain their pre-plan fp16 tolerance.
+# The generated plan cases use the per-class rows below.
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_Golden_*.*_fp16_*"]
 atol = 2.5e-3
 rtol = 2.5e-3
 
+# SdpaBackward defects DEF-01, DEF-02, DEF-04, DEF-05 and DEF-06 (test plan
+# sdpa-bwd.md §4.2): valid graphs this engine accepts today and computes wrong.
+# The fix for each is a typed decline in SdpaBwdPlanBuilder::isApplicable(); once
+# it lands the engine declines these cases itself, so each entry is temporary:
+# delete it in the pull request that adds the decline. Any other engine runs them.
+[[test_skips]]
+filters = ["*_SdpaBackward_*.def01*"]
+reason = "DEF-01: a head-dim stride other than 1 is accepted and misread; tracked as ALMIOPEN-3036 (fix: asm-sdpa-bwd-decline-strides)"
+
+[[test_skips]]
+filters = ["*_SdpaBackward_*.def02*"]
+reason = "DEF-02: Stats strides other than packed [B,Hq,Sq,1] give wrong, run-to-run different gradients; tracked as ALMIOPEN-3037 (fix: asm-sdpa-bwd-decline-strides)"
+
+[[test_skips]]
+filters = ["*_SdpaBackward_*.def04*"]
+reason = "DEF-04: Hk = Hq > Hv is accepted and dV is written past its buffer; tracked as ALMIOPEN-3039 (fix: asm-sdpa-bwd-decline-strides)"
+
+# H61 (random 0-8 element gaps per axis) hits DEF-05 whenever a gap makes a row
+# stride odd.
+[[test_skips]]
+filters = ["*_SdpaBackward_*.def05_*", "*_SdpaBackward_*.h61_*"]
+reason = "DEF-05: an odd element stride on a sequence or dK/dV head axis is misaddressed; tracked as ALMIOPEN-3038 (fix: asm-sdpa-bwd-decline-strides)"
+
+[[test_skips]]
+filters = ["*_SdpaBackward_*.def06*"]
+reason = "DEF-06: an attention scale of 0 or below gives NaN and Inf; tracked as ALMIOPEN-3043 (fix: asm-sdpa-decline-nonpositive-scale)"
+
+# The common suite's ToleranceResolver maps both SdpaAttributes and
+# SdpaBackwardAttributes to tol::sdpa::getToleranceFwd<T>() (tight, tuned for
+# forward), so SDPA backward gradients (dQ/dK/dV) get their own entries here:
+# one per tolerance class and dtype of the SdpaBackward test plan. The generator
+# writes the block between the BEGIN and END lines.
+# BEGIN SdpaBackward tolerance classes, written by generate_sdpa_bwd_bundles.py --toml
+# Plan sdpa-bwd.md §6.1. Each SdpaBackward case id ends with its tolerance class token
+# after its dtype token, so every id matches exactly one row. The class follows from the
+# case's shape, band, scale and value domain and a NumPy estimate of max|ref| per
+# gradient: atol <= 0.1 x the smallest non-zero max|ref| (the guard), and a
+# same-precision kernel model passes with margin >= 2.5 against an FP32-accumulating
+# reference. rtol is two ulps of the dtype. Do not edit: rerun the generator.
+
+# class D, bf16: diffuse softmax
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_bfp16_*_told"]
+atol = 0.0025
+rtol = 0.016
+
+# class D, fp16: diffuse softmax
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_fp16_*_told"]
+atol = 0.00075
+rtol = 0.004
+
+# class DL, bf16: diffuse softmax, long rows (bf16, Skv > 1024)
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_bfp16_*_toldl"]
+atol = 0.0012
+rtol = 0.016
+
+# class P, bf16: peaked softmax: causal, narrow window, Sq > Skv bottom-right, long Sq accumulation
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_bfp16_*_tolp"]
+atol = 0.012
+rtol = 0.016
+
+# class P, fp16: peaked softmax: causal, narrow window, Sq > Skv bottom-right, long Sq accumulation
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_fp16_*_tolp"]
+atol = 0.003
+rtol = 0.004
+
+# class S, bf16: short query side, Sq <= 17 (decode-like); tiny dK, dV
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_bfp16_*_tols"]
+atol = 0.0003
+rtol = 0.016
+
+# class S, fp16: short query side, Sq <= 17 (decode-like); tiny dK, dV
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_fp16_*_tols"]
+atol = 0.0002
+rtol = 0.004
+
+# class U, bf16: unscaled or high-spread logits (lambda >= 2): unset scale, scale ~1, peaked value domain
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_bfp16_*_tolu"]
+atol = 0.025
+rtol = 0.016
+
+# class U, fp16: unscaled or high-spread logits (lambda >= 2): unset scale, scale ~1, peaked value domain
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_fp16_*_tolu"]
+atol = 0.003
+rtol = 0.004
+
+# class K1, bf16: Skv <= 2: dQ and dK cancel to about 0 at Skv = 1; dV is a sum of dO rows
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_bfp16_*_tolk1"]
+atol = 0.0003
+rtol = 0.016
+
+# class K1, fp16: Skv <= 2: dQ and dK cancel to about 0 at Skv = 1; dV is a sum of dO rows
+[[tolerance_overrides]]
+filters = ["*_SdpaBackward_*.*_fp16_*_tolk1"]
+atol = 0.0003
+rtol = 0.004
+
+# class F32, fp32: fp32 IO (uncalibrated)
 [[tolerance_overrides]]
-filters = ["*_SdpaBackward_*.*_bfp16_*"]
-atol = 1e-2
-rtol = 1e-2
+filters = ["*_SdpaBackward_*.*_fp32_*_tolf32"]
+atol = 1e-05
+rtol = 1e-05
+# END SdpaBackward tolerance classes

dnn-providers/integration-tests/CMakeLists.txt

@@ -234,6 +234,18 @@ if(Python3_FOUND)
                          PROPERTIES SKIP_RETURN_CODE 77)
     set(HIPDNN_IT_GOLDEN_SDPA_FWD_PAGED_GENERATOR_TEST
         hipdnn_golden_sdpa_fwd_paged_generator_python_tests)
+    # The SdpaBackward bundles and their engine TOML rows are generated from the
+    # test plan; this pytest suite fails when they drift from the generator.
+    add_test(
+        NAME hipdnn_sdpa_bwd_bundle_generator_python_tests
+        COMMAND ${Python3_EXECUTABLE} -m pytest -q -p no:cacheprovider
+                ${CMAKE_CURRENT_SOURCE_DIR}/reference-data-scripts/tests/test_generate_sdpa_bwd_bundles.py
+    )
+    set_tests_properties(
+        hipdnn_sdpa_bwd_bundle_generator_python_tests
+        PROPERTIES ENVIRONMENT "PYTEST_DISABLE_PLUGIN_AUTOLOAD=1"
+    )
+    set(HIPDNN_IT_SDPA_BWD_GENERATOR_TEST hipdnn_sdpa_bwd_bundle_generator_python_tests)
 endif()
 
 # Install integration test bundles so installed tests can find them at
@@ -370,6 +382,9 @@ if(EXISTS "${HIPDNN_IT_GTEST_CATEGORIES_YAML}"
     if(HIPDNN_IT_GOLDEN_SDPA_FWD_PAGED_GENERATOR_TEST)
         _apply_provider_ctest_category_labels(${HIPDNN_IT_GOLDEN_SDPA_FWD_PAGED_GENERATOR_TEST})
     endif()
+    if(HIPDNN_IT_SDPA_BWD_GENERATOR_TEST)
+        _apply_provider_ctest_category_labels(${HIPDNN_IT_SDPA_BWD_GENERATOR_TEST})
+    endif()
 else()
     message(STATUS "hipdnn-integration-tests: shared/ctest or test_categories.yaml not found, using legacy CTest labels")
 endif()

dnn-providers/integration-tests/SUPPORT_MATRIX_OVERVIEW.md

@@ -2,7 +2,7 @@
 
 <!-- Generated by python3 dnn-providers/integration-tests/scripts/support_features/feature_matrix.py; edit the script, not this file. -->
 
-Generated from the committed support claims (12071 graphs, 488 catalogued options). Percentages are shares of each op's API options, not of its graphs. Options count singly, not in combination.
+Generated from the committed support claims (12335 graphs, 488 catalogued options). Percentages are shares of each op's API options, not of its graphs. Options count singly, not in combination.
 
 **Jump to:** [RDNA (consumer)](#rdna-consumer) · [CDNA (data center)](#cdna-data-center) · [By engine](#by-engine) · [Op groups](#op-groups) · [Options per op](#options-per-op) · [GPU targets](#gpu-targets)
 
@@ -33,7 +33,7 @@ Generated from the committed support claims (12071 graphs, 488 catalogued option
 
 | Op group (options) | [gfx942-linux](#gpu-gfx942-linux) | [gfx950-linux](#gpu-gfx950-linux) | [gfx90a-linux](#gpu-gfx90a-linux) |
 |---|---|---|---|
-| **[Attention](#group-attention)** (63) | 🟡 17% | 🟡 26% | ❌ |
+| **[Attention](#group-attention)** (63) | 🟡 42% | 🟡 26% | ❌ |
 | **[Batchnorm](#group-batchnorm)** (54) | 🟡 87% | 🟡 87% | 🟡 87% |
 | **[Block scale](#group-block-scale)** (39) | ❌ | ❌ | ❌ |
 | **[Convolution](#group-convolution)** (51) | 🟡 76% | 🟡 76% | 🟡 76% |
@@ -60,6 +60,7 @@ An op appears under an engine that claims at least one of its graphs somewhere.
 | Op (options) | [gfx90a](#gpu-gfx90a-linux)<br>L | [gfx942](#gpu-gfx942-linux)<br>L | [gfx950](#gpu-gfx950-linux)<br>L | [gfx1030](#gpu-gfx1030-windows)<br>W | [gfx1100](#gpu-gfx1100-linux)<br>L | [gfx1101](#gpu-gfx1101-linux)<br>L | [gfx1101](#gpu-gfx1101-windows)<br>W | [gfx1150](#gpu-gfx1150-linux)<br>L | [gfx1151](#gpu-gfx1151-linux)<br>L | [gfx1151](#gpu-gfx1151-windows)<br>W | [gfx1200](#gpu-gfx1200-linux)<br>L | [gfx1201](#gpu-gfx1201-linux)<br>L |
 |---|---|---|---|---|---|---|---|---|---|---|---|---|
 | **[Sdpa](#op-sdpa)** (38) | ❌ | 🟡 28% | 🟡 23% | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+| **[SdpaBackward](#op-sdpabackward)** (25) | ❌ | 🟡 64% | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 
 ### HIPBLASLT_ENGINE
 
@@ -153,7 +154,7 @@ The API node types (`<Op>Attributes` in the flatbuffers schema) that make up eac
 
 | Op group | Graphs | Options | Ops (options) |
 |---|---|---|---|
-| <a id="group-attention"></a>**Attention** | 203 | 63 | [Sdpa](#op-sdpa) (38), [SdpaBackward](#op-sdpabackward) (25) |
+| <a id="group-attention"></a>**Attention** | 467 | 63 | [Sdpa](#op-sdpa) (38), [SdpaBackward](#op-sdpabackward) (25) |
 | <a id="group-batchnorm"></a>**Batchnorm** | 2935 | 54 | [Batchnorm](#op-batchnorm) (20), [BatchnormBackward](#op-batchnormbackward) (12), [BatchnormInference](#op-batchnorminference) (10), [BatchnormInferenceVarianceExt](#op-batchnorminferencevarianceext) (12) |
 | <a id="group-block-scale"></a>**Block scale** | 126 | 39 | [BlockScaleDequantize](#op-blockscaledequantize) (20), [BlockScaleQuantize](#op-blockscalequantize) (19, **no test graph**) |
 | <a id="group-convolution"></a>**Convolution** | 1496 | 51 | [ConvolutionBwd](#op-convolutionbwd) (17), [ConvolutionFwd](#op-convolutionfwd) (17), [ConvolutionWrw](#op-convolutionwrw) (17) |
@@ -196,7 +197,7 @@ Every API option the percentages count, catalogued per op in `dnn-providers/inte
 | <a id="op-rmsnorm"></a>**RMSNorm** (21) | `dtype:fp32`, `dtype:fp16`, `dtype:bf16`, `precision:mixed_io`, `precision:mixed_affine`, `precision:mixed_stats`, `dims:4`, `dims:5`, `layout:nchw`, `layout:nhwc`, `norm:last_dim`, `norm:multi_dim`, `norm:all_but_batch`, `bias:none`, `bias:bound`, `phase:inference`, `phase:training`, `epsilon:fixed`, `epsilon:runtime` | `dims:2`, `dims:3` |
 | <a id="op-rmsnormbackward"></a>**RMSNormBackward** (17) | `dtype:fp32`, `dtype:fp16`, `dtype:bf16`, `precision:mixed_io`, `precision:mixed_affine`, `precision:mixed_stats`, `dims:4`, `dims:5`, `layout:nchw`, `layout:nhwc`, `norm:last_dim`, `norm:multi_dim`, `norm:all_but_batch`, `dbias:none`, `dbias:bound` | `dims:2`, `dims:3` |
 | <a id="op-sdpa"></a>**Sdpa** (38) | `dtype:bf16`, `dtype:fp16`, `dtype:fp8e4m3`, `dtype:fp8e5m2`, `head_dim:64`, `head_dim:128`, `head_dim:192x128`, `heads:mha`, `heads:gqa`, `heads:mqa`, `layout:bhsd`, `layout:bshd`, `layout:ragged`, `mask:none`, `mask:causal_top_left`, `mask:causal_bottom_right`, `mask:sliding_window`, `seq:q_eq_kv`, `seq:q_ne_kv`, `seq:decode`, `varlen:seq_lens`, `scale:fixed`, `scale:runtime`, `out:stats`, `out:max_sum_exp`, `fp8:amax`, `precision:mixed_output`, `bias:additive`, `bias:alibi`, `dropout:on`, `kv:paged`, `input:block_mask`, `input:sink_token` | `dtype:fp8e4m3_fnuz`, `dtype:fp8e5m2_fnuz`, `head_dim:192`, `scale:default`, `out:rng_dump` |
-| <a id="op-sdpabackward"></a>**SdpaBackward** (25) | `dtype:bf16`, `dtype:fp16`, `head_dim:64`, `head_dim:128`, `head_dim:192x128`, `heads:mha`, `heads:gqa`, `layout:bhsd`, `mask:none`, `mask:causal_top_left`, `mask:causal_bottom_right`, `mask:sliding_window`, `seq:q_eq_kv`, `seq:q_ne_kv`, `varlen:seq_lens`, `scale:fixed`, `bias:additive`, `dropout:on`, `out:dbias` | `head_dim:192`, `heads:mqa`, `layout:bshd`, `scale:runtime`, `scale:default`, `bias:alibi` |
+| <a id="op-sdpabackward"></a>**SdpaBackward** (25) | `dtype:bf16`, `dtype:fp16`, `head_dim:64`, `head_dim:128`, `head_dim:192`, `head_dim:192x128`, `heads:mha`, `heads:gqa`, `heads:mqa`, `layout:bhsd`, `layout:bshd`, `mask:none`, `mask:causal_top_left`, `mask:causal_bottom_right`, `mask:sliding_window`, `seq:q_eq_kv`, `seq:q_ne_kv`, `varlen:seq_lens`, `scale:fixed`, `scale:runtime`, `scale:default`, `bias:additive`, `dropout:on`, `out:dbias` | `bias:alibi` |
 
 ## GPU targets