benchmark: add Qwen3-VL vision-encoder (ViT) SDPA config + GB300 results

benchmark/sdpa_benchmark_training/README.md

@@ -16,6 +16,7 @@ This directory contains benchmarking tools for Scaled Dot Product Attention (SDP
   - `ltx2.py` - LTX-2 video DiT self-attention benchmarks (bidirectional, no mask)
   - `gpt_oss.py` - GPT-OSS sliding-window-attention GQA benchmarks (causal, SWA=128)
   - `qwen35.py` - Qwen 3.5 GQA benchmarks (head_dim=256, causal, bf16 bidirectional — Blackwell fp8/fa4 limits)
+  - `qwen3vl_vit.py` - Qwen3-VL vision-encoder (ViT) self-attention benchmarks (bidirectional, no mask, fwd-only, head_dim 72-in-80)
   - `auto_regressive_dit.py` - Autoregressive video DiT (short Q, long cached KV, bf16/mxfp8, no_mask)
 - `runner.py` - Configuration-based benchmark runner
 - `config_types.py` - Data types for benchmark configuration
@@ -56,6 +57,9 @@ python -m benchmark.sdpa_benchmark_training.runner --config ltx2
 # Run Qwen 3.5 benchmark suite (cuDNN bf16 at head_dim=256)
 python -m benchmark.sdpa_benchmark_training.runner --config qwen35
 
+# Run Qwen3-VL vision-encoder (ViT) benchmark suite (inference fwd-only)
+python -m benchmark.sdpa_benchmark_training.runner --config qwen3vl_vit
+
 # Run Autoregressive video DiT benchmark suite (short Q, long cached KV)
 python -m benchmark.sdpa_benchmark_training.runner --config auto_regressive_dit
 

benchmark/sdpa_benchmark_training/configs/qwen3vl_vit.py

@@ -0,0 +1,65 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Qwen3-VL Vision Encoder (ViT) SDPA Benchmark Configuration
+
+Benchmarks the self-attention of the Qwen3-VL vision encoder as exercised
+by image/video inference. ViT self-attention is bidirectional over the
+patchified image tokens, so only ``no_mask`` is benchmarked, forward pass
+only (inference workload).
+
+Architecture (vision tower):
+    - hidden dim           = 1152
+    - num_attention_heads  = 16   (MHA, no GQA)
+    - attention head_dim   = 72   (16 x 72 = 1152)
+    - benchmarked head_dim = 80   (72 zero-padded to the next 16-byte
+      multiple: fp8 kernels require 16B-aligned head dims, and production
+      integrations run this padded contract. Reported TFLOPS count d=80.)
+
+Sequence lengths are per-image patch-grid token counts taken from a
+production inference trace (native-resolution images; token count =
+grid_h x grid_w). Single-image (batch 1) forwards dominate that trace,
+and these six lengths span its per-forward FLOPs distribution from the
+10th to the 99th percentile:
+
+    ( 8836,  94 x  94 grid)
+    (15376, 124 x 124 grid)   <- most frequent single-image forward
+    (24336, 156 x 156 grid)
+    (35344, 188 x 188 grid)   <- FLOPs-median forward
+    (47376, non-square grid, e.g. 168 x 282)
+    (62500, 250 x 250 grid)
+
+Usage:
+    python -m benchmark.sdpa_benchmark_training.runner --config qwen3vl_vit
+    python -m benchmark.sdpa_benchmark_training.runner --config qwen3vl_vit --dry-run
+"""
+
+from ..config_types import ModelPreset, BenchmarkConfig
+
+QWEN3VL_VIT = ModelPreset(
+    name="qwen3vl_vit",
+    num_q_heads=16,
+    num_kv_heads=16,
+    head_dim=80,
+)
+
+CONFIG = BenchmarkConfig(
+    name="qwen3vl_vit",
+    models=[QWEN3VL_VIT],
+    seqlens=[
+        (8836, 8836),  # 94x94 patch grid
+        (15376, 15376),  # 124x124 (most frequent)
+        (24336, 24336),  # 156x156
+        (35344, 35344),  # 188x188 (FLOPs median)
+        (47376, 47376),  # non-square grid (168x282)
+        (62500, 62500),  # 250x250
+    ],
+    backends=["cudnn", "flash_attention_4"],
+    data_types=["bfloat16", "fp8"],
+    attn_masks=["no_mask"],
+    profile_pass="fwd",
+    batch_size=1,
+    num_iterations=10,
+    num_warmup_iterations=5,
+)

benchmark/sdpa_benchmark_training/results/qwen3vl_vit/gb300/qwen3vl_vit_20260814_122905.csv

@@ -0,0 +1,19 @@
+config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,error_message,gpu_name,cudnn_version,cudnn_backend_version
+qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,8836,8836,16,16,80,80,fwd,False,0.327,1224.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,8836,8836,16,16,80,80,fwd,False,0.260,1537.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,8836,8836,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,AssertionError: Only SM 10.x and 11.x are supported,,,
+qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,15376,15376,16,16,80,80,fwd,False,0.971,1247.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,15376,15376,16,16,80,80,fwd,False,0.739,1639.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,15376,15376,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,AssertionError: Only SM 10.x and 11.x are supported,,,
+qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,24336,24336,16,16,80,80,fwd,False,2.427,1249.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,24336,24336,16,16,80,80,fwd,False,1.790,1694.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,24336,24336,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,AssertionError: Only SM 10.x and 11.x are supported,,,
+qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,35344,35344,16,16,80,80,fwd,False,5.137,1245.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,35344,35344,16,16,80,80,fwd,False,3.679,1738.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,35344,35344,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,AssertionError: Only SM 10.x and 11.x are supported,,,
+qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,47376,47376,16,16,80,80,fwd,False,9.223,1246.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,47376,47376,16,16,80,80,fwd,False,6.691,1718.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,47376,47376,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,AssertionError: Only SM 10.x and 11.x are supported,,,
+qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,62500,62500,16,16,80,80,fwd,False,15.821,1264.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,62500,62500,16,16,80,80,fwd,False,11.940,1675.000,0.000,10,,True,,NVIDIA B300 SXM6 AC,1.27.0,92500.000
+qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,62500,62500,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,AssertionError: Only SM 10.x and 11.x are supported,,,