diff --git a/benchmark/sdpa_benchmark_training/README.md b/benchmark/sdpa_benchmark_training/README.md index d0e843b93d..f23434d5d6 100755 --- a/benchmark/sdpa_benchmark_training/README.md +++ b/benchmark/sdpa_benchmark_training/README.md @@ -16,6 +16,7 @@ This directory contains benchmarking tools for Scaled Dot Product Attention (SDP - `ltx2.py` - LTX-2 video DiT self-attention benchmarks (bidirectional, no mask) - `gpt_oss.py` - GPT-OSS sliding-window-attention GQA benchmarks (causal, SWA=128) - `qwen35.py` - Qwen 3.5 GQA benchmarks (head_dim=256, causal, bf16 bidirectional — Blackwell fp8/fa4 limits) + - `qwen3vl_vit.py` - Qwen3-VL vision-encoder (ViT) self-attention benchmarks (bidirectional, no mask, fwd-only, head_dim 72-in-80) - `auto_regressive_dit.py` - Autoregressive video DiT (short Q, long cached KV, bf16/mxfp8, no_mask) - `runner.py` - Configuration-based benchmark runner - `config_types.py` - Data types for benchmark configuration @@ -56,6 +57,9 @@ python -m benchmark.sdpa_benchmark_training.runner --config ltx2 # Run Qwen 3.5 benchmark suite (cuDNN bf16 at head_dim=256) python -m benchmark.sdpa_benchmark_training.runner --config qwen35 +# Run Qwen3-VL vision-encoder (ViT) benchmark suite (inference fwd-only) +python -m benchmark.sdpa_benchmark_training.runner --config qwen3vl_vit + # Run Autoregressive video DiT benchmark suite (short Q, long cached KV) python -m benchmark.sdpa_benchmark_training.runner --config auto_regressive_dit diff --git a/benchmark/sdpa_benchmark_training/configs/qwen3vl_vit.py b/benchmark/sdpa_benchmark_training/configs/qwen3vl_vit.py new file mode 100644 index 0000000000..bb8ed5cc45 --- /dev/null +++ b/benchmark/sdpa_benchmark_training/configs/qwen3vl_vit.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Qwen3-VL Vision Encoder (ViT) SDPA Benchmark Configuration + +Benchmarks the self-attention of the Qwen3-VL vision encoder as exercised +by image/video inference. ViT self-attention is bidirectional over the +patchified image tokens, so only ``no_mask`` is benchmarked, forward pass +only (inference workload). + +Architecture (vision tower): + - hidden dim = 1152 + - num_attention_heads = 16 (MHA, no GQA) + - attention head_dim = 72 (16 x 72 = 1152) + - benchmarked head_dim = 80 (72 zero-padded to the next 16-byte + multiple: fp8 kernels require 16B-aligned head dims, and production + integrations run this padded contract. Reported TFLOPS count d=80.) + +Sequence lengths are per-image patch-grid token counts taken from a +production inference trace (native-resolution images; token count = +grid_h x grid_w). Single-image (batch 1) forwards dominate that trace, +and these six lengths span its per-forward FLOPs distribution from the +10th to the 99th percentile: + + ( 8836, 94 x 94 grid) + (15376, 124 x 124 grid) <- most frequent single-image forward + (24336, 156 x 156 grid) + (35344, 188 x 188 grid) <- FLOPs-median forward + (47376, non-square grid, e.g. 168 x 282) + (62500, 250 x 250 grid) + +Usage: + python -m benchmark.sdpa_benchmark_training.runner --config qwen3vl_vit + python -m benchmark.sdpa_benchmark_training.runner --config qwen3vl_vit --dry-run +""" + +from ..config_types import ModelPreset, BenchmarkConfig + +QWEN3VL_VIT = ModelPreset( + name="qwen3vl_vit", + num_q_heads=16, + num_kv_heads=16, + head_dim=80, +) + +CONFIG = BenchmarkConfig( + name="qwen3vl_vit", + models=[QWEN3VL_VIT], + seqlens=[ + (8836, 8836), # 94x94 patch grid + (15376, 15376), # 124x124 (most frequent) + (24336, 24336), # 156x156 + (35344, 35344), # 188x188 (FLOPs median) + (47376, 47376), # non-square grid (168x282) + (62500, 62500), # 250x250 + ], + backends=["cudnn", "cudnn_oss", "flash_attention_4"], + data_types=["bfloat16", "fp8"], + attn_masks=["no_mask"], + profile_pass="fwd", + batch_size=1, + num_iterations=10, + num_warmup_iterations=5, +) diff --git a/benchmark/sdpa_benchmark_training/results/qwen3vl_vit/gb300/qwen3vl_vit_20260814_193600.csv b/benchmark/sdpa_benchmark_training/results/qwen3vl_vit/gb300/qwen3vl_vit_20260814_193600.csv new file mode 100644 index 0000000000..cd490d98de --- /dev/null +++ b/benchmark/sdpa_benchmark_training/results/qwen3vl_vit/gb300/qwen3vl_vit_20260814_193600.csv @@ -0,0 +1,31 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,8836,8836,16,16,80,80,fwd,False,0.329,1215.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,8836,8836,16,16,80,80,fwd,False,0.258,1547.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,4927.000 +qwen3vl_vit,qwen3vl_vit,cudnn_oss,bfloat16,no_mask,1,8836,8836,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,cudnn_oss,fp8,no_mask,1,8836,8836,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,8836,8836,16,16,80,80,fwd,False,0.274,1457.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,15376,15376,16,16,80,80,fwd,False,0.975,1241.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,15376,15376,16,16,80,80,fwd,False,0.743,1629.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,4927.000 +qwen3vl_vit,qwen3vl_vit,cudnn_oss,bfloat16,no_mask,1,15376,15376,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,cudnn_oss,fp8,no_mask,1,15376,15376,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,15376,15376,16,16,80,80,fwd,False,0.811,1492.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,24336,24336,16,16,80,80,fwd,False,2.436,1245.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,24336,24336,16,16,80,80,fwd,False,1.777,1707.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,4927.000 +qwen3vl_vit,qwen3vl_vit,cudnn_oss,bfloat16,no_mask,1,24336,24336,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,cudnn_oss,fp8,no_mask,1,24336,24336,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,24336,24336,16,16,80,80,fwd,False,2.054,1476.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,35344,35344,16,16,80,80,fwd,False,5.154,1241.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,35344,35344,16,16,80,80,fwd,False,3.717,1721.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,4927.000 +qwen3vl_vit,qwen3vl_vit,cudnn_oss,bfloat16,no_mask,1,35344,35344,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,cudnn_oss,fp8,no_mask,1,35344,35344,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,35344,35344,16,16,80,80,fwd,False,4.291,1491.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,47376,47376,16,16,80,80,fwd,False,9.175,1253.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,47376,47376,16,16,80,80,fwd,False,6.723,1709.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,4927.000 +qwen3vl_vit,qwen3vl_vit,cudnn_oss,bfloat16,no_mask,1,47376,47376,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,cudnn_oss,fp8,no_mask,1,47376,47376,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,47376,47376,16,16,80,80,fwd,False,7.639,1504.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,bfloat16,no_mask,1,62500,62500,16,16,80,80,fwd,False,15.758,1269.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 +qwen3vl_vit,qwen3vl_vit,cudnn,fp8,no_mask,1,62500,62500,16,16,80,80,fwd,False,12.135,1648.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,4927.000 +qwen3vl_vit,qwen3vl_vit,cudnn_oss,bfloat16,no_mask,1,62500,62500,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,cudnn_oss,fp8,no_mask,1,62500,62500,16,16,80,80,fwd,False,inf,0.000,0.000,10,,False,True,UNSUPPORTED_CONFIG: no FROST OSS engine serves this fwd graph (shape/pass/feature not covered),,,, +qwen3vl_vit,qwen3vl_vit,flash_attention_4,bfloat16,no_mask,1,62500,62500,16,16,80,80,fwd,False,14.051,1423.000,0.000,10,,True,False,,NVIDIA B300 SXM6 AC,1.27.0,92500.000,2464.000 diff --git a/benchmark/sdpa_benchmark_training/results/qwen3vl_vit/gb300/qwen3vl_vit_no_mask.png b/benchmark/sdpa_benchmark_training/results/qwen3vl_vit/gb300/qwen3vl_vit_no_mask.png new file mode 100644 index 0000000000..0f753a0e68 Binary files /dev/null and b/benchmark/sdpa_benchmark_training/results/qwen3vl_vit/gb300/qwen3vl_vit_no_mask.png differ