diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052553.csv b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260821_052414.csv similarity index 55% rename from benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052553.csv rename to benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260821_052414.csv index e3388de98..46c60c7d4 100644 --- a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052553.csv +++ b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260821_052414.csv @@ -1,21 +1,21 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.23184,1333.841,325.645,4.0706,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,11.76816,1515.292,48.717,0.609,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.249216,1132.972,1168.438,14.6055,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.416448,1409.704,710.993,8.8874,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.748832,1567.957,408.007,5.1001,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.429984,1642.168,226.858,2.8357,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.069152,4.145,4145.348,51.8168,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.067456,0.537,268.548,3.3569,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.069088,0.804,268.143,3.3518,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.066208,1.075,268.871,3.3609,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.35376,874.145,213.414,2.6677,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,12.388768,1439.384,46.277,0.5785,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.131776,249.479,257.289,3.2161,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,1.132864,518.216,261.366,3.2671,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,1.162432,1010.069,262.836,3.2854,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.91392,1226.944,169.497,2.1187,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,1.103072,0.26,259.873,3.2484,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.129952,0.507,253.695,3.1712,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.126752,0.763,254.42,3.1803,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.13104,1.014,253.46,3.1682,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.228064,1355.925,331.036,4.138,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,11.7408,1518.823,48.83,0.6104,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.248672,1135.45,1170.994,14.6374,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.430656,1363.196,687.536,8.5942,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.762848,1539.149,400.511,5.0064,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.403392,1673.284,231.157,2.8895,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.069024,4.153,4153.035,51.9129,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.063456,0.539,269.559,3.3695,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.062912,0.809,269.701,3.3713,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.06192,1.08,269.957,3.3745,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.356224,868.099,211.938,2.6492,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,12.515648,1424.792,45.807,0.5726,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.128096,250.293,258.128,3.2266,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,1.132768,518.26,261.388,3.2673,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,1.167008,1006.108,261.805,3.2726,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.918848,1223.793,169.061,2.1133,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,1.106464,0.259,259.077,3.2385,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.129888,0.507,253.71,3.1714,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.129408,0.761,253.822,3.1728,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.127904,1.017,254.164,3.1771,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_context.png b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_context.png index f90fd2ac7..21fdd281f 100755 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_context.png and b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_context.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_generation.png b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_generation.png index 6efa7402b..64da1a396 100755 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_generation.png and b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_generation.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052426.csv b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260821_052506.csv similarity index 57% rename from benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052426.csv rename to benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260821_052506.csv index 1e4984e15..919036653 100644 --- a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052426.csv +++ b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260821_052506.csv @@ -1,21 +1,21 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.173344,1783.954,435.536,5.4442,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,9.20752,1936.7,62.265,0.7783,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.192832,1464.252,1510.088,18.8761,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.33296,1763.18,889.271,11.1159,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.592128,1982.91,515.984,6.4498,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.143488,2053.606,283.696,3.5462,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.070976,4.039,4038.817,50.4852,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.054912,0.543,271.742,3.3968,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.055968,0.814,271.474,3.3934,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.05536,1.086,271.635,3.3954,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.300768,1028.16,251.016,3.1377,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,10.121312,1761.847,56.644,0.708,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.972672,290.288,299.375,3.7422,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.965376,608.124,306.711,3.8339,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.997888,1176.622,306.175,3.8272,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.609376,1459.12,201.571,2.5196,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.938336,0.305,305.497,3.8187,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,0.987808,0.58,290.202,3.6275,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,0.987968,0.87,290.159,3.627,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,0.991264,1.157,289.199,3.615,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.177696,1740.262,424.869,5.3109,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,9.269696,1923.709,61.848,0.7731,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.191104,1477.492,1523.743,19.0468,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.334944,1752.736,884.003,11.05,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.595328,1972.252,513.211,6.4151,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.141472,2057.233,284.197,3.5525,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.069664,4.115,4114.881,51.436,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.053568,0.544,272.088,3.4011,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.053504,0.816,272.109,3.4014,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.054144,1.088,271.949,3.3994,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.30304,1020.452,249.134,3.1142,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,10.303392,1730.712,55.643,0.6955,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.968096,291.66,300.79,3.7599,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.97216,603.88,304.571,3.8071,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.998976,1175.34,305.842,3.823,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.61568,1453.427,200.784,2.5098,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.94464,0.303,303.459,3.7932,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.003008,0.572,285.804,3.5725,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,0.999968,0.86,286.677,3.5835,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,0.994976,1.152,288.12,3.6015,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_context.png b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_context.png index ca70ecfd3..b1f53d885 100755 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_context.png and b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_context.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_generation.png b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_generation.png index 0859d7a2c..e6c531642 100755 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_generation.png and b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_generation.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260820_060850.csv b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260821_061355.csv similarity index 62% rename from benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260820_060850.csv rename to benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260821_061355.csv index 644c5806a..559578fe1 100644 --- a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260820_060850.csv +++ b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260821_061355.csv @@ -1,11 +1,11 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.519808,594.907,145.241,3.0259,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,28.927776,616.439,19.819,0.4129,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.723808,390.096,402.307,8.3814,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,1.475136,397.976,200.722,4.1817,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,2.246816,522.578,135.983,2.833,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,3.931488,597.299,82.514,1.719,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.087136,3.29,3289.789,68.5373,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,0.727232,0.788,394.185,8.2122,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,0.727008,1.183,394.312,8.2148,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,0.726944,1.577,394.353,8.2157,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.506208,610.89,149.143,3.1071,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,28.922176,616.558,19.822,0.413,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.727712,388.003,400.149,8.3364,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,1.46864,399.736,201.609,4.2002,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,2.204032,532.722,138.623,2.888,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,3.974592,590.821,81.619,1.7004,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.08704,3.293,3293.418,68.6129,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,0.732576,0.783,391.309,8.1523,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,0.732736,1.174,391.23,8.1506,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,0.732608,1.565,391.305,8.1522,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_context.png b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_context.png index 6ecca2374..2b46fad5b 100755 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_context.png and b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_context.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_generation.png b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_generation.png index bbebc7d6b..5bf54c8ac 100755 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_generation.png and b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_generation.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260820_064536.csv b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260821_062423.csv similarity index 55% rename from benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260820_064536.csv rename to benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260821_062423.csv index c84d1fb0b..01d981874 100644 --- a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260820_064536.csv +++ b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260821_062423.csv @@ -1,21 +1,21 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,1.14048,271.147,66.198,3.6941,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,49.426144,360.785,11.599,0.6473,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.984544,142.277,146.731,8.1881,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,2.0136,291.552,147.046,8.2057,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,4.011264,292.71,76.168,4.2504,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,7.968928,294.679,40.709,2.2717,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.205024,1.398,1398.173,78.0231,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.141312,0.502,251.17,14.0162,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.144416,0.751,250.493,13.9784,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.142432,1.004,250.932,14.0029,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,1.076832,287.174,70.111,3.9124,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,45.51936,391.75,12.595,0.7028,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.413088,199.814,206.069,11.4994,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,2.456096,239.025,120.554,6.7273,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,3.97616,295.294,76.84,4.288,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,7.509984,312.687,43.196,2.4105,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,1.381376,0.208,207.517,11.5802,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.392928,0.412,205.799,11.4843,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.42944,0.602,200.546,11.1912,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.428928,0.802,200.621,11.1954,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,1.155296,267.67,65.349,3.6467,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,49.428959,360.764,11.599,0.6472,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.983552,142.348,146.804,8.1922,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,2.013568,291.556,147.048,8.2058,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,4.01008,292.796,76.19,4.2517,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,7.973408,294.513,40.686,2.2704,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.205728,1.393,1393.389,77.7561,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.1888,0.482,241.137,13.4563,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.18896,0.723,241.108,13.4547,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.190432,0.963,240.814,13.4383,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,1.150144,268.869,65.642,3.663,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,45.354336,393.175,12.641,0.7054,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.483136,190.377,196.336,10.9563,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,2.465568,238.107,120.091,6.7015,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,3.979392,295.054,76.778,4.2845,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,7.533504,311.711,43.061,2.403,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,1.437632,0.199,199.397,11.127,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.44752,0.396,198.038,11.0512,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.449504,0.593,197.77,11.0363,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.450112,0.791,197.69,11.0318,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], diff --git a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_context.png b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_context.png index 26b220e45..e995be0c5 100755 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_context.png and b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_context.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_generation.png b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_generation.png index e4c6162e0..9ce6a9ee8 100755 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_generation.png and b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_generation.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260819_055704.csv b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260819_055704.csv deleted file mode 100644 index f1d879a24..000000000 --- a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260819_055704.csv +++ /dev/null @@ -1,225 +0,0 @@ -config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.558912,491.809,484.034,6.0504,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,8.597472,511.551,125.866,1.5733,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,8.625952,509.862,15.56,0.1945,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,17.451103,504.042,11.537,0.1442,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,16.525024,532.289,12.183,0.1523,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,33.608894,523.438,7.987,0.0998,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.39824,43.139,337.356,4.217,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,38.641216,56.909,445.034,5.5629,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.616,3.237,12.668,0.1583,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1276.078979,3.447,13.489,0.1686,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.620256,4.853,12.675,0.1584,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1274.32666,5.177,13.521,0.169,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.604288,6.48,12.706,0.1588,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1273.384644,6.908,13.544,0.1693,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.599232,458.717,451.466,5.6433,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.922912,1121.118,275.849,3.4481,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.770816,1587.275,48.44,0.6055,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.774976,1523.139,34.862,0.4358,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.986528,1469.315,33.63,0.4204,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,11.871968,1481.826,22.611,0.2826,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.644672,6.496,50.8,0.635,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,315.228149,6.976,54.553,0.6819,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.688896,12.778,50.013,0.6252,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,317.050995,13.872,54.292,0.6787,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.707712,19.034,49.714,0.6214,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,317.656586,20.768,54.242,0.678,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.684384,25.6,50.195,0.6274,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,316.928223,27.754,54.419,0.6802,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.30528,450.206,446.524,5.5816,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,4.062528,541.294,134.217,1.6777,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,4.063168,541.209,24.775,0.3097,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,8.559328,513.831,19.601,0.245,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,8.464448,519.59,15.857,0.1982,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,17.563393,500.82,11.463,0.1433,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.22912,37.491,586.083,7.326,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,22.42016,49.041,766.643,9.583,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.687584,1.607,12.571,0.1571,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,621.229736,3.54,27.682,0.346,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.66912,2.415,12.598,0.1575,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,620.331604,5.317,27.735,0.3467,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.662048,3.223,12.613,0.1577,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,619.824951,7.096,27.771,0.3471,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.329152,417.555,414.14,5.1767,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.987712,1106.309,274.315,3.4289,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.442144,1524.829,69.801,0.8725,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.768832,1588.412,60.593,0.7574,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.775936,1584.347,48.35,0.6044,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.79872,1516.903,34.719,0.434,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.356448,6.333,98.996,1.2375,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,158.135681,6.953,108.693,1.3587,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.39408,12.323,96.371,1.2046,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,157.529984,13.959,109.164,1.3646,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.394784,18.476,96.369,1.2046,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,158.188477,20.852,108.763,1.3595,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.398912,24.562,96.132,1.2016,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,157.492157,27.925,109.297,1.3662,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.173408,396.288,399.094,4.9887,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.816,605.458,152.436,1.9055,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,2.07504,529.875,40.426,0.5053,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,4.26064,516.125,35.439,0.443,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,4.023232,546.581,25.021,0.3128,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,8.59312,511.81,19.524,0.2441,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.139264,30.84,964.0,12.05,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.858624,46.359,1449.077,18.1135,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.618624,0.809,12.646,0.1581,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,312.820984,3.515,54.946,0.6868,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.611968,1.214,12.657,0.1582,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,312.224365,5.282,55.064,0.6883,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.60352,1.62,12.67,0.1584,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,312.292053,7.042,55.066,0.6883,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.199456,344.535,346.974,4.3372,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.040704,1056.508,265.997,3.325,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.77504,1418.652,108.235,1.3529,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.411744,1557.664,106.956,1.337,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.43664,1530.671,70.069,0.8759,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.849216,1543.599,58.884,0.736,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.351328,3.178,99.347,1.2418,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,79.039261,6.955,217.412,2.7176,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.392512,6.169,96.432,1.2054,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,79.475037,13.835,216.272,2.7034,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.399232,9.209,95.993,1.1999,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,79.385986,20.775,216.568,2.7071,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.39408,12.323,96.371,1.2046,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,79.229568,27.755,217.048,2.7131,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.09952,345.255,358.235,4.4779,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.9568,574.578,149.045,1.8631,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,1.831904,300.101,41.213,0.5152,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,3.721216,295.471,38.323,0.479,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,1.911328,575.261,43.889,0.5486,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,4.176992,526.461,36.149,0.4519,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.139776,15.364,960.352,12.0044,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,11.858816,23.179,1448.877,18.111,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.611328,0.405,12.652,0.1581,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,158.748383,3.463,108.247,1.3531,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.603808,0.608,12.662,0.1583,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,158.647293,5.198,108.329,1.3541,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.599072,0.81,12.669,0.1584,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,158.535614,6.935,108.419,1.3552,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.157216,218.551,226.768,2.8346,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.55712,986.782,255.971,3.1996,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,0.75792,725.348,99.611,1.2451,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,1.398272,786.336,101.988,1.2748,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,0.776512,1415.962,108.029,1.3504,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,1.419552,1549.097,106.368,1.3296,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,1.351008,1.59,99.358,1.242,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,39.710274,6.922,432.683,5.4085,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,1.392256,3.085,96.427,1.2053,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,40.053185,13.726,429.031,5.3629,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,1.39376,4.622,96.334,1.2042,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,39.690529,20.777,433.004,5.4126,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,1.394464,6.16,96.297,1.2037,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,39.829056,27.606,431.551,5.3944,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.077536,510.197,500.186,6.2523,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,17.346464,507.083,124.283,1.5535,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,16.592833,530.114,12.133,0.1517,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,33.557472,524.241,7.999,0.1,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,33.469727,525.615,10.025,0.1253,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,67.924416,517.993,5.928,0.0741,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.715904,47.995,187.846,2.3481,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,73.038017,60.216,235.678,2.946,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,23.764032,2.892,5.67,0.0709,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2690.69043,3.269,6.41,0.0801,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,23.748192,4.341,5.685,0.0711,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2688.665527,4.907,6.427,0.0803,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,23.740641,5.789,5.698,0.0712,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2684.7771,6.553,6.449,0.0806,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.12256,489.734,480.124,6.0016,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,7.8728,1117.276,273.838,3.423,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,5.916128,1486.799,34.03,0.4254,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,11.582976,1518.797,23.175,0.2897,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,11.523968,1526.574,29.117,0.364,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,22.773504,1544.97,17.681,0.221,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,5.222304,6.579,25.751,0.3219,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,634.460999,6.932,27.131,0.3391,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,5.277984,13.02,25.529,0.3191,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,637.260986,13.803,27.064,0.3383,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,5.26512,19.578,25.641,0.3205,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,638.134094,20.676,27.08,0.3385,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,5.267648,26.091,25.679,0.321,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,639.128906,27.525,27.09,0.3386,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.552288,497.708,489.84,6.123,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,8.7088,505.012,124.257,1.5532,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,8.561312,513.712,15.677,0.196,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,17.627808,498.99,11.421,0.1428,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,16.567583,530.922,12.152,0.1519,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,33.467999,525.642,8.021,0.1003,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.394112,43.591,340.89,4.2611,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,38.611904,56.952,445.372,5.5671,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.619808,3.235,12.663,0.1583,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1275.861328,3.447,13.492,0.1686,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.612736,4.856,12.684,0.1585,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1274.179932,5.178,13.523,0.169,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.602176,6.482,12.709,0.1589,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1272.534058,6.912,13.553,0.1694,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.597952,459.699,452.432,5.6554,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.884576,1132.182,278.571,3.4821,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.898432,1517.388,46.307,0.5788,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.978336,1471.328,33.676,0.421,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.848192,1504.07,34.425,0.4303,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,11.8256,1487.636,22.7,0.2837,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.64464,6.496,50.8,0.635,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,315.169983,6.977,54.563,0.682,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.686272,12.791,50.062,0.6258,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,316.288727,13.905,54.423,0.6803,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.6832,19.208,50.168,0.6271,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,316.526489,20.842,54.435,0.6804,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.689248,25.553,50.104,0.6263,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,315.72699,27.86,54.626,0.6828,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.304608,451.199,447.509,5.5939,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,4.179936,526.09,130.447,1.6306,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,4.056512,542.097,24.815,0.3102,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,8.563264,513.595,19.592,0.2449,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,8.5776,512.736,15.647,0.1956,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,17.283232,508.938,11.649,0.1456,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.229696,37.397,584.613,7.3077,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,22.410273,49.063,766.981,9.5873,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.686592,1.608,12.572,0.1571,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,620.949585,3.541,27.694,0.3462,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.664992,2.416,12.603,0.1575,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,620.364685,5.317,27.734,0.3467,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.65712,3.224,12.619,0.1577,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,619.917969,7.095,27.767,0.3471,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.330112,416.34,412.935,5.1617,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.982528,1109.202,275.032,3.4379,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.455232,1511.115,69.173,0.8647,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.816096,1561.753,59.576,0.7447,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.906784,1513.028,46.174,0.5772,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.902912,1490.128,34.106,0.4263,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.355232,6.338,99.085,1.2386,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,156.882751,7.008,109.561,1.3695,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.40176,12.256,95.843,1.198,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,157.391129,13.972,109.261,1.3658,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.399552,18.413,96.041,1.2005,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,157.935043,20.885,108.937,1.3617,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.397568,24.585,96.224,1.2028,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,157.446686,27.934,109.329,1.3666,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.172224,399.012,401.837,5.023,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.823584,602.94,151.802,1.8975,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.954944,562.426,42.91,0.5364,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,4.111264,534.878,36.727,0.4591,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,4.007872,548.676,25.116,0.314,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,8.630624,509.586,19.439,0.243,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.139232,30.848,964.222,12.0528,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.854432,46.376,1449.59,18.1199,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.616064,0.809,12.649,0.1581,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,312.44635,3.519,55.012,0.6876,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.60496,1.215,12.665,0.1583,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,312.206543,5.283,55.068,0.6883,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.599648,1.621,12.675,0.1584,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,312.025665,7.048,55.113,0.6889,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.19984,343.872,346.307,4.3288,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.039616,1057.613,266.275,3.3284,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.775648,1417.539,108.15,1.3519,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.41984,1548.782,106.346,1.3293,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.453248,1513.178,69.268,0.8658,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.861408,1537.022,58.633,0.7329,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.350816,3.18,99.385,1.2423,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,79.123489,6.948,217.18,2.7148,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.397664,6.146,96.077,1.201,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,79.271965,13.87,216.826,2.7103,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.395712,9.232,96.235,1.2029,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,79.486145,20.749,216.295,2.7037,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.395808,12.308,96.252,1.2031,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,79.293953,27.733,216.872,2.7109,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260821_055608.csv b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260821_055608.csv new file mode 100644 index 000000000..f36a54b9a --- /dev/null +++ b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260821_055608.csv @@ -0,0 +1,225 @@ +config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.557984,492.627,484.839,6.0605,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,8.694336,505.852,124.464,1.5558,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,8.65872,507.933,15.501,0.1938,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,17.440001,504.363,11.544,0.1443,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,16.615295,529.397,12.117,0.1515,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,33.622112,523.233,7.984,0.0998,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.390528,43.991,344.018,4.3002,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,38.748352,56.751,443.803,5.5475,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.603168,3.241,12.683,0.1585,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1277.24939,3.443,13.477,0.1685,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.59392,4.865,12.706,0.1588,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1275.855835,5.171,13.505,0.1688,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.584576,6.492,12.73,0.1591,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1274.075562,6.904,13.537,0.1692,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.60624,453.414,446.247,5.5781,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.928064,1119.647,275.487,3.4436,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.773472,1585.755,48.393,0.6049,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.92432,1484.743,33.983,0.4248,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.778016,1522.338,34.844,0.4355,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,11.717056,1501.417,22.91,0.2864,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.653856,6.474,50.624,0.6328,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,316.020844,6.958,54.416,0.6802,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.699776,12.727,49.811,0.6226,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,316.746765,13.885,54.344,0.6793,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.705152,19.052,49.761,0.622,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,317.426483,20.783,54.281,0.6785,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.6944,25.505,50.008,0.6251,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,317.769409,27.681,54.275,0.6784,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.3032,453.295,449.587,5.6198,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,4.218016,521.341,129.269,1.6159,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,4.20656,522.76,23.93,0.2991,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,8.751776,502.532,19.17,0.2396,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,8.62224,510.082,15.566,0.1946,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,17.2488,509.954,11.672,0.1459,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.2288,37.543,586.902,7.3363,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,22.484415,48.901,764.452,9.5557,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.665024,1.611,12.597,0.1575,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,621.126587,3.54,27.686,0.3461,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.644288,2.421,12.628,0.1578,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,620.593811,5.315,27.724,0.3465,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.636192,3.23,12.644,0.158,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,620.219116,7.091,27.754,0.3469,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.337824,406.836,403.509,5.0439,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.997184,1101.062,273.014,3.4127,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.458816,1507.403,69.003,0.8625,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.889888,1521.874,58.055,0.7257,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.786976,1578.071,48.159,0.602,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.775872,1522.903,34.856,0.4357,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.356672,6.332,98.98,1.2372,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,157.800156,6.968,108.924,1.3616,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.40272,12.248,95.777,1.1972,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,158.058884,13.913,108.799,1.36,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.399968,18.407,96.012,1.2002,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,158.142212,20.858,108.795,1.3599,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.40176,24.512,95.936,1.1992,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,158.283676,27.786,108.75,1.3594,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.166976,411.553,414.467,5.1808,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.858368,591.654,148.961,1.862,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.926752,570.656,43.538,0.5442,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,4.170688,527.257,36.204,0.4525,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,4.217792,521.368,23.866,0.2983,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,8.628672,509.701,19.444,0.243,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.133952,32.063,1002.228,12.5279,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.860512,46.352,1448.847,18.1106,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.591008,0.811,12.679,0.1585,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,313.474548,3.507,54.831,0.6854,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.586816,1.217,12.687,0.1586,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,313.686859,5.258,54.808,0.6851,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.582496,1.623,12.695,0.1587,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,312.976715,7.026,54.945,0.6868,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.20144,341.141,343.556,4.2945,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.046464,1050.692,264.533,3.3067,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.784416,1401.695,106.941,1.3368,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.431072,1536.627,105.512,1.3189,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.469984,1495.95,68.479,0.856,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.806688,1566.988,59.776,0.7472,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.352512,3.176,99.26,1.2408,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,79.20784,6.941,216.949,2.7119,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.399232,6.139,95.969,1.1996,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,79.784355,13.781,215.434,2.6929,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.397088,9.223,96.14,1.2017,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,79.257347,20.809,216.919,2.7115,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.395872,12.308,96.247,1.2031,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,79.402237,27.695,216.576,2.7072,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.098176,349.981,363.139,4.5392,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.942464,583.318,151.312,1.8914,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,1.823424,301.496,41.404,0.5176,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,3.722496,295.369,38.309,0.4789,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,1.897632,579.412,44.206,0.5526,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,3.88224,566.432,38.894,0.4862,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.136192,15.768,985.624,12.3203,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,11.846656,23.203,1450.364,18.1296,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.591392,0.406,12.675,0.1584,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,159.90387,3.438,107.465,1.3433,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.583456,0.609,12.686,0.1586,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,159.627716,5.166,107.664,1.3458,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.579968,0.812,12.692,0.1587,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,159.643524,6.887,107.666,1.3458,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.157184,218.596,226.814,2.8352,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.563392,975.796,253.121,3.164,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,0.758528,724.767,99.532,1.2441,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,1.405568,782.254,101.458,1.2682,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,0.78272,1404.732,107.173,1.3397,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,1.424448,1543.772,106.002,1.325,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,1.351456,1.589,99.326,1.2416,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,39.78912,6.908,431.826,5.3978,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,1.39392,3.081,96.311,1.2039,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,40.130657,13.699,428.203,5.3525,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,1.395104,4.618,96.241,1.203,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,39.54208,20.855,434.63,5.4329,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,1.3952,6.157,96.247,1.2031,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,40.475552,27.165,424.658,5.3082,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.077312,510.303,500.29,6.2536,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,17.488352,502.969,123.275,1.5409,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,16.611391,529.522,12.12,0.1515,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,33.601727,523.55,7.989,0.0999,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,33.615425,523.337,9.982,0.1248,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,67.463326,521.533,5.968,0.0746,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.718784,47.803,187.094,2.3387,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,73.27562,60.021,234.913,2.9364,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,23.765024,2.892,5.67,0.0709,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2692.979004,3.266,6.404,0.0801,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,23.752192,4.34,5.684,0.071,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2691.038086,4.903,6.422,0.0803,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,23.739391,5.789,5.698,0.0712,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2687.777832,6.545,6.442,0.0805,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.136,483.94,474.444,5.9305,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,7.881344,1116.065,273.541,3.4193,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,6.084224,1445.721,33.09,0.4136,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,11.558304,1522.039,23.224,0.2903,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,11.577152,1519.561,28.983,0.3623,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,23.04784,1526.58,17.47,0.2184,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,5.2376,6.56,25.676,0.3209,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,633.03418,6.948,27.192,0.3399,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,5.287008,12.998,25.485,0.3186,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,634.68866,13.859,27.174,0.3397,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,5.287392,19.495,25.533,0.3192,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,635.392212,20.765,27.197,0.34,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,5.284736,26.007,25.596,0.3199,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,635.839355,27.668,27.23,0.3404,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.558336,492.316,484.534,6.0567,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,8.72208,504.243,124.068,1.5508,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,8.591936,511.881,15.621,0.1953,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,17.369984,506.396,11.59,0.1449,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,16.489504,533.436,12.209,0.1526,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,33.909119,518.804,7.916,0.099,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.392096,43.815,342.643,4.283,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,38.729313,56.779,444.021,5.5503,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.603552,3.24,12.683,0.1585,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1276.868896,3.444,13.481,0.1685,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.596704,4.864,12.703,0.1588,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1275.686401,5.171,13.507,0.1688,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.58848,6.49,12.725,0.1591,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1273.779663,6.906,13.54,0.1693,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.6072,452.697,445.541,5.5693,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.922944,1121.109,275.847,3.4481,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.82624,1556.148,47.49,0.5936,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.712224,1539.872,35.245,0.4406,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.903328,1490.023,34.104,0.4263,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,11.562944,1521.428,23.215,0.2902,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.658048,6.463,50.544,0.6318,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,314.070068,7.002,54.754,0.6844,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.698176,12.734,49.841,0.623,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,315.939331,13.921,54.483,0.681,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.698624,19.098,49.881,0.6235,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,316.046417,20.874,54.518,0.6815,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.700448,25.447,49.896,0.6237,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,315.332977,27.895,54.694,0.6837,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.302528,454.302,450.586,5.6323,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,3.967296,554.288,137.439,1.718,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,3.937792,558.441,25.563,0.3195,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,8.52448,515.931,19.681,0.246,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,8.529856,515.606,15.735,0.1967,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,16.994305,517.591,11.847,0.1481,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.22784,37.702,589.375,7.3672,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,22.502048,48.863,763.853,9.5482,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.660992,1.611,12.602,0.1575,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,621.425476,3.539,27.673,0.3459,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.643776,2.421,12.628,0.1579,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,620.661865,5.315,27.72,0.3465,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.639136,3.23,12.64,0.158,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,619.977966,7.094,27.765,0.3471,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.33696,407.879,404.543,5.0568,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,2.023968,1086.491,269.401,3.3675,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.45392,1512.479,69.236,0.8654,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.808864,1565.774,59.73,0.7466,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.92528,1503.462,45.882,0.5735,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.77024,1524.39,34.891,0.4361,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.355392,6.338,99.073,1.2384,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,157.235046,6.993,109.316,1.3664,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.403904,12.237,95.697,1.1962,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,157.833466,13.933,108.954,1.3619,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.401472,18.388,95.909,1.1989,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,158.05751,20.869,108.853,1.3607,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.401056,24.524,95.985,1.1998,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,158.07225,27.823,108.896,1.3612,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.171232,401.324,404.165,5.0521,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.81456,605.938,152.557,1.907,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.941632,566.282,43.204,0.54,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,4.189408,524.901,36.042,0.4505,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,4.081376,538.795,24.664,0.3083,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,8.68928,506.146,19.308,0.2413,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.136992,31.352,979.988,12.2498,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.863712,46.339,1448.456,18.1057,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.596544,0.811,12.672,0.1584,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,314.019379,3.501,54.736,0.6842,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.588352,1.217,12.685,0.1586,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,312.966187,5.27,54.934,0.6867,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.583776,1.623,12.694,0.1587,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,312.786041,7.03,54.979,0.6872,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.20352,337.655,340.045,4.2506,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.052768,1044.401,262.949,3.2869,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.78272,1404.732,107.173,1.3397,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.433728,1533.78,105.316,1.3165,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.473344,1492.539,68.323,0.854,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.98672,1472.534,56.173,0.7022,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.353376,3.174,99.197,1.24,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,79.042816,6.955,217.402,2.7175,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.400256,6.135,95.899,1.1987,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,79.460991,13.837,216.311,2.7039,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.39888,9.211,96.017,1.2002,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,79.266014,20.807,216.896,2.7112,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.396672,12.301,96.192,1.2024,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,79.521347,27.653,216.252,2.7031,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_context.png b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_context.png index 64caef7b5..6c202a42f 100755 Binary files a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_context.png and b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_context.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_generation.png b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_generation.png index 91897e5f3..8a3cfe784 100755 Binary files a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_generation.png and b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_generation.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260819_055559.csv b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260819_055559.csv deleted file mode 100644 index 9dd802d55..000000000 --- a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260819_055559.csv +++ /dev/null @@ -1,225 +0,0 @@ -config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.488576,562.61,553.717,6.9215,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,7.103904,619.103,152.329,1.9041,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,7.417568,592.923,18.095,0.2262,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,15.072256,583.595,13.357,0.167,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,13.5552,648.909,14.852,0.1857,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,27.301121,644.376,9.832,0.1229,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.385888,44.52,348.155,4.3519,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,37.888962,58.039,453.87,5.6734,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.459424,3.285,12.857,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1244.87439,3.533,13.827,0.1728,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.438176,4.938,12.896,0.1612,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1243.452271,5.305,13.857,0.1732,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.428064,6.59,12.921,0.1615,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1242.27124,7.081,13.883,0.1735,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.577312,476.134,468.607,5.8576,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.819808,1151.379,283.294,3.5412,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.727776,1612.319,49.204,0.6151,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.385152,1633.397,37.385,0.4673,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.405664,1627.199,37.244,0.4655,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,10.721632,1640.812,25.037,0.313,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.632832,6.525,51.028,0.6379,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,299.032074,7.354,57.508,0.7188,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.67616,12.839,50.251,0.6281,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,299.847076,14.668,57.407,0.7176,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.681056,19.224,50.208,0.6276,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,299.682098,22.014,57.495,0.7187,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.680128,25.64,50.274,0.6284,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,299.921387,29.328,57.505,0.7188,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.26576,517.154,512.925,6.4116,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,3.364608,653.575,162.057,2.0257,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,3.615424,608.234,27.843,0.348,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,7.449792,590.358,22.52,0.2815,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,7.50688,585.869,17.879,0.2235,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,14.996512,586.543,13.425,0.1678,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.223776,38.386,600.079,7.501,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,21.877056,50.259,785.675,9.8209,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.509888,1.635,12.783,0.1598,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,606.695129,3.625,28.345,0.3543,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.486528,2.457,12.818,0.1602,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,605.82782,5.445,28.399,0.355,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.48592,3.277,12.825,0.1603,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,605.638367,7.262,28.422,0.3553,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.320864,428.34,424.837,5.3105,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.945088,1130.552,280.326,3.5041,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.419168,1549.516,70.931,0.8866,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.710752,1622.445,61.891,0.7736,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.728128,1612.112,49.198,0.615,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.364128,1639.799,37.532,0.4692,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.347712,6.374,99.638,1.2455,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,149.034363,7.378,115.331,1.4416,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.39168,12.345,96.537,1.2067,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,149.31897,14.727,115.167,1.4396,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.392512,18.506,96.527,1.2066,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,149.243332,22.102,115.282,1.441,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.389632,24.726,96.774,1.2097,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,149.055969,29.506,115.483,1.4435,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.148064,464.12,467.406,5.8426,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.652512,665.358,167.517,2.094,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.783264,616.573,47.041,0.588,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,3.569184,616.114,42.305,0.5288,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,3.564608,616.905,28.24,0.353,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,7.464608,589.187,22.476,0.2809,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.13456,31.919,997.7,12.4712,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.398464,48.231,1507.577,18.8447,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.448288,0.822,12.852,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,307.523682,3.575,55.892,0.6987,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.4248,1.236,12.884,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,307.361572,5.366,55.936,0.6992,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.418816,1.649,12.895,0.1612,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,306.934021,7.164,56.027,0.7003,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.19344,355.25,357.765,4.4721,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.006144,1092.797,275.134,3.4392,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.755648,1455.058,111.012,1.3877,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.40288,1567.506,107.632,1.3454,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.41456,1554.563,71.162,0.8895,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.696448,1631.052,62.22,0.7777,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.346496,3.19,99.704,1.2463,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,74.538849,7.375,230.538,2.8817,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.388384,6.187,96.719,1.209,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,74.661728,14.727,230.215,2.8777,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.38896,9.277,96.703,1.2088,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,74.657249,22.091,230.285,2.8786,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.391392,12.347,96.557,1.207,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,74.770912,29.41,229.991,2.8749,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.090048,381.571,395.918,4.949,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.837376,656.522,170.301,2.1288,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,1.701344,323.13,44.375,0.5547,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,3.467872,317.057,41.122,0.514,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,1.759584,624.87,47.674,0.5959,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,3.535328,622.014,42.71,0.5339,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.134784,15.933,995.92,12.449,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,11.395232,24.122,1507.821,18.8478,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.440544,0.411,12.859,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,156.61264,3.51,109.723,1.3715,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.416384,0.618,12.89,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,156.401886,5.273,109.885,1.3736,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.407744,0.825,12.902,0.1613,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,156.339111,7.033,109.942,1.3743,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.154464,222.445,230.808,2.8851,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.534336,1028.858,266.885,3.3361,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,0.752928,730.157,100.272,1.2534,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,1.387008,792.722,102.816,1.2852,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,0.758848,1448.922,110.544,1.3818,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,1.396896,1574.221,108.093,1.3512,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,1.344416,1.597,99.846,1.2481,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,37.341698,7.361,460.128,5.7516,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,1.38512,3.101,96.923,1.2115,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,37.496288,14.662,458.287,5.7286,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,1.382432,4.66,97.124,1.214,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,37.477409,22.003,458.574,5.7322,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,1.38272,6.212,97.115,1.2139,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,37.555614,29.277,457.675,5.7209,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,0.953632,576.486,565.174,7.0647,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,14.507648,606.307,148.602,1.8575,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,13.640192,644.866,14.76,0.1845,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,27.584448,637.757,9.731,0.1216,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,28.125216,625.495,11.93,0.1491,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,57.374016,613.246,7.018,0.0877,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.699392,49.128,192.281,2.4035,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,70.82045,62.101,243.057,3.0382,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,23.310305,2.948,5.78,0.0723,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2616.91333,3.361,6.591,0.0824,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,23.283232,4.427,5.798,0.0725,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2615.435791,5.045,6.607,0.0826,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,23.283712,5.903,5.809,0.0726,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2614.49585,6.729,6.622,0.0828,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.08848,505.067,495.157,6.1895,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,7.569824,1161.994,284.798,3.56,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,5.454464,1612.641,36.91,0.4614,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,10.732416,1639.164,25.012,0.3126,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,10.346976,1700.225,32.429,0.4054,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,20.5056,1715.842,19.636,0.2455,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,5.19376,6.616,25.893,0.3237,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,597.522461,7.36,28.808,0.3601,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,5.245856,13.1,25.685,0.3211,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,601.865784,14.615,28.656,0.3582,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,5.24736,19.644,25.728,0.3216,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,601.77832,21.925,28.716,0.3589,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,5.23104,26.274,25.858,0.3232,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,601.027893,29.27,28.807,0.3601,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.497376,552.656,543.92,6.799,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,6.937504,633.952,155.983,1.9498,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,7.44976,590.361,18.016,0.2252,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,15.040736,584.818,13.385,0.1673,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,13.556192,648.862,14.851,0.1856,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,28.117376,625.67,9.547,0.1193,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.382528,44.911,351.213,4.3902,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,37.918465,57.993,453.516,5.669,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.455232,3.286,12.862,0.1608,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1244.385986,3.534,13.833,0.1729,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.433664,4.94,12.902,0.1613,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1243.915771,5.303,13.852,0.1731,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.425824,6.591,12.924,0.1615,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1242.8479,7.077,13.877,0.1735,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.579328,474.477,466.977,5.8372,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.813472,1153.292,283.765,3.5471,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.728192,1612.074,49.197,0.615,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.341344,1646.794,37.692,0.4712,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.377536,1635.711,37.438,0.468,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,10.72144,1640.842,25.037,0.313,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.636224,6.517,50.963,0.637,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,298.720856,7.361,57.568,0.7196,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.676192,12.839,50.25,0.6281,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,299.616241,14.679,57.452,0.7181,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.68112,19.223,50.207,0.6276,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,299.665924,22.015,57.498,0.7187,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.674528,25.694,50.38,0.6297,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,299.657471,29.354,57.556,0.7194,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.265376,517.903,513.667,6.4208,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,3.279744,670.486,166.251,2.0781,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,3.607808,609.518,27.902,0.3488,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,7.384224,595.6,22.72,0.284,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,7.352224,598.193,18.255,0.2282,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,15.003104,586.285,13.419,0.1677,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.224672,38.233,597.686,7.4711,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,21.868769,50.278,785.973,9.8247,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.506848,1.635,12.787,0.1598,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,606.50531,3.626,28.354,0.3544,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.484512,2.458,12.82,0.1603,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,605.378906,5.449,28.42,0.3553,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.481568,3.278,12.83,0.1604,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,605.303101,7.266,28.438,0.3555,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.319904,429.626,426.112,5.3264,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.942048,1132.322,280.765,3.5096,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.42,1548.608,70.89,0.8861,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.702752,1627.247,62.075,0.7759,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.719296,1617.347,49.358,0.617,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.367328,1638.822,37.51,0.4689,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.348064,6.372,99.612,1.2451,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,148.894684,7.384,115.439,1.443,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.39056,12.355,96.615,1.2077,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,149.10672,14.748,115.331,1.4416,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.394048,18.486,96.42,1.2053,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,149.073883,22.127,115.413,1.4427,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.390816,24.705,96.691,1.2086,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,149.110535,29.495,115.441,1.443,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.146688,468.474,471.791,5.8974,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.649472,666.584,167.826,2.0978,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.783456,616.506,47.036,0.5879,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,3.577632,614.659,42.205,0.5276,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,3.554816,618.604,28.317,0.354,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,7.444416,590.785,22.537,0.2817,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.135072,31.798,993.918,12.424,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.408544,48.188,1506.245,18.8281,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.441952,0.823,12.86,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,307.884766,3.571,55.827,0.6978,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.421856,1.236,12.888,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,307.282318,5.367,55.95,0.6994,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.415104,1.65,12.899,0.1612,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,306.473572,7.175,56.111,0.7014,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.194656,353.03,355.53,4.4441,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.007424,1091.409,274.784,3.4348,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.753856,1458.517,111.276,1.3909,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.400096,1570.623,107.846,1.3481,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.41888,1549.83,70.946,0.8868,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.699616,1629.138,62.147,0.7768,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.348288,3.185,99.571,1.2446,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,74.42128,7.387,230.903,2.8863,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.38672,6.194,96.835,1.2104,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,74.65123,14.729,230.247,2.8781,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.392992,9.25,96.423,1.2053,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,74.656067,22.092,230.289,2.8786,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.387904,12.378,96.8,1.21,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,74.573441,29.488,230.6,2.8825,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260821_055741.csv b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260821_055741.csv new file mode 100644 index 000000000..eb5518c62 --- /dev/null +++ b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260821_055741.csv @@ -0,0 +1,225 @@ +config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.490304,560.628,551.765,6.8971,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,7.268416,605.09,148.881,1.861,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,7.476896,588.218,17.951,0.2244,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,15.155552,580.388,13.284,0.1661,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,13.834528,635.807,14.552,0.1819,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,27.687616,635.381,9.695,0.1212,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.382368,44.93,351.36,4.392,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,37.921921,57.988,453.475,5.6684,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.452576,3.287,12.866,0.1608,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1246.775635,3.528,13.806,0.1726,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.429056,4.942,12.907,0.1613,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1245.844727,5.295,13.83,0.1729,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.418464,6.596,12.933,0.1617,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1244.886108,7.066,13.854,0.1732,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.585024,469.857,462.43,5.7804,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.842688,1144.523,281.608,3.5201,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.737504,1606.59,49.029,0.6129,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.4464,1615.029,36.965,0.4621,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.493856,1601.078,36.646,0.4581,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,10.90112,1613.796,24.625,0.3078,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.63536,6.519,50.979,0.6372,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,300.487762,7.318,57.229,0.7154,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.67424,12.848,50.287,0.6286,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,301.338531,14.595,57.123,0.714,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.682848,19.211,50.175,0.6272,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,301.404785,21.888,57.166,0.7146,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.682784,25.615,50.225,0.6278,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,301.486694,29.176,57.206,0.7151,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.262592,523.394,519.113,6.4889,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,3.4968,628.867,155.931,1.9491,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,3.616928,607.981,27.831,0.3479,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,7.511104,585.539,22.337,0.2792,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,7.496928,586.646,17.903,0.2238,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,15.076704,583.423,13.353,0.1669,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.22096,38.876,607.727,7.5966,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,21.930529,50.136,783.759,9.797,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.507008,1.635,12.787,0.1598,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,607.627441,3.619,28.301,0.3538,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.489984,2.457,12.814,0.1602,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,606.762634,5.436,28.355,0.3544,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.48672,3.277,12.824,0.1603,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,606.380249,7.253,28.387,0.3548,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.325792,421.861,418.411,5.2301,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.964256,1119.52,277.591,3.4699,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.419264,1549.411,70.926,0.8866,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.714176,1620.398,61.813,0.7727,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.73472,1608.226,49.079,0.6135,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.400832,1628.655,37.277,0.466,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.349472,6.365,99.508,1.2438,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,149.613724,7.349,114.884,1.4361,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.393056,12.333,96.442,1.2055,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,149.939713,14.666,114.69,1.4336,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.390688,18.53,96.653,1.2082,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,150.015579,21.988,114.688,1.4336,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.388832,24.74,96.829,1.2104,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,150.088486,29.303,114.689,1.4336,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.145376,472.702,476.048,5.9506,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.655808,664.033,167.184,2.0898,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.787776,615.016,46.922,0.5865,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,3.67824,597.847,41.051,0.5131,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,3.674848,598.398,27.393,0.3424,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,7.50848,585.744,22.344,0.2793,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.133344,32.21,1006.798,12.585,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.403904,48.208,1506.858,18.8357,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.4448,0.822,12.856,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,307.806091,3.572,55.841,0.698,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.42128,1.236,12.889,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,307.196075,5.369,55.966,0.6996,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.411264,1.65,12.904,0.1613,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,307.241394,7.157,55.971,0.6996,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.195808,350.953,353.438,4.418,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.014208,1084.109,272.946,3.4118,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.761152,1444.536,110.209,1.3776,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.404864,1565.293,107.48,1.3435,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.415872,1553.123,71.096,0.8887,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.713216,1620.972,61.835,0.7729,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.34,3.205,100.187,1.2523,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,74.996674,7.33,229.131,2.8641,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.392928,6.167,96.404,1.205,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,75.174622,14.626,228.644,2.8581,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.380864,9.331,97.27,1.2159,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,75.245537,21.918,228.485,2.8561,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.390336,12.357,96.63,1.2079,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,75.239876,29.227,228.558,2.857,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.08048,426.935,442.987,5.5373,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.842176,652.78,169.331,2.1166,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,1.700672,323.258,44.393,0.5549,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,3.46672,317.162,41.136,0.5142,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,1.76368,623.419,47.563,0.5945,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,3.543264,620.621,42.615,0.5327,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.131776,16.296,1018.654,12.7332,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,11.410496,24.09,1505.804,18.8225,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.44016,0.411,12.859,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,156.833313,3.505,109.569,1.3696,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.418336,0.618,12.888,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,156.816544,5.259,109.594,1.3699,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.410752,0.825,12.899,0.1612,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,156.457626,7.028,109.859,1.3732,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.15984,214.963,223.045,2.7881,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.539392,1019.214,264.383,3.3048,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,0.755008,728.146,99.996,1.2499,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,1.401024,784.791,101.787,1.2723,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,0.764576,1438.067,109.716,1.3714,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,1.403392,1566.934,107.593,1.3449,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,1.345088,1.597,99.796,1.2474,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,37.678944,7.295,456.01,5.7001,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,1.379776,3.113,97.299,1.2162,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,37.732033,14.57,455.424,5.6928,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,1.390368,4.634,96.569,1.2071,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,37.871326,21.775,453.804,5.6726,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,1.386272,6.196,96.866,1.2108,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,38.071423,28.88,451.474,5.6434,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,0.953632,576.486,565.174,7.0647,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,14.574848,603.512,147.917,1.849,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,13.85856,634.705,14.527,0.1816,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,27.723743,634.553,9.683,0.121,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,27.708672,634.898,12.11,0.1514,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,56.568226,621.981,7.118,0.089,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.6984,49.198,192.554,2.4069,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,70.961311,61.978,242.575,3.0322,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,23.312063,2.948,5.78,0.0722,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2620.61499,3.356,6.581,0.0823,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,23.30912,4.422,5.792,0.0724,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2618.705322,5.038,6.599,0.0825,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,23.289663,5.901,5.808,0.0726,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2617.378662,6.721,6.615,0.0827,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.1016,499.052,489.259,6.1157,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,7.62208,1154.028,282.846,3.5356,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,5.462624,1610.232,36.855,0.4607,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,10.813152,1626.925,24.825,0.3103,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,10.49488,1676.264,31.972,0.3997,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,20.555616,1711.667,19.588,0.2449,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,5.200928,6.606,25.857,0.3232,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,601.778809,7.308,28.604,0.3576,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,5.252192,13.084,25.654,0.3207,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,605.166321,14.535,28.5,0.3562,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,5.253568,19.621,25.698,0.3212,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,605.55072,21.789,28.537,0.3567,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,5.248672,26.185,25.772,0.3221,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,603.57251,29.147,28.686,0.3586,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.495136,555.156,546.38,6.8298,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,7.217152,609.388,149.939,1.8742,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,7.516352,585.13,17.857,0.2232,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,15.083296,583.168,13.348,0.1668,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,13.793376,637.704,14.596,0.1824,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,27.635073,636.589,9.714,0.1214,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.380736,45.123,352.866,4.4108,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,37.944416,57.954,453.206,5.6651,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.45024,3.288,12.869,0.1609,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1247.059814,3.527,13.803,0.1725,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.431584,4.941,12.904,0.1613,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1245.526855,5.297,13.834,0.1729,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.420768,6.594,12.93,0.1616,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1244.409424,7.068,13.86,0.1732,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.584448,470.321,462.886,5.7861,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.84432,1144.038,281.488,3.5186,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.732512,1609.525,49.119,0.614,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.487264,1603.002,36.69,0.4586,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.396128,1630.075,37.309,0.4664,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,10.837344,1623.293,24.769,0.3096,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.637216,6.514,50.943,0.6368,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,299.85141,7.334,57.351,0.7169,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.684768,12.798,50.09,0.6261,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,300.424866,14.639,57.297,0.7162,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.680256,19.229,50.223,0.6278,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,300.543213,21.95,57.33,0.7166,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.677696,25.664,50.32,0.629,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,300.684631,29.254,57.359,0.717,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.262208,524.16,519.873,6.4984,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,3.506048,627.209,155.52,1.944,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,3.626048,606.452,27.761,0.347,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,7.500576,586.361,22.368,0.2796,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,7.493568,586.91,17.911,0.2239,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,14.968544,587.639,13.45,0.1681,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.221376,38.802,606.585,7.5823,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,21.936352,50.123,783.551,9.7944,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.505888,1.635,12.788,0.1598,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,607.599915,3.619,28.303,0.3538,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.491264,2.456,12.812,0.1602,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,606.932556,5.435,28.348,0.3543,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.486848,3.276,12.824,0.1603,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,606.362,7.253,28.388,0.3549,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.323712,424.572,421.099,5.2637,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.962176,1120.706,277.885,3.4736,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.41968,1548.957,70.906,0.8863,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.736544,1607.154,61.308,0.7664,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.72832,1611.998,49.194,0.6149,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.476128,1606.261,36.764,0.4596,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.348992,6.368,99.543,1.2443,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,149.548325,7.352,114.934,1.4367,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.388736,12.371,96.742,1.2093,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,149.929276,14.667,114.698,1.4337,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.395136,18.471,96.345,1.2043,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,149.952286,21.997,114.737,1.4342,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.39104,24.701,96.676,1.2084,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,149.955322,29.329,114.79,1.4349,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.145792,471.353,474.69,5.9336,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.657632,663.303,167.0,2.0875,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.791168,613.852,46.833,0.5854,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,3.648224,602.765,41.389,0.5174,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,3.603328,610.276,27.936,0.3492,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,7.488928,587.273,22.403,0.28,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.133184,32.248,1008.008,12.6001,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.403808,48.208,1506.871,18.8359,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.440992,0.823,12.861,0.1608,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,307.691345,3.573,55.862,0.6983,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.423392,1.236,12.886,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,307.765106,5.359,55.862,0.6983,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.411232,1.65,12.904,0.1613,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,306.819977,7.167,56.048,0.7006,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.190816,360.135,362.685,4.5336,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.01536,1082.879,272.636,3.408,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.7616,1443.686,110.145,1.3768,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.405344,1564.758,107.443,1.343,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.41232,1557.029,71.275,0.8909,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.713536,1620.781,61.828,0.7728,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.344352,3.195,99.863,1.2483,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,74.981056,7.332,229.179,2.8647,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.387904,6.189,96.753,1.2094,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,75.160004,14.629,228.689,2.8586,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.39184,9.257,96.502,1.2063,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,75.201088,21.931,228.62,2.8577,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.391168,12.349,96.573,1.2072,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,75.193565,29.245,228.698,2.8587,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_context.png b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_context.png index 8e05f9af2..48b122e91 100755 Binary files a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_context.png and b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_context.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_generation.png b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_generation.png index bc5e374d4..c544081d1 100755 Binary files a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_generation.png and b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_generation.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260820_063046.csv b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260820_063046.csv deleted file mode 100644 index fbce533ea..000000000 --- a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260820_063046.csv +++ /dev/null @@ -1,113 +0,0 @@ -config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,2.900576,94.767,93.269,1.9431,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,44.16547,99.581,24.502,0.5105,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.479519,88.886,2.713,0.0565,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,99.701057,88.225,2.019,0.0421,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,96.730431,90.934,2.081,0.0434,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,197.76358,88.956,1.357,0.0283,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.430464,39.91,312.102,6.5021,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,44.705471,49.189,384.665,8.0139,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.843296,3.169,12.402,0.2584,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1515.352783,2.902,11.359,0.2367,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.864384,4.744,12.39,0.2581,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1517.114136,4.348,11.357,0.2366,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.87312,6.32,12.392,0.2582,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1517.066406,5.798,11.369,0.2368,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.507968,91.142,90.396,1.8833,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,22.101824,99.495,24.67,0.514,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.728672,88.926,4.071,0.0848,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.697567,88.496,3.376,0.0703,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.5336,90.619,2.765,0.0576,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,98.925346,88.916,2.035,0.0424,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.235744,36.438,569.615,11.867,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,23.077473,47.644,744.807,15.5168,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.33024,1.663,13.005,0.2709,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,737.226135,2.983,23.326,0.486,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.325088,2.496,13.018,0.2712,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,736.672729,4.478,23.355,0.4866,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.31456,3.331,13.038,0.2716,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,736.653015,5.97,23.367,0.4868,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.810848,84.75,85.35,1.7781,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.1392,98.707,24.851,0.5177,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,12.227488,89.921,6.86,0.1429,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,24.472288,89.858,6.17,0.1285,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.434528,89.997,4.12,0.0858,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.525536,88.804,3.388,0.0706,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.135936,31.596,987.601,20.575,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,12.312352,44.651,1395.677,29.0766,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.130656,0.848,13.255,0.2761,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,362.943634,3.029,47.358,0.9866,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.135648,1.271,13.252,0.2761,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,362.758087,4.546,47.394,0.9874,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.140064,1.694,13.249,0.276,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,362.095367,6.073,47.492,0.9894,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.460992,74.534,77.337,1.6112,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,5.670144,96.956,25.15,0.524,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,5.49008,100.136,13.752,0.2865,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,11.057184,99.439,12.897,0.2687,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,12.211552,90.039,6.869,0.1431,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,24.467775,89.874,6.171,0.1286,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.135456,15.854,990.979,20.6454,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,12.300576,22.347,1396.842,29.1009,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.13632,0.424,13.245,0.2759,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,183.553665,2.995,93.619,1.9504,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.143744,0.635,13.236,0.2758,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,183.492355,4.494,93.661,1.9513,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.184256,0.843,13.185,0.2747,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,183.127167,6.004,93.86,1.9554,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,5.731296,95.922,94.039,1.9592,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,89.181885,98.631,24.174,0.5036,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,99.047684,88.807,2.033,0.0423,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,200.097275,87.918,1.342,0.0279,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,192.697601,91.294,1.741,0.0363,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,395.031342,89.067,1.019,0.0212,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.756576,45.415,177.748,3.7031,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,87.557854,50.23,196.595,4.0957,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,24.455904,2.81,5.51,0.1148,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,3169.266846,2.775,5.442,0.1134,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,24.460608,4.214,5.519,0.115,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,3169.713867,4.163,5.452,0.1136,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,24.457439,5.62,5.531,0.1152,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,3170.342773,5.549,5.461,0.1138,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,2.914144,94.325,92.834,1.934,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,44.189758,99.526,24.488,0.5102,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.484833,88.877,2.712,0.0565,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,99.752228,88.179,2.018,0.042,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,96.757057,90.909,2.081,0.0433,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,197.692703,88.988,1.358,0.0283,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.429056,40.041,313.126,6.5235,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,44.754177,49.136,384.247,8.0051,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.873856,3.16,12.367,0.2577,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1515.567871,2.902,11.358,0.2366,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.870528,4.741,12.383,0.258,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1516.871216,4.349,11.359,0.2366,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.843584,6.337,12.426,0.2589,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1517.119995,5.798,11.368,0.2368,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.505824,91.272,90.525,1.8859,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,22.097441,99.515,24.675,0.5141,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.717152,88.968,4.073,0.0848,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.67971,88.528,3.377,0.0704,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.511169,90.66,2.767,0.0576,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,98.915489,88.925,2.035,0.0424,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.235584,36.462,570.002,11.875,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,23.08688,47.625,744.503,15.5105,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.338048,1.662,12.996,0.2707,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,735.951538,2.988,23.367,0.4868,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.330144,2.495,13.012,0.2711,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,736.717773,4.477,23.354,0.4865,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.327904,3.327,13.021,0.2713,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,733.80835,5.993,23.458,0.4887,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.8096,84.881,85.482,1.7809,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.12832,98.803,24.876,0.5182,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,12.23984,89.831,6.854,0.1428,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,24.49424,89.777,6.165,0.1284,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.441025,89.973,4.119,0.0858,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.483936,88.878,3.39,0.0706,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.137184,31.308,978.616,20.3878,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,12.308576,44.664,1396.105,29.0855,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.12896,0.848,13.257,0.2762,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,363.20694,3.027,47.324,0.9859,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.133984,1.271,13.254,0.2761,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,362.625427,4.548,47.411,0.9877,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.144,1.694,13.244,0.2759,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,362.598969,6.065,47.426,0.988,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260821_063604.csv b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260821_063604.csv new file mode 100644 index 000000000..c8272d0d8 --- /dev/null +++ b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260821_063604.csv @@ -0,0 +1,113 @@ +config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,2.919392,94.156,92.667,1.9306,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,44.105953,99.715,24.535,0.5111,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.491936,88.864,2.712,0.0565,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,99.81498,88.124,2.017,0.042,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,96.718468,90.945,2.082,0.0434,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,197.774521,88.951,1.357,0.0283,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.430816,39.878,311.847,6.4968,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,44.752609,49.137,384.26,8.0054,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.890496,3.155,12.348,0.2573,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1515.303955,2.902,11.36,0.2367,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.859552,4.746,12.396,0.2582,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1517.061401,4.349,11.358,0.2366,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.869632,6.322,12.396,0.2583,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1515.832886,5.803,11.378,0.237,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.516384,90.636,89.895,1.8728,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,22.082527,99.582,24.692,0.5144,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.742529,88.876,4.068,0.0848,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.715874,88.464,3.375,0.0703,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.499489,90.682,2.767,0.0577,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,98.948158,88.896,2.035,0.0424,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.235488,36.477,570.234,11.8799,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,23.09664,47.605,744.189,15.5039,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.382272,1.655,12.94,0.2696,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,735.884949,2.988,23.369,0.4868,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.319328,2.497,13.025,0.2714,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,735.471985,4.485,23.393,0.4874,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.360896,3.316,12.98,0.2704,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,737.140564,5.966,23.352,0.4865,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.8128,84.547,85.145,1.7739,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.135584,98.739,24.859,0.5179,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,12.20608,90.079,6.872,0.1432,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,24.46032,89.902,6.173,0.1286,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.482847,89.819,4.112,0.0857,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.539169,88.779,3.387,0.0706,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.136288,31.514,985.05,20.5219,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,12.378496,44.412,1388.219,28.9212,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.149856,0.846,13.23,0.2756,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,363.123657,3.028,47.334,0.9861,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.117344,1.274,13.276,0.2766,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,362.520874,4.549,47.425,0.988,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.166656,1.69,13.215,0.2753,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,363.726196,6.046,47.279,0.985,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.459968,74.7,77.509,1.6148,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,5.694208,96.546,25.044,0.5218,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,5.531328,99.389,13.649,0.2844,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,11.039296,99.6,12.918,0.2691,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,12.197728,90.141,6.877,0.1433,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,24.439137,89.98,6.178,0.1287,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.1344,15.978,998.766,20.8076,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,12.358656,22.242,1390.278,28.9641,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.162912,0.423,13.21,0.2752,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,183.097794,3.003,93.852,1.9552,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.178752,0.633,13.191,0.2748,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,183.343781,4.498,93.737,1.9529,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.220384,0.84,13.139,0.2737,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,183.207169,6.001,93.819,1.9546,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,5.750912,95.595,93.719,1.9525,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,87.948738,100.014,24.513,0.5107,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,99.075394,88.782,2.032,0.0423,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,200.056122,87.936,1.342,0.028,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,192.568222,91.356,1.742,0.0363,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,394.158691,89.264,1.022,0.0213,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.75856,45.296,177.283,3.6934,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,87.556641,50.231,196.598,4.0958,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,24.541952,2.8,5.49,0.1144,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,3169.980957,2.775,5.441,0.1133,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,24.594208,4.191,5.489,0.1144,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,3170.125732,4.162,5.451,0.1136,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,24.546528,5.599,5.511,0.1148,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,3171.532471,5.547,5.459,0.1137,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,2.92304,94.038,92.552,1.9282,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,44.12339,99.676,24.525,0.5109,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.477505,88.89,2.713,0.0565,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,99.893982,88.054,2.015,0.042,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,96.71405,90.949,2.082,0.0434,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,197.823837,88.929,1.357,0.0283,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.432128,39.756,310.9,6.4771,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,44.749439,49.141,384.287,8.006,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.964416,3.134,12.265,0.2555,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1516.034058,2.901,11.354,0.2365,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.937088,4.712,12.308,0.2564,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1517.22876,4.348,11.356,0.2366,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.9168,6.295,12.343,0.2571,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1516.856079,5.799,11.37,0.2369,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.518368,90.518,89.777,1.8704,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,22.206623,99.026,24.554,0.5115,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.739616,88.887,4.069,0.0848,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.735233,88.429,3.373,0.0703,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.558689,90.572,2.764,0.0576,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,99.021919,88.83,2.033,0.0424,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.235904,36.413,569.228,11.8589,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,23.118816,47.559,743.475,15.4891,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.369856,1.657,12.956,0.2699,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,735.598572,2.989,23.378,0.487,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.321312,2.497,13.023,0.2713,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,736.326111,4.48,23.366,0.4868,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.317408,3.33,13.034,0.2715,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,736.929993,5.968,23.358,0.4866,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.812896,84.537,85.135,1.7736,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.137472,98.722,24.855,0.5178,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,12.235584,89.862,6.856,0.1428,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,24.461121,89.899,6.173,0.1286,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.515648,89.699,4.106,0.0855,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.531487,88.793,3.387,0.0706,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.137408,31.257,977.021,20.3546,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,12.322048,44.616,1394.579,29.0537,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.159424,0.846,13.218,0.2754,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,363.098206,3.028,47.338,0.9862,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.159104,1.268,13.221,0.2754,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,362.171722,4.554,47.47,0.989,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.162368,1.691,13.22,0.2754,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,362.320831,6.069,47.462,0.9888,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_context.png b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_context.png index 42606d147..81d609d51 100755 Binary files a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_context.png and b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_context.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_generation.png b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_generation.png index 70d011ffd..464041fea 100755 Binary files a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_generation.png and b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_generation.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260820_070404.csv b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260820_070404.csv deleted file mode 100644 index 26126b471..000000000 --- a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260820_070404.csv +++ /dev/null @@ -1,113 +0,0 @@ -config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,3.144224,87.423,86.041,4.8014,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,46.739391,94.097,23.152,1.292,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.08419,89.602,2.734,0.1526,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,98.374878,89.414,2.047,0.1142,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,91.999268,95.61,2.188,0.1221,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,185.479431,94.847,1.447,0.0808,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.623616,27.549,215.435,12.0221,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,68.158852,32.263,252.302,14.0794,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,12.385056,2.774,10.858,0.6059,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1072.839233,4.099,16.045,0.8954,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,12.392192,4.159,10.863,0.6062,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1106.209473,5.964,15.576,0.8692,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,12.393088,5.545,10.872,0.6067,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1128.900024,7.792,15.278,0.8525,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.635072,84.057,83.369,4.6523,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,23.575968,93.274,23.128,1.2906,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.655775,89.189,4.083,0.2278,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.217056,89.36,3.409,0.1902,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.903809,89.933,2.745,0.1532,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,98.165535,89.605,2.051,0.1144,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.33072,25.973,406.033,22.6581,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,36.48624,30.135,471.089,26.2884,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,9.797216,1.754,13.713,0.7652,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,539.824524,4.074,31.856,1.7777,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,9.801344,2.629,13.714,0.7653,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,544.313293,6.06,31.609,1.7639,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,9.804736,3.504,13.716,0.7654,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,548.66156,8.016,31.373,1.7508,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.8776,78.304,78.858,4.4006,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.99648,91.653,23.075,1.2877,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,16.048449,68.512,5.227,0.2917,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,32.341217,67.994,4.669,0.2605,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.404127,90.109,4.125,0.2302,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.009663,89.738,3.423,0.191,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.221504,19.39,606.086,33.8218,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,18.440767,29.812,931.852,52.0007,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,9.73536,0.882,13.793,0.7697,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,269.899719,4.074,63.684,3.5538,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,9.752128,1.321,13.773,0.7686,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,271.62323,6.072,63.295,3.5321,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,9.760096,1.76,13.765,0.7681,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,272.995361,8.055,62.992,3.5152,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.50448,68.109,70.67,3.9436,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,6.208,88.556,22.971,1.2819,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,7.355968,74.736,10.263,0.5727,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,14.702304,74.785,9.7,0.5413,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,15.946752,68.949,5.26,0.2935,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,31.897856,68.94,4.734,0.2642,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.216512,9.919,619.985,34.5974,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,18.165184,15.132,945.874,52.7831,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,9.7736,0.439,13.736,0.7665,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,136.584869,4.025,125.812,7.0208,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,9.807104,0.657,13.691,0.764,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,137.132187,6.013,125.326,6.9936,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,9.775712,0.879,13.736,0.7665,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,137.525436,7.995,124.982,6.9745,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,6.158752,89.264,87.513,4.8835,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,93.648994,93.926,23.021,1.2846,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,92.341888,95.256,2.18,0.1217,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,186.004837,94.579,1.443,0.0805,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,184.729568,95.232,1.816,0.1014,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,373.785034,94.13,1.077,0.0601,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,1.201344,28.601,111.941,6.2467,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,135.89389,32.364,126.668,7.0685,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,21.217249,3.239,6.351,0.3544,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2237.774414,3.931,7.707,0.4301,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,21.246336,4.852,6.354,0.3546,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2297.407471,5.743,7.522,0.4197,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,21.261761,6.464,6.362,0.355,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2308.772949,7.62,7.499,0.4185,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,3.142624,87.468,86.085,4.8038,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,47.14896,93.28,22.951,1.2808,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.125023,89.528,2.732,0.1525,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,98.356064,89.431,2.047,0.1142,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,92.736542,94.85,2.171,0.1211,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,187.083359,94.034,1.435,0.0801,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.624768,27.498,215.038,11.9999,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,68.170143,32.258,252.261,14.077,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,12.384672,2.774,10.859,0.6059,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1083.5979,4.059,15.885,0.8865,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,12.40032,4.156,10.855,0.6058,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1116.61792,5.908,15.431,0.8611,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,12.396096,5.544,10.87,0.6066,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1130.931152,7.778,15.25,0.851,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.633536,84.136,83.448,4.6567,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,23.576609,93.271,23.127,1.2906,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.606785,89.367,4.091,0.2283,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.361439,89.099,3.399,0.1897,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.899742,89.94,2.745,0.1532,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,98.127235,89.64,2.052,0.1145,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.330016,26.029,406.899,22.7064,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,36.581409,30.057,469.863,26.22,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,9.796736,1.754,13.714,0.7653,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,538.621338,4.083,31.927,1.7816,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,9.782816,2.634,13.74,0.7667,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,543.81897,6.066,31.637,1.7655,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,9.794944,3.508,13.73,0.7662,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,548.555664,8.018,31.38,1.7511,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.879776,78.11,78.663,4.3897,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.99376,91.674,23.081,1.288,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,16.021376,68.628,5.236,0.2922,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,32.345505,67.985,4.668,0.2605,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.413631,90.074,4.123,0.2301,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.027714,89.705,3.422,0.191,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.220992,19.435,607.49,33.9001,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,18.400801,29.877,933.876,52.1136,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,9.767232,0.879,13.748,0.7672,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,269.916351,4.074,63.68,3.5536,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,9.752608,1.321,13.772,0.7685,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,271.740936,6.069,63.268,3.5306,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,9.761952,1.76,13.762,0.768,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,273.245941,8.048,62.935,3.512,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260821_064242.csv b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260821_064242.csv new file mode 100644 index 000000000..389c436d7 --- /dev/null +++ b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260821_064242.csv @@ -0,0 +1,113 @@ +config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,3.136928,87.626,86.241,4.8126,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,46.705185,94.166,23.169,1.2929,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,48.981026,89.791,2.74,0.1529,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,97.985023,89.77,2.055,0.1147,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,91.938339,95.674,2.19,0.1222,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,185.206726,94.987,1.449,0.0809,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.622208,27.611,215.923,12.0493,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,68.076286,32.302,252.608,14.0965,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,12.367904,2.778,10.873,0.6068,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1067.491211,4.12,16.125,0.8998,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,12.391936,4.159,10.863,0.6062,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1080.499512,6.106,15.947,0.8899,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,12.39536,5.544,10.87,0.6066,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1086.689941,8.094,15.871,0.8857,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.633472,84.139,83.451,4.6569,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,23.56736,93.308,23.136,1.2911,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.577024,89.475,4.096,0.2286,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.036736,89.689,3.421,0.1909,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.88509,89.967,2.746,0.1532,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,98.088638,89.675,2.052,0.1145,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.329088,26.102,408.047,22.7705,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,36.572063,30.064,469.983,26.2267,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,9.7832,1.756,13.733,0.7663,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,535.561462,4.106,32.11,1.7918,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,9.80112,2.629,13.714,0.7653,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,541.545227,6.091,31.77,1.7729,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,9.846688,3.489,13.657,0.7621,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,544.372925,8.079,31.621,1.7645,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.878464,78.227,78.781,4.3962,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.98912,91.709,23.09,1.2885,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,15.976096,68.822,5.251,0.293,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,32.120255,68.462,4.701,0.2623,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.334721,90.366,4.137,0.2308,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,48.965408,89.819,3.426,0.1912,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.22224,19.326,604.079,33.7098,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,18.355392,29.951,936.186,52.2425,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,9.75024,0.881,13.772,0.7685,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,268.930145,4.088,63.913,3.5666,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,9.77184,1.319,13.745,0.767,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,270.475769,6.098,63.564,3.5471,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,9.775936,1.757,13.743,0.7669,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,272.008301,8.084,63.221,3.528,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.500096,68.706,71.289,3.9782,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,6.20608,88.583,22.978,1.2823,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,7.352288,74.773,10.269,0.573,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,14.692992,74.832,9.706,0.5416,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,15.951808,68.927,5.259,0.2935,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,31.737375,69.288,4.758,0.2655,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.215296,9.975,623.486,34.7928,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,18.159231,15.137,946.184,52.8004,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,9.804192,0.438,13.693,0.7641,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,136.192612,4.037,126.175,7.041,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,9.799392,0.657,13.702,0.7646,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,136.886627,6.024,125.55,7.0062,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,9.812736,0.875,13.685,0.7636,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,137.509308,7.996,124.997,6.9753,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,6.152544,89.354,87.601,4.8884,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,93.500061,94.076,23.057,1.2867,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,92.308578,95.29,2.181,0.1217,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,185.683044,94.743,1.446,0.0807,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,184.197922,95.507,1.822,0.1017,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,372.684296,94.408,1.08,0.0603,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,1.199744,28.639,112.09,6.255,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,135.914169,32.359,126.649,7.0675,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,21.20928,3.24,6.353,0.3545,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2150.618164,4.09,8.02,0.4475,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,21.235136,4.854,6.358,0.3548,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2220.725098,5.941,7.781,0.4342,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,21.263777,6.464,6.361,0.355,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2232.101074,7.881,7.757,0.4329,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,3.140704,87.521,86.138,4.8068,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,47.034592,93.507,23.007,1.2839,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.071968,89.624,2.735,0.1526,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,97.973824,89.78,2.055,0.1147,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,92.404259,95.191,2.179,0.1216,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,186.310425,94.424,1.441,0.0804,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.622144,27.614,215.945,12.0505,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,68.109406,32.287,252.486,14.0896,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,12.377344,2.776,10.865,0.6063,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1070.776611,4.107,16.076,0.8971,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,12.395712,4.158,10.859,0.606,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1081.752075,6.099,15.928,0.8888,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,12.40992,5.537,10.858,0.6059,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1086.92395,8.093,15.868,0.8855,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.633952,84.114,83.426,4.6555,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,23.558464,93.343,23.145,1.2916,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.592672,89.418,4.093,0.2284,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.066143,89.635,3.419,0.1908,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.917183,89.908,2.744,0.1531,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,97.87635,89.869,2.057,0.1148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.330144,26.019,406.742,22.6976,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,36.584225,30.054,469.827,26.218,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,9.802528,1.753,13.706,0.7648,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,535.471985,4.107,32.115,1.7921,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,9.802432,2.629,13.712,0.7652,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,540.899414,6.098,31.808,1.775,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,9.841952,3.491,13.664,0.7625,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,544.269226,8.081,31.627,1.7649,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.875488,78.493,79.049,4.4112,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.982112,91.763,23.103,1.2892,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,15.870976,69.278,5.286,0.2949,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,32.176769,68.342,4.693,0.2619,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.370848,90.232,4.13,0.2305,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,48.886623,89.964,3.432,0.1915,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.221216,19.415,606.875,33.8658,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,18.385824,29.901,934.637,52.1561,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,9.745984,0.881,13.778,0.7689,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,269.014404,4.087,63.893,3.5655,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,9.788864,1.316,13.721,0.7657,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,270.62149,6.094,63.53,3.5452,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,9.752288,1.762,13.776,0.7688,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,271.85379,8.089,63.257,3.53,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_context.png b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_context.png index c36e31706..87922e6e1 100755 Binary files a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_context.png and b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_context.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_generation.png b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_generation.png index 8fddc5ae6..76941e1e5 100755 Binary files a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_generation.png and b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_generation.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260819_053417.csv b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260821_053236.csv similarity index 61% rename from benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260819_053417.csv rename to benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260821_053236.csv index 561d5ff0e..3ea5bd4f0 100644 --- a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260819_053417.csv +++ b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260821_053236.csv @@ -1,209 +1,209 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.064864,529.72,581.967,7.2746,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.136832,4017.743,1103.506,13.7938,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.435008,20220.532,1388.434,17.3554,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.045728,12022.301,212.109,2.6514,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.0432,25451.657,224.521,2.8065,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.049856,22053.748,383.836,4.7979,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.048672,45180.456,393.173,4.9147,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.03712,115.705,8.055,0.1007,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.295328,1861.509,129.595,1.6199,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.038496,167.354,8.246,0.1031,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.295936,2786.527,137.301,1.7163,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.03856,222.768,8.71,0.1089,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.295904,3715.771,145.289,1.8161,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.229408,149.776,164.548,2.0569,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.513888,1069.797,293.829,3.6729,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.83792,4785.896,328.621,4.1078,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.26544,2071.112,36.541,0.4568,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.413152,2661.276,23.476,0.2935,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.293408,3747.381,65.222,0.8153,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.444032,4952.398,43.097,0.5387,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.404544,10.617,0.739,0.0092,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,41.955872,13.103,0.912,0.0114,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.406848,15.835,0.78,0.0098,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,41.9664,19.65,0.968,0.0121,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.400864,21.429,0.838,0.0105,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,41.969566,26.198,1.024,0.0128,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.049184,349.298,383.75,4.7969,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.08448,3253.763,893.673,11.1709,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.235744,18656.027,1281.008,16.0126,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.042752,6429.592,113.437,1.418,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.04384,12540.051,110.622,1.3828,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.04432,12404.238,215.89,2.6986,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.045184,24334.092,211.762,2.647,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.03888,55.234,3.845,0.0481,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.162432,1692.264,117.812,1.4727,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.039488,81.575,4.019,0.0502,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.164,2514.127,123.879,1.5485,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.039616,108.415,4.239,0.053,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.162528,3382.53,132.259,1.6532,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.182336,94.221,103.514,1.2939,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.310624,884.922,243.051,3.0381,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.961792,4572.763,313.987,3.9248,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.1816,1513.645,26.705,0.3338,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.259456,2118.879,18.692,0.2336,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.19184,2865.7,49.876,0.6235,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.266112,4131.763,35.956,0.4494,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.255584,8.402,0.585,0.0073,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,21.56768,12.745,0.887,0.0111,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.254464,12.659,0.624,0.0078,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,21.265152,19.389,0.955,0.0119,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.251392,17.085,0.668,0.0084,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,21.271648,25.845,1.011,0.0126,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.042848,200.475,220.248,2.7531,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.062816,2187.961,600.941,7.5118,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.136128,16154.086,1109.213,13.8652,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.045952,2990.924,52.769,0.6596,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.043488,6320.776,55.759,0.697,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.035616,7717.821,134.325,1.6791,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.042304,12995.362,113.089,1.4136,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.040032,26.822,1.867,0.0233,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.098912,1389.507,96.735,1.2092,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.037824,42.582,2.098,0.0262,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.095936,2148.916,105.884,1.3235,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.03968,54.12,2.116,0.0265,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.097952,2806.251,109.726,1.3716,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.161472,53.198,58.445,0.7306,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.228768,600.779,165.009,2.0626,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.512032,4294.699,294.894,3.6862,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.163776,839.189,14.806,0.1851,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.188288,1459.88,12.878,0.161,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.152192,1806.126,31.435,0.3929,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.185632,2961.536,25.772,0.3222,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.178624,6.011,0.418,0.0052,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,10.803072,12.722,0.886,0.0111,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.178976,8.999,0.443,0.0055,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,10.83376,19.029,0.938,0.0117,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.183168,11.724,0.458,0.0057,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,10.813824,25.419,0.994,0.0124,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.04208,102.067,112.134,1.4017,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.049376,1391.759,382.258,4.7782,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.0856,12844.761,881.98,11.0247,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.044384,1548.294,27.317,0.3415,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.04512,3046.076,26.871,0.3359,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.0424,3241.485,56.417,0.7052,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.041632,6602.563,57.457,0.7182,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.03936,13.64,0.95,0.0119,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.06528,1052.688,73.286,0.9161,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.03696,21.789,1.074,0.0134,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.065856,1565.221,77.123,0.964,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.038368,27.985,1.094,0.0137,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.064224,2139.994,83.675,1.0459,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.134144,32.018,35.176,0.4397,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.15968,430.357,118.201,1.4775,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.287584,3823.271,262.523,3.2815,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.12736,539.569,9.52,0.119,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.134784,1019.698,8.995,0.1124,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.13696,1003.497,17.465,0.2183,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.133824,2054.025,17.875,0.2234,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.133824,4.012,0.279,0.0035,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.257888,266.47,18.551,0.2319,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.137568,5.854,0.288,0.0036,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.255616,403.258,19.87,0.2484,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.135552,7.921,0.31,0.0039,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.255552,537.812,21.029,0.2629,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.07984,430.357,472.805,5.9101,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.583712,941.827,258.681,3.2335,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,8.44144,1042.013,71.549,0.8944,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.66096,831.754,215.756,2.697,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.284096,856.253,215.579,2.6947,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.275872,861.773,118.346,1.4793,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.517984,873.327,113.27,1.4159,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.458336,1.747,109.207,1.3651,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,102.393341,5.369,335.607,4.1951,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.458464,2.621,109.208,1.3651,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,102.400452,8.053,335.604,4.1951,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.460032,3.492,109.145,1.3643,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,102.389763,10.738,335.66,4.1957,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.221216,155.322,170.642,2.133,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.971712,565.76,155.391,1.9424,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,11.673056,753.538,51.741,0.6468,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,1.008768,544.977,141.367,1.7671,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.929504,569.842,143.469,1.7934,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.788032,614.928,84.448,1.0556,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,3.471456,633.458,82.159,1.027,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.92384,2.232,139.548,1.7444,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,206.474396,2.663,166.432,2.0804,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.925568,3.346,139.431,1.7429,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,206.557053,3.992,166.375,2.0797,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.926368,4.459,139.382,1.7423,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,206.563293,5.323,166.381,2.0798,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.051552,333.253,366.123,4.5765,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.31232,880.116,241.731,3.0216,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,4.202112,1046.628,71.866,0.8983,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.65376,420.457,109.066,1.3633,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.283648,428.276,107.827,1.3478,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.660832,831.915,114.246,1.4281,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.270304,865.55,112.262,1.4033,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.467136,0.87,54.409,0.6801,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,52.835903,5.202,325.195,4.0649,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.466432,1.306,54.428,0.6803,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,52.851231,7.801,325.12,4.064,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.46368,1.743,54.492,0.6811,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,52.828735,10.406,325.279,4.066,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.191264,89.823,98.682,1.2335,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.542144,507.02,139.257,1.7407,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,5.866624,749.672,51.476,0.6434,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.932224,294.863,76.487,0.9561,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.769056,310.762,78.241,0.978,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.933376,588.997,80.886,1.0111,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.77488,619.485,80.347,1.0043,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.767616,1.215,75.941,0.9493,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,103.877502,2.646,165.406,2.0676,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.764,1.826,76.101,0.9513,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,103.53331,3.982,165.966,2.0746,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.765536,2.433,76.04,0.9505,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,103.52784,5.31,165.985,2.0748,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.056832,151.146,166.054,2.0757,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.178752,768.881,211.179,2.6397,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.078816,1057.825,72.635,0.9079,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.659232,208.483,54.08,0.676,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.281344,214.523,54.01,0.6751,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.656192,418.899,57.527,0.7191,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.281952,428.843,55.621,0.6953,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.30592,0.466,29.106,0.3638,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,26.53376,5.18,323.776,4.0472,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.306976,0.698,29.095,0.3637,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,26.537249,7.769,323.753,4.0469,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.306368,0.931,29.104,0.3638,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,26.539841,10.357,323.741,4.0468,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.164832,52.113,57.253,0.7157,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.350368,392.27,107.74,1.3468,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.924672,751.887,51.628,0.6453,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.903072,152.19,39.478,0.4935,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.698752,161.812,40.739,0.5092,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.91184,301.454,41.398,0.5175,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.691424,325.025,42.156,0.5269,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.69744,0.633,39.54,0.4943,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,51.943359,2.646,165.391,2.0674,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.694784,0.95,39.605,0.4951,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,51.913376,3.971,165.497,2.0687,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.69808,1.265,39.53,0.4941,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,51.957535,5.29,165.366,2.0671,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.053984,79.56,87.407,1.0926,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.111776,614.796,168.859,2.1107,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.06608,1031.359,70.818,0.8852,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.657056,104.587,27.13,0.3391,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.272128,108.039,27.201,0.34,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.659232,208.483,28.631,0.3579,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.27664,215.314,27.926,0.3491,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.30368,0.233,14.567,0.1821,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,15.1152,4.546,284.184,3.5523,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.30416,0.35,14.565,0.1821,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,15.116544,6.819,284.176,3.5522,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.305632,0.466,14.557,0.182,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,15.116544,9.092,284.193,3.5524,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.139904,30.699,33.727,0.4216,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.264992,259.327,71.226,0.8903,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.503264,731.416,50.222,0.6278,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.874592,78.573,20.382,0.2548,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.639456,83.832,21.106,0.2638,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.876352,156.831,21.537,0.2692,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.634208,168.203,21.816,0.2727,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.635232,0.328,20.522,0.2565,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,21.558529,3.188,199.248,2.4906,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.640576,0.491,20.457,0.2557,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,21.547264,4.784,199.364,2.4921,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.636192,0.656,20.513,0.2564,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,21.537088,6.382,199.471,2.4934,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.061696,556.92,611.851,7.6481,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.135424,4059.515,1114.979,13.9372,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.432032,20359.819,1397.998,17.475,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.045056,12201.612,215.273,2.6909,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.03952,27821.651,245.428,3.0679,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.042048,26148.964,455.111,5.6889,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.047168,46621.082,405.71,5.0714,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.033504,128.193,8.925,0.1116,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.292704,1878.197,130.757,1.6345,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.037504,171.78,8.464,0.1058,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.29408,2804.114,138.168,1.7271,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.032768,262.144,10.25,0.1281,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.294464,3733.943,146.0,1.825,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.236096,145.533,159.887,1.9986,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.529472,1038.31,285.18,3.5648,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.889376,4655.554,319.672,3.9959,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.275328,1996.73,35.228,0.4404,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.432832,2540.273,22.409,0.2801,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.298496,3683.505,64.11,0.8014,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.45632,4819.038,41.937,0.5242,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.418176,10.271,0.715,0.0089,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,43.228642,12.717,0.885,0.0111,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.418784,15.384,0.758,0.0095,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,43.251007,19.066,0.939,0.0117,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.418656,20.518,0.802,0.01,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,43.242622,25.427,0.994,0.0124,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.04864,353.205,388.042,4.8505,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.082848,3317.858,911.277,11.391,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.234528,18752.756,1287.65,16.0956,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.037376,7354.396,129.753,1.6219,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.041728,13174.746,116.221,1.4528,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.041312,13307.412,231.61,2.8951,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.03952,27821.651,242.112,3.0264,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.036064,59.546,4.146,0.0518,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.159424,1724.194,120.035,1.5004,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.035328,91.181,4.493,0.0562,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.160832,2563.649,126.319,1.579,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.038048,112.883,4.414,0.0552,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.159424,3448.388,134.834,1.6854,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.18256,94.105,103.387,1.2923,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.320928,856.51,235.247,2.9406,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.986432,4458.54,306.144,3.8268,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.190016,1446.604,25.522,0.319,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.271264,2026.645,17.878,0.2235,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.197664,2781.264,48.407,0.6051,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.27584,3986.048,34.688,0.4336,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.263104,8.162,0.568,0.0071,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,22.168192,12.4,0.863,0.0108,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.256256,12.57,0.619,0.0077,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,21.870337,18.853,0.929,0.0116,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.260032,16.517,0.646,0.0081,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,21.890688,25.114,0.982,0.0123,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.032192,266.834,293.153,3.6644,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.06,2290.649,629.146,7.8643,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.135968,16173.094,1110.518,13.8815,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.035392,3883.334,68.514,0.8564,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.039648,6932.958,61.159,0.7645,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.034336,8005.531,139.333,1.7417,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.042592,12907.49,112.325,1.4041,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.03616,29.694,2.067,0.0258,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.096256,1427.848,99.404,1.2426,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.034016,47.349,2.333,0.0292,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.095328,2162.622,106.559,1.332,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.037216,57.703,2.256,0.0282,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.095872,2867.134,112.107,1.4013,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.165792,51.812,56.922,0.7115,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.227424,604.329,165.984,2.0748,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.529632,4151.984,285.094,3.5637,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.166944,823.264,14.525,0.1816,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.192832,1425.479,12.575,0.1572,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.162688,1689.602,29.407,0.3676,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.19376,2837.303,24.691,0.3086,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.186176,5.767,0.402,0.005,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,11.11888,12.361,0.861,0.0108,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.1856,8.678,0.428,0.0053,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,11.10896,18.558,0.914,0.0114,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.184576,11.635,0.455,0.0057,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,11.112128,24.737,0.967,0.0121,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.03648,117.735,129.347,1.6168,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.046944,1463.861,402.061,5.0258,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.083936,13099.405,899.465,11.2433,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.041792,1644.321,29.011,0.3626,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.04048,3395.231,29.951,0.3744,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.042432,3239.04,56.374,0.7047,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.038656,7110.873,61.881,0.7735,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.0368,14.589,1.016,0.0127,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.062752,1095.096,76.239,0.953,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.03488,23.088,1.138,0.0142,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.06352,1622.784,79.96,0.9995,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.030944,34.7,1.357,0.017,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.061248,2243.975,87.741,1.0968,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.1416,30.332,33.323,0.4165,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.164512,417.717,114.729,1.4341,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.294336,3735.566,256.501,3.2063,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.139104,494.015,8.716,0.1089,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.13584,1011.771,8.925,0.1116,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.131008,1049.088,18.259,0.2282,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.132512,2074.362,18.052,0.2256,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.138944,3.864,0.269,0.0034,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.268672,255.775,17.807,0.2226,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.138496,5.815,0.287,0.0036,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.268864,383.388,18.891,0.2361,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.1336,8.037,0.314,0.0039,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.269824,509.365,19.917,0.249,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.071168,482.798,530.417,6.6302,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.578176,950.845,261.157,3.2645,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,8.498112,1035.064,71.072,0.8884,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.659296,833.853,216.301,2.7038,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.282688,857.193,215.816,2.6977,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.280256,858.822,117.941,1.4743,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.505856,877.554,113.818,1.4227,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.492768,1.723,107.699,1.3462,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,101.856354,5.397,337.376,4.2172,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.495424,2.582,107.591,1.3449,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,101.847427,8.097,337.427,4.2178,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.493376,3.445,107.686,1.3461,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,101.838974,10.797,337.475,4.2184,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.219072,156.842,172.312,2.1539,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.99024,555.174,152.483,1.906,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,11.788864,746.136,51.233,0.6404,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,1.014592,541.849,140.555,1.7569,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.936288,567.845,142.966,1.7871,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.795872,612.244,84.079,1.051,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,3.472704,633.231,82.13,1.0266,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.931488,2.224,138.996,1.7374,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,207.187134,2.653,165.859,2.0732,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.926176,3.345,139.387,1.7423,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,207.207642,3.98,165.853,2.0732,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.926912,4.458,139.343,1.7418,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,207.238586,5.306,165.838,2.073,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.05488,313.044,343.921,4.299,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.311104,883.556,242.676,3.0334,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,4.234464,1038.631,71.317,0.8915,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.65824,417.595,108.324,1.354,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.278624,429.959,108.251,1.3531,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.65712,836.614,114.891,1.4361,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.273312,863.505,111.996,1.4,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.497312,0.86,53.751,0.6719,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,52.822975,5.204,325.274,4.0659,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.4976,1.29,53.749,0.6719,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,52.816959,7.807,325.331,4.0666,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.496192,1.721,53.782,0.6723,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,52.814304,10.409,325.368,4.0671,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.1896,90.611,99.548,1.2444,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.547584,501.983,137.874,1.7234,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,5.966144,737.167,50.617,0.6327,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.933696,294.398,76.367,0.9546,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.772992,310.072,78.067,0.9758,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.937824,586.204,80.503,1.0063,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.777152,618.693,80.244,1.0031,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.773632,1.211,75.683,0.946,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,104.196609,2.638,164.899,2.0612,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.770432,1.819,75.825,0.9478,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,103.894943,3.969,165.388,2.0674,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.774432,2.42,75.658,0.9457,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,103.901215,5.291,165.388,2.0674,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.05328,161.222,177.124,2.2141,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.17712,775.965,213.125,2.6641,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.073952,1060.306,72.805,0.9101,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.655872,209.551,54.358,0.6795,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.277984,215.087,54.152,0.6769,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.6544,420.046,57.684,0.7211,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.280672,429.271,55.676,0.696,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.30432,0.466,29.127,0.3641,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,26.423807,5.201,325.123,4.064,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.30288,0.699,29.147,0.3643,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,26.419584,7.803,325.195,4.0649,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.303296,0.932,29.143,0.3643,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,26.422304,10.403,325.181,4.0648,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.1712,50.175,55.124,0.689,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.359264,382.557,105.072,1.3134,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.945696,746.521,51.26,0.6407,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.911744,150.743,39.103,0.4888,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.697984,161.885,40.758,0.5095,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.909408,302.26,41.509,0.5189,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.696928,323.971,42.019,0.5252,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.696064,0.633,39.572,0.4947,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,52.120354,2.637,164.83,2.0604,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.695712,0.95,39.583,0.4948,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,52.147167,3.953,164.755,2.0594,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.697632,1.265,39.541,0.4943,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,52.16317,5.27,164.715,2.0589,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.052256,82.191,90.298,1.1287,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.109248,629.023,172.766,2.1596,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.059424,1037.839,71.263,0.8908,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.653888,105.094,27.261,0.3408,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.262272,108.882,27.413,0.3427,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.655872,209.551,28.778,0.3597,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.267136,216.928,28.136,0.3517,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.302304,0.233,14.576,0.1822,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,15.112128,4.547,284.241,3.553,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.30352,0.35,14.569,0.1821,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,15.111552,6.821,284.27,3.5534,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.301184,0.467,14.585,0.1823,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,15.11328,9.094,284.254,3.5532,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.145792,29.46,32.365,0.4046,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.266304,258.049,70.875,0.8859,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.518112,724.263,49.731,0.6216,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.87776,78.29,20.308,0.2539,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.646016,83.498,21.022,0.2628,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.88,156.181,21.448,0.2681,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.64608,166.989,21.658,0.2707,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.64656,0.326,20.381,0.2548,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,21.491327,3.198,199.871,2.4984,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.637664,0.492,20.493,0.2562,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,21.50704,4.793,199.737,2.4967,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.640768,0.654,20.455,0.2557,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,21.492672,6.395,199.883,2.4985,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_context.png b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_context.png index 87c574491..8a177f5e1 100755 Binary files a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_context.png and b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_context.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_generation.png b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_generation.png index 3dc6e3f61..1eea82a59 100755 Binary files a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_generation.png and b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_generation.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260819_053320.csv b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260821_053358.csv similarity index 62% rename from benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260819_053320.csv rename to benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260821_053358.csv index 708a7b0c4..b5164abda 100644 --- a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260819_053320.csv +++ b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260821_053358.csv @@ -1,209 +1,209 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.059456,577.902,634.902,7.9363,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.12288,4473.924,1228.8,15.36,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.384352,22885.514,1571.424,19.6428,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.04432,12404.238,218.848,2.7356,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.043328,25376.469,223.858,2.7982,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.047296,23247.456,404.612,5.0576,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.048448,45389.35,394.991,4.9374,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.038816,110.649,7.703,0.0963,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.293536,1872.874,130.386,1.6298,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.040448,159.277,7.848,0.0981,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.294912,2796.203,137.778,1.7222,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.038848,221.117,8.646,0.1081,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.295136,3725.441,145.667,1.8208,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.225856,152.131,167.136,2.0892,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.512736,1072.2,294.489,3.6811,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.831008,4803.962,329.862,4.1233,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.264128,2081.399,36.722,0.459,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.411616,2671.207,23.564,0.2946,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.289568,3797.076,66.086,0.8261,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.43312,5077.168,44.183,0.5523,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.407264,10.546,0.734,0.0092,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,41.397377,13.28,0.925,0.0116,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.400928,16.069,0.792,0.0099,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,41.456352,19.892,0.98,0.0123,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.405824,21.167,0.828,0.0103,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,41.412735,26.55,1.038,0.013,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.046656,368.224,404.543,5.0568,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.078592,3497.53,960.625,12.0078,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.208544,21089.298,1448.087,18.1011,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.046464,5915.933,104.375,1.3047,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.04224,13015.052,114.812,1.4352,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.043232,12716.409,221.323,2.7665,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.042048,26148.964,227.556,2.8444,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.036768,58.406,4.066,0.0508,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.16032,1714.558,119.364,1.4921,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.0376,85.671,4.221,0.0528,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.159296,2588.369,127.537,1.5942,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.039072,109.924,4.298,0.0537,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.158816,3461.59,135.35,1.6919,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.17728,96.908,106.466,1.3308,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.322048,853.531,234.429,2.9304,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.955488,4602.932,316.058,3.9507,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.172352,1594.863,28.138,0.3517,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.264736,2076.619,18.319,0.229,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.190336,2888.344,50.27,0.6284,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.269728,4076.372,35.474,0.4434,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.24864,8.637,0.601,0.0075,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,21.237663,12.943,0.901,0.0113,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.258112,12.48,0.615,0.0077,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,20.980097,19.653,0.968,0.0121,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.248512,17.283,0.676,0.0084,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,20.98192,26.201,1.024,0.0128,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.040832,210.373,231.122,2.889,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.05728,2399.423,659.021,8.2378,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.123296,17835.317,1224.654,15.3082,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.045152,3043.917,53.704,0.6713,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.0344,7990.637,70.489,0.8811,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.03472,7916.99,137.792,1.7224,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.033728,16299.686,141.844,1.7731,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.03904,27.504,1.915,0.0239,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.097984,1402.667,97.651,1.2206,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.034592,46.56,2.294,0.0287,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.0984,2095.106,103.233,1.2904,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.038624,55.6,2.174,0.0272,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.094944,2895.158,113.203,1.415,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.160576,53.495,58.771,0.7346,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.222144,618.693,169.929,2.1241,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.51456,4273.599,293.445,3.6681,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.161632,850.32,15.002,0.1875,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.180704,1521.15,13.419,0.1677,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.160416,1713.532,29.823,0.3728,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.182112,3018.779,26.27,0.3284,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.179424,5.984,0.417,0.0052,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,10.67024,12.881,0.897,0.0112,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.171648,9.383,0.462,0.0058,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,10.662944,19.334,0.953,0.0119,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.17488,12.28,0.48,0.006,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,10.6656,25.772,1.008,0.0126,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.04256,100.916,110.869,1.3859,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.0472,1455.921,399.881,4.9985,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.076896,14298.685,981.813,12.2727,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.044864,1531.729,27.024,0.3378,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.04368,3146.496,27.757,0.347,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.043488,3160.388,55.005,0.6876,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.04272,6434.408,55.994,0.6999,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.03808,14.099,0.982,0.0123,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.063904,1075.355,74.864,0.9358,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.03888,20.713,1.021,0.0128,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.06368,1618.706,79.759,0.997,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.039424,27.236,1.065,0.0133,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.061888,2220.769,86.834,1.0854,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.131456,32.672,35.895,0.4487,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.14816,463.819,127.392,1.5924,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.28096,3913.41,268.713,3.3589,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.13008,528.286,9.321,0.1165,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.125088,1098.738,9.693,0.1212,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.134752,1019.94,17.752,0.2219,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.139136,1975.606,17.192,0.2149,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.125632,4.273,0.298,0.0037,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.249856,275.036,19.148,0.2393,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.125024,6.441,0.317,0.004,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.248352,415.053,20.451,0.2556,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.127808,8.401,0.328,0.0041,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.2496,550.637,21.53,0.2691,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.072576,473.431,520.127,6.5016,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.46848,1173.488,322.308,4.0289,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,6.58656,1335.461,91.699,1.1462,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.543328,1011.83,262.468,3.2809,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.046304,1050.853,264.573,3.3072,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.041312,1055.891,145.005,1.8126,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.048512,1073.473,139.229,1.7404,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.4648,1.743,108.921,1.3615,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,106.177536,5.178,323.646,4.0456,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.466112,2.612,108.87,1.3609,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,106.181633,7.766,323.653,4.0457,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.465504,3.484,108.903,1.3613,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,106.185951,10.355,323.66,4.0457,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.21088,162.935,179.006,2.2376,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.851136,645.908,177.404,2.2176,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,9.759424,901.292,61.887,0.7736,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.878048,626.111,162.413,2.0302,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.671168,657.93,165.647,2.0706,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.523968,721.479,99.08,1.2385,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.960128,742.881,96.351,1.2044,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.657504,2.591,161.971,2.0246,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,177.084793,3.104,194.054,2.4257,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.663808,3.872,161.368,2.0171,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,177.093155,4.656,194.056,2.4257,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.659968,5.175,161.751,2.0219,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,177.068924,6.21,194.095,2.4262,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.052832,325.179,357.253,4.4657,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.25152,1092.867,300.165,3.7521,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,3.224448,1363.969,93.656,1.1707,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.538016,510.91,132.53,1.6566,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.039264,528.986,133.183,1.6648,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.540064,1017.946,139.794,1.7474,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.046208,1050.949,136.308,1.7038,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.462816,0.872,54.504,0.6813,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,53.258911,5.161,322.612,4.0327,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.465504,1.307,54.448,0.6806,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,53.249058,7.743,322.691,4.0336,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.4632,1.744,54.502,0.6813,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,53.235905,10.327,322.791,4.0349,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.1776,96.733,106.275,1.3284,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.489152,561.948,154.344,1.9293,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,4.861344,904.698,62.121,0.7765,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.796032,345.31,89.573,1.1197,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.515872,362.666,91.309,1.1414,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.803808,683.939,93.925,1.1741,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.51552,725.501,94.097,1.1762,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.514464,1.418,88.635,1.1079,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,89.498306,3.071,191.981,2.3998,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.509568,2.134,88.928,1.1116,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,89.230751,4.621,192.568,2.4071,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.503232,2.857,89.308,1.1163,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,89.223228,6.162,192.596,2.4075,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.050432,170.327,187.127,2.3391,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.146656,937.152,257.396,3.2175,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,1.627584,1351.097,92.772,1.1597,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.535168,256.815,66.618,0.8327,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.04304,263.535,66.35,0.8294,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.540288,508.762,69.868,0.8733,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.043168,527.006,68.353,0.8544,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.268256,0.473,29.59,0.3699,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,27.705088,4.961,310.087,3.8761,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.267008,0.71,29.608,0.3701,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,27.718336,7.438,309.958,3.8745,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.269728,0.946,29.574,0.3697,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,27.70912,9.92,310.08,3.876,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.15904,54.011,59.338,0.7417,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.31344,438.486,120.434,1.5054,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.455744,895.461,61.486,0.7686,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.785152,175.048,45.407,0.5676,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.431584,192.01,48.342,0.6043,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.765344,359.156,49.323,0.6165,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.435008,383.103,49.688,0.6211,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.437408,0.747,46.693,0.5837,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,44.91312,3.06,191.28,2.391,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.429024,1.127,46.97,0.5871,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,44.879425,4.594,191.435,2.3929,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.439424,1.492,46.633,0.5829,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,44.934113,6.117,191.214,2.3902,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.050176,85.598,94.041,1.1755,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.093824,732.43,201.168,2.5146,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,0.838208,1311.741,90.07,1.1259,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.540096,127.236,33.005,0.4126,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.03968,132.194,33.282,0.416,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.538048,255.44,35.079,0.4385,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.040128,264.273,34.276,0.4285,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.266912,0.237,14.804,0.185,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,14.043616,4.893,305.868,3.8233,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.267872,0.355,14.798,0.185,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,14.058816,7.332,305.556,3.8194,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.266784,0.474,14.806,0.1851,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,14.060736,9.775,305.533,3.8192,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.142688,30.1,33.069,0.4134,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.237824,288.951,79.363,0.992,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.274496,862.703,59.237,0.7405,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.747616,91.918,23.844,0.298,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.384064,99.301,25.001,0.3125,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.75328,182.454,25.056,0.3132,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.395168,197.021,25.554,0.3194,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.383232,0.388,24.261,0.3033,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,17.878719,3.844,240.257,3.0032,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.38208,0.583,24.283,0.3035,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,17.87808,5.766,240.28,3.0035,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.380064,0.778,24.32,0.304,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,17.852961,7.698,240.633,3.0079,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.051232,670.669,736.82,9.2102,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.116544,4717.152,1295.605,16.1951,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.376704,23350.145,1603.327,20.0416,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.032,17179.868,303.104,3.7888,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.039104,28117.626,248.039,3.1005,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.046528,23631.182,411.29,5.1411,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.042144,52178.797,454.074,5.6759,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.035104,122.35,8.518,0.1065,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.290912,1889.767,131.562,1.6445,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.036224,177.85,8.763,0.1095,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.291552,2828.428,139.366,1.7421,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.035008,245.371,9.594,0.1199,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.293024,3752.292,146.717,1.834,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.231392,148.491,163.138,2.0392,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.514336,1068.865,293.573,3.6697,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.834304,4795.33,329.269,4.1159,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.25856,2126.221,37.513,0.4689,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.419392,2621.68,23.127,0.2891,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.289184,3802.118,66.174,0.8272,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.444224,4950.258,43.079,0.5385,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.400288,10.73,0.747,0.0093,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,41.473888,13.255,0.923,0.0115,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.397056,16.226,0.799,0.01,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,41.456928,19.891,0.98,0.0123,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.410112,20.945,0.819,0.0102,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,41.441601,26.532,1.037,0.013,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.043232,397.388,436.583,5.4573,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.0744,3694.596,1014.751,12.6844,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.205984,21351.399,1466.084,18.3261,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.043168,6367.631,112.344,1.4043,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.041408,13276.561,117.119,1.464,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.03776,14559.211,253.397,3.1675,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.039552,27799.141,241.916,3.0239,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.033088,64.902,4.518,0.0565,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.158528,1733.939,120.714,1.5089,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.036224,88.925,4.382,0.0548,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.159328,2587.849,127.512,1.5939,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.035936,119.517,4.673,0.0584,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.15952,3446.313,134.753,1.6844,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.18064,95.106,104.486,1.3061,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.314272,874.65,240.23,3.0029,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.964512,4559.867,313.101,3.9138,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.186848,1471.131,25.955,0.3244,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.264864,2075.615,18.31,0.2289,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.189632,2899.067,50.457,0.6307,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.265312,4144.221,36.064,0.4508,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.249792,8.597,0.599,0.0075,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,21.261312,12.929,0.9,0.0113,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.249376,12.917,0.636,0.008,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,20.99552,19.638,0.968,0.0121,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.25184,17.054,0.667,0.0083,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,20.999777,26.179,1.024,0.0128,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.039136,219.489,241.138,3.0142,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.056672,2425.165,666.091,8.3261,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.122528,17947.108,1232.33,15.4041,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.037792,3636.721,64.163,0.802,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.040128,6850.028,60.427,0.7553,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.042176,6517.401,113.432,1.4179,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.040064,13721.94,119.412,1.4927,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.033856,31.715,2.208,0.0276,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.097024,1416.546,98.617,1.2327,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.035104,45.881,2.261,0.0283,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.094688,2177.239,107.279,1.341,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.033632,63.852,2.497,0.0312,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.095424,2880.595,112.633,1.4079,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.153344,56.017,61.543,0.7693,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.221152,621.468,170.691,2.1336,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.51408,4277.59,293.719,3.6715,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.154528,889.411,15.692,0.1961,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.184192,1492.344,13.165,0.1646,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.175776,1563.797,27.217,0.3402,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.191008,2878.182,25.047,0.3131,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.177248,6.058,0.422,0.0053,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,10.676896,12.873,0.896,0.0112,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.174208,9.245,0.456,0.0057,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,10.679424,19.304,0.951,0.0119,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.175744,12.219,0.478,0.006,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,10.675808,25.748,1.007,0.0126,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.038304,112.128,123.188,1.5398,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.043776,1569.798,431.158,5.3895,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.076128,14442.934,991.718,12.3965,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.038944,1764.572,31.132,0.3892,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.039776,3455.324,30.481,0.381,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.039616,3469.279,60.381,0.7548,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.049344,5570.645,48.477,0.606,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.035392,15.169,1.056,0.0132,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.063104,1088.988,75.813,0.9477,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.036704,21.941,1.081,0.0135,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.063072,1634.31,80.528,1.0066,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.037952,28.292,1.106,0.0138,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.062688,2192.428,85.725,1.0716,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.130624,32.88,36.123,0.4515,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.16144,425.666,116.913,1.4614,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.283552,3877.637,266.256,3.3282,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.131104,524.16,9.248,0.1156,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.130496,1053.204,9.291,0.1161,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.134784,1019.698,17.747,0.2218,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.136512,2013.581,17.523,0.219,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.124288,4.32,0.301,0.0038,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.25952,264.795,18.435,0.2304,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.142944,5.634,0.278,0.0035,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.257248,400.7,19.744,0.2468,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.124224,8.644,0.338,0.0042,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.257088,534.599,20.903,0.2613,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.067968,505.528,555.39,6.9424,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.465408,1181.234,324.436,4.0554,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,6.58144,1336.5,91.77,1.1471,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.540704,1016.741,263.742,3.2968,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.044768,1052.398,264.962,3.312,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.044736,1052.43,144.529,1.8066,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.039584,1078.172,139.839,1.748,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.4688,1.74,108.744,1.3593,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,106.176285,5.178,323.65,4.0456,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.465184,2.613,108.911,1.3614,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,106.158943,7.768,323.722,4.0465,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.469952,3.478,108.707,1.3588,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,106.159325,10.357,323.741,4.0468,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.202208,169.923,186.683,2.3335,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.863872,636.386,174.789,2.1849,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,9.953856,883.687,60.678,0.7585,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.88848,618.76,160.506,2.0063,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.688224,651.283,163.974,2.0497,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.544576,711.853,97.758,1.222,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.991168,735.172,95.352,1.1919,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.677568,2.56,160.034,2.0004,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,179.110107,3.069,191.859,2.3982,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.676512,3.843,160.145,2.0018,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,179.102905,4.604,191.879,2.3985,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.675456,5.127,160.255,2.0032,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,179.100128,6.139,191.893,2.3987,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.050848,337.867,371.192,4.6399,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.251264,1093.98,300.471,3.7559,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,3.20352,1372.879,94.268,1.1784,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.538272,510.667,132.467,1.6558,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.043968,526.602,132.583,1.6573,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.539232,1019.516,140.009,1.7501,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.04448,1052.688,136.533,1.7067,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.466496,0.871,54.423,0.6803,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,53.234592,5.164,322.759,4.0345,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.4648,1.307,54.464,0.6808,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,53.249249,7.743,322.69,4.0336,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.4656,1.742,54.449,0.6806,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,53.24445,10.325,322.739,4.0342,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.180352,95.257,104.653,1.3082,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.49648,553.654,152.065,1.9008,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,5.000576,879.508,60.391,0.7549,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.810976,338.947,87.923,1.099,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.52896,359.562,90.527,1.1316,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.815552,674.09,92.572,1.1572,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.535104,716.246,92.897,1.1612,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.52432,1.409,88.062,1.1008,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,90.342499,3.043,190.187,2.3773,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.519232,2.12,88.362,1.1045,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,90.035454,4.579,190.847,2.3856,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.52976,2.808,87.759,1.097,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,90.027519,6.107,190.876,2.3859,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.050656,169.574,186.299,2.3287,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.13712,1002.326,275.297,3.4412,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,1.623424,1354.559,93.01,1.1626,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.534176,257.292,66.741,0.8343,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.040608,264.151,66.505,0.8313,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.538592,510.364,70.088,0.8761,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.042304,527.443,68.409,0.8551,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.267648,0.474,29.598,0.37,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,27.706656,4.961,310.069,3.8759,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.26752,0.71,29.601,0.37,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,27.704416,7.441,310.113,3.8764,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.263392,0.949,29.657,0.3707,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,27.701504,9.923,310.165,3.8771,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.162272,52.935,58.157,0.727,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.322752,425.835,116.959,1.462,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.525056,870.881,59.799,0.7475,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.793952,173.107,44.904,0.5613,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.458848,188.421,47.439,0.593,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.78416,350.538,48.139,0.6017,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.4624,375.927,48.758,0.6095,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.452512,0.739,46.208,0.5776,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,45.214462,3.04,190.005,2.3751,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.456128,1.106,46.096,0.5762,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,45.252514,4.556,189.857,2.3732,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.455808,1.475,46.109,0.5764,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,45.22179,6.078,189.998,2.375,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.048352,88.827,97.588,1.2199,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.091584,750.344,206.088,2.5761,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,0.837504,1312.843,90.146,1.1268,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.53696,127.979,33.198,0.415,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.039072,132.271,33.302,0.4163,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.53888,255.046,35.025,0.4378,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.042336,263.713,34.204,0.4275,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.264128,0.237,14.822,0.1853,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,13.99712,4.91,306.884,3.836,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.26512,0.356,14.816,0.1852,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,13.999392,7.363,306.853,3.8357,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.264928,0.474,14.818,0.1852,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,13.99952,9.817,306.869,3.8359,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.136864,31.381,34.477,0.431,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.23536,291.976,80.194,1.0024,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.316384,835.251,57.352,0.7169,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.758368,90.615,23.505,0.2938,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.403616,97.918,24.653,0.3082,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.75744,181.452,24.919,0.3115,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.401472,196.135,25.439,0.318,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.392704,0.385,24.096,0.3012,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,18.063328,3.804,237.802,2.9725,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.393376,0.578,24.086,0.3011,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,18.048897,5.711,238.006,2.9751,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.399584,0.767,23.98,0.2998,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,18.061089,7.61,237.86,2.9733,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_context.png b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_context.png index dcd1b7697..4529e77c0 100755 Binary files a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_context.png and b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_context.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_generation.png b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_generation.png index 638219726..c1aff7ea8 100755 Binary files a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_generation.png and b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_generation.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260820_064028.csv b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260821_064605.csv similarity index 56% rename from benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260820_064028.csv rename to benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260821_064605.csv index 594b8b55e..9ed158afc 100644 --- a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260820_064028.csv +++ b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260821_064605.csv @@ -1,105 +1,105 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.051008,673.615,740.055,15.4178,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.148224,3708.953,1018.694,21.2228,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.52928,16618.978,1141.135,23.7736,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.027776,19792.477,349.198,7.275,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.02752,39953.185,352.447,7.3426,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.037152,29594.951,515.087,10.731,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.0376,58484.662,508.95,10.6031,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.023232,184.873,12.871,0.2681,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.342976,1602.899,111.591,2.3248,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.023008,280.009,13.797,0.2874,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.342944,2404.573,118.481,2.4684,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.022848,375.96,14.7,0.3063,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.342784,3207.593,125.419,2.6129,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.03456,497.103,546.133,11.3778,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.084448,3254.996,894.011,18.6252,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.274848,16001.74,1098.752,22.8907,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.02224,12359.618,218.06,4.5429,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.023232,23663.732,208.749,4.3489,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.027808,19769.7,344.083,7.1684,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.02752,39953.185,347.684,7.2434,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.022848,93.99,6.543,0.1363,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.186912,1470.627,102.382,2.133,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.023456,137.331,6.767,0.141,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.185728,2220.004,109.387,2.2789,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.023008,186.673,7.299,0.1521,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.186144,2953.39,115.479,2.4058,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.02656,323.416,355.316,7.4024,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.050848,2702.937,742.384,15.4663,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.148384,14819.813,1017.596,21.1999,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.022336,6153.248,108.562,2.2617,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.022112,12431.164,109.661,2.2846,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.022848,12030.72,209.389,4.3623,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.022848,24061.441,209.389,4.3623,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.022816,47.061,3.276,0.0683,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.104832,1311.04,91.272,1.9015,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.022592,71.291,3.513,0.0732,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.105056,1962.367,96.692,2.0144,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.022528,95.325,3.727,0.0777,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.104896,2620.48,102.462,2.1346,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.021984,195.368,214.638,4.4716,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.034816,1973.79,542.118,11.2941,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.084448,13019.984,894.011,18.6252,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.02288,3003.474,52.99,1.104,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.022656,6066.338,53.514,1.1149,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.022432,6126.915,106.636,2.2216,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.022912,11997.116,104.402,2.175,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.022592,23.764,1.654,0.0345,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.061632,1114.997,77.624,1.6172,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.024416,32.983,1.625,0.0339,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.062304,1654.456,81.52,1.6983,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.022656,47.393,1.853,0.0386,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.061184,2246.322,87.833,1.8298,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.111968,306.871,337.139,7.0237,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.210624,454.109,124.725,2.5984,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,19.163391,459.005,31.517,0.6566,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,1.15408,476.358,123.567,2.5743,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,2.281952,481.829,121.31,2.5273,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,2.288832,480.381,65.97,1.3744,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,4.5152,487.027,63.167,1.316,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.146272,3.747,234.21,4.8794,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,71.781631,7.659,478.729,9.9735,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.14432,5.63,234.624,4.888,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,72.056831,11.444,476.93,9.936,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.146304,7.494,234.232,4.8798,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,72.211678,15.226,475.936,9.9153,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.070464,243.811,267.858,5.5804,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.641312,428.618,117.723,2.4526,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,9.67904,454.389,31.2,0.65,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.588928,466.743,121.073,2.5224,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.14768,479.015,120.602,2.5125,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,1.151328,477.497,65.574,1.3661,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,2.28784,480.589,62.332,1.2986,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.14544,1.875,117.19,2.4415,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,36.227745,7.587,474.276,9.8808,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.145664,2.812,117.174,2.4411,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,36.336224,11.347,472.889,9.8519,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.145024,3.751,117.247,2.4426,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,36.338818,15.129,472.884,9.8518,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.050944,168.615,185.246,3.8593,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.353888,388.368,106.669,2.2223,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,4.754176,462.546,31.76,0.6617,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.582176,236.078,61.238,1.2758,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.144864,240.097,60.449,1.2594,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.587616,467.785,64.24,1.3383,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.1528,476.887,61.852,1.2886,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.14496,0.938,58.62,1.2212,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,18.065727,7.608,475.54,9.9071,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.144768,1.407,58.633,1.2215,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,18.052704,11.42,475.913,9.9148,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.145696,1.874,58.589,1.2206,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,18.039553,15.238,476.289,9.9227,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.036064,119.093,130.839,2.7258,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.210048,327.161,89.857,1.872,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,2.382912,461.415,31.683,0.6601,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.581856,118.104,30.636,0.6383,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.14496,120.038,30.222,0.6296,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.582336,236.013,32.411,0.6752,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.14448,240.177,31.151,0.649,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.145376,0.469,29.299,0.6104,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,9.0352,7.606,475.417,9.9045,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.145088,0.703,29.308,0.6106,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,9.053536,11.386,474.484,9.8851,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.145696,0.937,29.295,0.6103,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,9.046016,15.193,474.907,9.8939,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.051264,670.251,736.36,15.3408,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.148064,3712.961,1019.795,21.2457,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.532192,16528.045,1134.891,23.6436,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.027872,19724.305,347.995,7.2499,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.028384,38737.022,341.718,7.1191,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.037984,28946.706,503.805,10.4959,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.037184,59138.964,514.644,10.7217,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.02272,189.039,13.161,0.2742,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.344224,1597.087,111.186,2.3164,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.023648,272.431,13.424,0.2797,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.344352,2394.741,117.996,2.4583,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.022688,378.611,14.804,0.3084,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.344192,3194.472,124.906,2.6022,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.034656,495.726,544.621,11.3463,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.084384,3257.465,894.689,18.6394,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.276864,15885.224,1090.752,22.724,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.023136,11880.961,209.615,4.367,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.023808,23091.221,203.699,4.2437,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.028032,19611.723,341.333,7.1111,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.027872,39448.609,343.293,7.1519,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.023776,90.321,6.288,0.131,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.187264,1467.863,102.19,2.129,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.0232,138.846,6.841,0.1425,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.186432,2211.621,108.974,2.2703,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.023744,180.886,7.073,0.1473,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.185888,2957.457,115.638,2.4091,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.026368,325.771,357.903,7.4563,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.051328,2677.66,735.441,15.3217,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.148288,14829.408,1018.255,21.2136,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.022656,6066.338,107.028,2.2298,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.022464,12236.374,107.943,2.2488,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.023072,11913.917,207.356,4.3199,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.022656,24265.353,211.164,4.3992,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.022432,47.867,3.332,0.0694,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.10464,1313.446,91.44,1.905,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.02256,71.392,3.518,0.0733,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.105024,1962.965,96.722,2.015,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.02304,93.207,3.644,0.0759,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.104832,2622.08,102.525,2.1359,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.022176,193.676,212.779,4.4329,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.035008,1962.965,539.144,11.2322,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.084288,13044.699,895.708,18.6606,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.02256,3046.076,53.742,1.1196,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.02224,6179.809,54.515,1.1357,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.022944,5990.192,104.257,2.172,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.022336,12306.496,107.095,2.2311,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.023104,23.237,1.618,0.0337,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.062144,1105.81,76.985,1.6038,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.023168,34.759,1.713,0.0357,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.061472,1676.848,82.624,1.7213,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.023072,46.539,1.82,0.0379,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.061536,2233.472,87.33,1.8194,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.111712,307.574,337.911,7.0398,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.213664,452.972,124.412,2.5919,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,19.172735,458.781,31.502,0.6563,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,1.160256,473.823,122.909,2.5606,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,2.291168,479.891,120.822,2.5171,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,2.292288,479.657,65.871,1.3723,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,4.769664,461.044,59.797,1.2458,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.146496,3.746,234.164,4.8784,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,71.552414,7.683,480.262,10.0055,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.154656,5.58,232.523,4.8442,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,71.61203,11.515,479.892,9.9977,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.153856,7.445,232.699,4.8479,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,71.625954,15.351,479.828,9.9964,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.070784,242.708,266.647,5.5552,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.645568,425.792,116.947,2.4364,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,9.671168,454.759,31.226,0.6505,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.592352,464.045,120.373,2.5078,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.157152,475.094,119.614,2.492,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,1.156128,475.515,65.302,1.3605,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,2.295712,478.941,62.119,1.2941,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.1536,1.862,116.361,2.4242,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,36.043648,7.626,476.699,9.9312,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.153024,2.794,116.426,2.4255,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,36.046303,11.439,476.693,9.9311,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.154816,3.719,116.253,2.4219,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,36.043488,15.253,476.759,9.9325,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.051168,167.877,184.435,3.8424,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.35568,386.412,106.131,2.2111,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,4.75984,461.995,31.723,0.6609,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.586144,234.48,60.824,1.2672,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.151552,238.702,60.098,1.252,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.590592,465.428,63.917,1.3316,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.156096,475.528,61.676,1.2849,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.145344,0.937,58.6,1.2208,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,18.203039,7.55,471.953,9.8324,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.152704,1.397,58.229,1.2131,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,18.193056,11.332,472.241,9.8384,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.15328,1.862,58.204,1.2126,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,18.213087,15.092,471.75,9.8281,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.036576,117.426,129.008,2.6877,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.21088,325.87,89.503,1.8646,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,2.419264,454.482,31.207,0.6501,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.586304,117.208,30.404,0.6334,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.1592,118.564,29.851,0.6219,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.586304,234.416,32.192,0.6707,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.151584,238.695,30.959,0.645,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.153472,0.465,29.093,0.6061,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,9.126784,7.529,470.647,9.8051,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.145376,0.703,29.301,0.6104,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,9.123296,11.298,470.855,9.8095,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.152928,0.931,29.111,0.6065,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,9.10336,15.098,471.915,9.8316,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_context.png b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_context.png index c1b97232d..0303af5de 100755 Binary files a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_context.png and b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_context.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_generation.png b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_generation.png index 447bac34b..a07bad9c5 100755 Binary files a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_generation.png and b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_generation.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260820_071827.csv b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260821_065725.csv similarity index 65% rename from benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260820_071827.csv rename to benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260821_065725.csv index 4f8b6b13a..d856033c2 100644 --- a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260820_071827.csv +++ b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260821_065725.csv @@ -1,209 +1,209 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.0552,622.459,683.854,38.1615,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.182112,3018.779,829.132,46.2685,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.658144,13364.998,917.702,51.211,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.035232,15603.878,275.299,15.3627,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.0352,31236.126,275.549,15.3766,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.055616,19769.7,344.083,19.2011,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.058112,37841.122,329.304,18.3763,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.021312,201.528,14.03,0.7829,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.21584,2547.053,177.321,9.8952,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.02128,302.747,14.917,0.8324,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.216096,3816.053,188.029,10.4927,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.021312,403.056,15.76,0.8795,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.216384,5081.298,198.682,11.0872,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.11376,302.037,331.828,18.5172,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.390048,1409.457,387.119,21.6026,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.551104,5670.86,389.387,21.7292,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.270848,2029.758,35.811,1.9984,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.467072,2354.052,20.766,1.1588,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.293984,3740.039,65.094,3.6325,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.49056,4482.68,39.01,2.1769,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.454048,9.459,0.659,0.0367,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,51.77808,10.618,0.739,0.0412,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.45312,14.218,0.701,0.0391,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,51.776897,15.927,0.785,0.0438,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.452992,18.963,0.741,0.0414,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,51.775009,21.236,0.83,0.0463,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.036768,467.251,513.337,28.646,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.094176,2918.768,801.664,44.7357,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.347648,12650.861,868.666,48.4746,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.034592,7946.286,140.196,7.8234,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.03424,16055.953,141.637,7.9039,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.034944,15732.48,273.817,15.28,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.03584,30678.337,266.971,14.898,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.022368,96.007,6.684,0.373,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.11952,2299.849,160.111,8.9348,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.021664,148.69,7.326,0.4088,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.122944,3353.697,165.247,9.2214,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.02192,195.938,7.661,0.4275,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.122976,4470.432,174.797,9.7543,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.093568,183.608,201.718,11.2566,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.206368,1331.979,365.839,20.4151,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.79664,5520.745,379.08,21.154,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.132832,2069.365,36.51,2.0374,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.256192,2145.874,18.93,1.0564,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.162656,3379.868,58.825,3.2827,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.264096,4163.303,36.23,2.0218,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.249152,8.619,0.6,0.0335,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,25.495424,10.781,0.751,0.0419,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.250112,12.879,0.635,0.0354,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,25.580544,16.118,0.794,0.0443,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.250304,17.159,0.671,0.0374,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,25.581633,21.49,0.84,0.0469,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.028768,298.593,328.045,18.3061,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.059232,2320.35,637.303,35.5638,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.189056,11631.598,798.678,44.5691,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.03504,3922.345,69.202,3.8617,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.035072,7837.532,69.139,3.8582,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.035104,7830.387,136.284,7.6052,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.03824,14376.459,125.108,6.9815,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.023104,46.474,3.235,0.1806,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.072128,1905.487,132.657,7.4027,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.021248,75.801,3.735,0.2084,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.072352,2849.381,140.398,7.8347,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.021376,100.462,3.928,0.2192,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.072128,3810.974,149.012,8.3154,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.077792,110.422,121.313,6.7697,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.117248,1172.207,321.956,17.9663,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.401792,5473.039,375.804,20.9712,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.07424,1851.279,32.662,1.8227,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.131104,2096.64,18.495,1.0321,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.078176,3516.142,61.197,3.415,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.13232,4154.745,36.156,2.0176,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.110016,9.76,0.679,0.0379,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,12.685664,10.834,0.754,0.0421,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.110464,14.58,0.718,0.0401,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,12.687712,16.249,0.801,0.0447,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.11056,19.424,0.759,0.0424,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,12.687456,21.665,0.847,0.0473,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.022432,191.466,210.351,11.7383,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.036992,1857.685,510.228,28.4726,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.098272,11188.452,768.25,42.8711,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.035488,1936.415,34.164,1.9065,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.035584,3862.381,34.072,1.9013,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.035136,3911.628,68.08,3.7991,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.035008,7851.86,68.329,3.813,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.021728,24.709,1.72,0.096,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.045088,1524.119,106.106,5.9211,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.021024,38.304,1.887,0.1053,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.044736,2304.167,113.534,6.3356,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.021568,49.784,1.947,0.1086,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.04528,3035.313,118.683,6.6229,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.06128,70.088,77.001,4.2969,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.081728,840.832,230.941,12.8873,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.187328,5869.446,403.023,22.4901,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.061344,1120.231,19.764,1.1029,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.061664,2228.836,19.662,1.0972,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.06,2290.649,39.868,2.2248,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.060992,4506.786,39.219,2.1886,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.06128,8.761,0.61,0.034,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.086528,794.188,55.29,3.0854,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.060608,13.287,0.655,0.0365,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.086816,1187.33,58.504,3.2647,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.06128,17.522,0.685,0.0382,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.087424,1572.096,61.47,3.4302,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.16464,208.696,229.28,12.7947,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.699296,323.52,88.857,4.9586,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,25.621632,343.307,23.573,1.3155,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,2.339456,234.993,60.957,3.4016,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,4.650784,236.414,59.522,3.3215,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,4.642176,236.853,32.527,1.8151,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,9.268864,237.248,30.771,1.7171,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.576288,1.667,104.207,5.8151,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,98.122307,5.603,350.215,19.5433,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.491904,2.585,107.743,6.0124,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,98.120224,8.404,350.244,19.5449,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.491424,3.448,107.77,6.014,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,98.121475,11.206,350.261,19.5458,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.18288,187.881,206.413,11.5186,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.637376,335.754,92.218,5.1461,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,23.680897,371.443,25.505,1.4233,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,2.114176,260.033,67.452,3.7641,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,4.367552,251.746,63.382,3.5369,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,3.826368,287.351,39.462,2.2021,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,7.482496,293.889,38.117,2.1271,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.595616,2.692,168.254,9.3892,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,136.28009,4.034,252.157,14.0712,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.58896,4.055,168.969,9.4291,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,136.529251,6.04,251.712,14.0464,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.588736,5.407,169.003,9.431,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,136.735367,8.041,251.348,14.0261,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.0976,176.023,193.385,10.7916,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.891616,308.292,84.675,4.7252,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,12.938528,339.919,23.34,1.3025,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,2.330528,117.947,30.595,1.7073,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,4.643264,118.399,29.809,1.6635,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,2.3344,235.502,32.341,1.8048,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,4.65152,236.377,30.658,1.7108,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.490464,0.862,53.899,3.0078,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,49.163105,5.591,349.489,19.5027,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.489728,1.294,53.918,3.0088,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,49.162434,8.387,349.515,19.5042,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.490144,1.725,53.913,3.0085,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,49.166527,11.182,349.507,19.5038,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.13376,128.438,141.106,7.8742,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.828128,331.927,91.166,5.0874,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,11.752384,374.226,25.696,1.4339,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,1.245856,220.634,57.232,3.1938,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,2.59936,211.497,53.249,2.9715,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,1.822944,301.576,41.415,2.3111,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,3.676992,299.025,38.783,2.1643,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.375904,1.561,97.561,5.4442,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,67.734749,4.058,253.665,14.1554,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.372288,2.347,97.824,5.4589,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,67.825027,6.079,253.343,14.1375,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.37104,3.133,97.919,5.4642,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,67.821983,8.106,253.37,14.139,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.068256,125.849,138.262,7.7155,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.503648,272.887,74.951,4.1825,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,6.361696,345.666,23.735,1.3245,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,2.330176,58.982,15.3,0.8538,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,4.643008,59.203,14.905,0.8318,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,2.32976,117.985,16.203,0.9042,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,4.642848,118.409,15.358,0.857,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.4904,0.431,26.95,1.5039,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,24.678305,5.569,348.119,19.4263,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.490432,0.647,26.952,1.504,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,24.675968,8.355,348.173,19.4293,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.491072,0.862,26.946,1.5037,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,24.676607,11.139,348.185,19.43,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.100576,85.407,93.831,5.2361,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.437504,314.143,86.282,4.8148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,5.526688,397.892,27.321,1.5246,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.63184,217.522,56.425,3.1487,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.239104,221.836,55.852,3.1167,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,1.194816,230.059,31.594,1.763,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,2.353696,233.571,30.294,1.6905,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.257664,0.854,53.366,2.978,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,33.867744,4.058,253.663,14.1553,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.26176,1.276,53.196,2.9686,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,33.874977,6.086,253.624,14.1531,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.254784,1.711,53.495,2.9852,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,33.876511,8.114,253.628,14.1533,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.053728,79.939,87.824,4.9009,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.303808,226.194,62.126,3.4669,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,3.346304,328.575,22.561,1.259,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,2.3296,29.498,7.652,0.427,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,4.642848,29.602,7.453,0.4159,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,2.330592,58.972,8.099,0.4519,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,4.644608,59.182,7.676,0.4283,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.569344,0.209,13.061,0.7289,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,13.526592,5.08,317.559,17.7209,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.519776,0.32,13.319,0.7432,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,13.524832,7.621,317.62,17.7243,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.568768,0.418,13.066,0.7291,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,13.526336,10.161,317.604,17.7234,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.072352,59.362,65.217,3.6394,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.27776,247.406,67.952,3.792,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,2.768672,397.126,27.268,1.5217,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.628544,109.331,28.36,1.5826,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.20256,114.289,28.774,1.6057,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.644096,213.383,29.304,1.6352,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.212288,226.743,29.409,1.6411,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.201312,0.447,27.935,1.5589,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,11.669408,5.889,368.099,20.5412,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.201696,0.67,27.928,1.5585,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,11.667264,8.835,368.189,20.5462,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.200032,0.895,27.968,1.5607,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,11.667552,11.78,368.202,20.547,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.05984,574.193,630.828,35.2024,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.195008,2819.145,774.301,43.2088,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.6984,12594.635,864.805,48.2592,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.036352,15123.124,266.817,14.8893,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.036768,29904.036,263.798,14.7209,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.06144,17895.698,311.467,17.381,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.059648,36866.672,320.824,17.9031,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.022496,190.921,13.292,0.7417,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.229504,2395.408,166.764,9.306,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.02368,272.063,13.405,0.7481,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.230944,3570.709,175.94,9.8181,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.023296,368.73,14.418,0.8046,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.230912,4761.605,186.182,10.3896,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.125056,274.755,301.855,16.8446,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.401728,1368.478,375.864,20.9745,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.604384,5482.536,376.456,21.0076,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.272256,2019.261,35.626,1.988,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.46992,2339.785,20.64,1.1518,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.297792,3692.214,64.261,3.586,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.4952,4440.677,38.644,2.1565,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.452576,9.49,0.661,0.0369,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,51.768738,10.619,0.739,0.0413,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.454112,14.187,0.699,0.039,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,51.77235,15.928,0.785,0.0438,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.453184,18.955,0.741,0.0414,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,51.774975,21.236,0.83,0.0463,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.03744,458.864,504.123,28.1319,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.100512,2734.777,751.129,41.9157,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.366432,12002.353,824.136,45.9897,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.035456,7752.648,136.78,7.6328,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.035648,15421.786,136.043,7.5917,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.03696,14874.346,258.881,14.4465,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.035456,31010.593,269.863,15.0593,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.02176,98.69,6.871,0.3834,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.125984,2181.848,151.896,8.4764,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.021472,150.02,7.392,0.4125,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.125824,3276.933,161.465,9.0103,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.021696,197.961,7.74,0.4319,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.125056,4396.077,171.889,9.592,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.094208,182.361,200.348,11.1801,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.209056,1314.853,361.135,20.1526,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.800288,5495.58,377.352,21.0576,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.133728,2055.5,36.265,2.0237,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.257312,2136.534,18.847,1.0518,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.163456,3363.326,58.537,3.2666,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.265824,4136.239,35.995,2.0086,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.250688,8.566,0.596,0.0333,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,25.494368,10.782,0.751,0.0419,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.24944,12.914,0.636,0.0355,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,25.577375,16.12,0.794,0.0443,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.249888,17.188,0.672,0.0375,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,25.578465,21.493,0.84,0.0469,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.029152,294.66,323.723,18.0649,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.060864,2258.132,620.214,34.6102,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.193504,11364.226,780.319,43.5446,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.035808,3838.219,67.718,3.7789,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.035776,7683.305,67.778,3.7823,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.03536,7773.696,135.298,7.5501,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.035776,15366.609,133.725,7.4623,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.021824,49.2,3.425,0.1911,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.073376,1873.078,130.4,7.2768,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.021888,73.584,3.626,0.2023,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.073088,2820.688,138.984,7.7558,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.022176,96.838,3.786,0.2113,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.073024,3764.213,147.183,8.2133,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.080704,106.438,116.936,6.5254,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.117792,1166.794,320.469,17.8833,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.39712,5537.428,380.225,21.2179,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.072992,1882.932,33.221,1.8538,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.131168,2095.617,18.486,1.0316,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.079872,3441.48,59.897,3.3425,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.134016,4102.165,35.698,1.9921,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.110656,9.703,0.676,0.0377,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,12.685536,10.834,0.754,0.0421,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.111328,14.467,0.713,0.0398,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,12.687904,16.248,0.801,0.0447,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.111648,19.234,0.752,0.042,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,12.688192,21.664,0.847,0.0473,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.021856,196.512,215.895,12.0477,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.037024,1856.079,509.787,28.448,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.100224,10970.542,753.287,42.0361,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.03552,1934.67,34.133,1.9048,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.035712,3848.537,33.95,1.8945,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.035232,3900.969,67.895,3.7888,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.035072,7837.532,68.204,3.806,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.02224,24.14,1.681,0.0938,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.045504,1510.185,105.136,5.867,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.021888,36.792,1.813,0.1012,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.046112,2235.41,110.146,6.1465,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.022368,48.003,1.877,0.1047,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.045984,2988.843,116.866,6.5215,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.060832,70.604,77.568,4.3285,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.08288,829.144,227.731,12.7082,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.188928,5819.739,399.61,22.2997,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.059936,1146.548,20.229,1.1288,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.06096,2254.576,19.889,1.1099,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.0608,2260.509,39.343,2.1955,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.061664,4457.672,38.792,2.1647,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.061056,8.793,0.612,0.0342,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.086688,792.722,55.188,3.0797,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.061152,13.169,0.649,0.0362,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.087264,1181.234,58.203,3.2479,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.061408,17.485,0.684,0.0382,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.08736,1573.248,61.515,3.4328,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.166432,206.449,226.812,12.6569,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.682176,326.812,89.762,5.009,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,25.690111,342.392,23.51,1.312,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,2.341728,234.765,60.898,3.3983,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,4.651488,236.378,59.513,3.321,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,4.640064,236.96,32.542,1.8159,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,9.268288,237.263,30.773,1.7172,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.621056,1.639,102.428,5.7158,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,98.135262,5.602,350.169,19.5407,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.491808,2.585,107.747,6.0127,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,98.121117,8.404,350.241,19.5447,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.491616,3.448,107.762,6.0135,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,98.121666,11.206,350.26,19.5458,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.18256,188.211,206.774,11.5388,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.63296,336.662,92.467,5.16,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,23.712671,370.945,25.471,1.4214,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,2.123232,258.924,67.165,3.748,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,4.360896,252.13,63.479,3.5423,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,3.8344,286.749,39.379,2.1975,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,7.495808,293.367,38.05,2.1233,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.641312,2.617,163.569,9.1278,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,136.20752,4.036,252.291,14.0787,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.595872,4.037,168.237,9.3882,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,136.42041,6.045,251.913,14.0576,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.588864,5.406,168.989,9.4302,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,136.536774,8.053,251.713,14.0465,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.096832,177.419,194.919,10.8772,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.89104,308.491,84.73,4.7282,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,12.95104,339.59,23.318,1.3012,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,2.331328,117.906,30.585,1.7067,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,4.645376,118.345,29.796,1.6627,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,2.335776,235.363,32.322,1.8037,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,4.651744,236.365,30.657,1.7107,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.490496,0.862,53.899,3.0077,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,49.177887,5.589,349.384,19.4969,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.49152,1.293,53.88,3.0067,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,49.174946,8.385,349.426,19.4992,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.489984,1.725,53.916,3.0087,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,49.175678,11.179,349.442,19.5001,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.133984,128.223,140.87,7.8611,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.831456,330.598,90.802,5.067,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,11.719104,375.289,25.769,1.438,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,1.32672,207.186,53.744,2.9991,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,2.597024,211.687,53.296,2.9741,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,1.886624,291.397,40.017,2.2331,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,3.67808,298.936,38.772,2.1636,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.427744,1.504,94.018,5.2466,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,67.681885,4.061,253.864,14.1665,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.430368,2.252,93.852,5.2373,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,67.771011,6.084,253.545,14.1487,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.425792,3.012,94.159,5.2544,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,67.772324,8.112,253.556,14.1493,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.07344,116.965,128.502,7.1709,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.504384,272.489,74.841,4.1764,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,6.368768,345.282,23.709,1.323,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,2.375232,57.863,15.01,0.8376,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,4.642144,59.214,14.908,0.8319,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,2.33808,117.566,16.145,0.901,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,4.6432,118.4,15.356,0.8569,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.608896,0.412,25.726,1.4356,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,24.683617,5.568,348.044,19.4221,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.60848,0.617,25.732,1.4359,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,24.673088,8.356,348.214,19.4316,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.608544,0.823,25.733,1.436,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,24.684736,11.136,348.071,19.4236,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.10272,83.625,91.873,5.1268,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.461888,297.559,81.727,4.5607,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,5.559552,395.54,27.16,1.5156,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.672096,204.493,53.045,2.9601,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.309728,209.874,52.84,2.9487,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,1.276544,215.33,29.571,1.6502,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,2.352384,233.702,30.311,1.6915,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.278496,0.84,52.497,2.9295,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,33.869473,4.058,253.65,14.1546,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.28032,1.258,52.425,2.9255,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,33.868351,6.087,253.674,14.1559,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.27824,1.68,52.514,2.9305,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,33.868286,8.116,253.69,14.1568,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.054944,78.17,85.88,4.7924,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.324416,211.825,58.18,3.2466,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,3.349984,328.214,22.537,1.2576,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,2.438912,28.176,7.309,0.4079,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,4.86048,28.277,7.119,0.3973,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,2.434976,56.444,7.751,0.4326,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,4.853056,56.64,7.346,0.4099,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.602784,0.206,12.893,0.7195,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,13.523616,5.081,317.629,17.7248,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.60192,0.31,12.898,0.7198,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,13.525216,7.621,317.611,17.7238,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.602304,0.413,12.897,0.7197,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,13.525856,10.161,317.615,17.7241,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.072832,58.971,64.787,3.6154,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.290944,236.195,64.873,3.6201,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,2.785056,394.79,27.108,1.5127,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.635872,108.071,28.034,1.5644,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.22352,112.331,28.282,1.5782,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.651296,211.024,28.98,1.6172,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.234912,222.589,28.87,1.611,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.216096,0.441,27.595,1.5399,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,11.66816,5.889,368.138,20.5434,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.216736,0.662,27.582,1.5392,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,11.669152,8.833,368.129,20.5429,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.215488,0.883,27.612,1.5409,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,11.6712,11.776,368.087,20.5406,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], diff --git a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_context.png b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_context.png index 360ddacd4..3014e1666 100755 Binary files a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_context.png and b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_context.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_generation.png b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_generation.png index bf95cc312..b2941dc3a 100755 Binary files a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_generation.png and b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_generation.png differ diff --git a/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260819_054736.csv b/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260821_054645.csv similarity index 58% rename from benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260819_054736.csv rename to benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260821_054645.csv index 8658064bc..dfcc0f7f0 100644 --- a/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260819_054736.csv +++ b/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260821_054645.csv @@ -1,33 +1,33 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.648672,42.21,233.098,2.9137,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,68.220062,51.373,283.701,3.5463,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.567425,1.917,5.3,0.0663,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2383.392578,2.941,8.132,0.1016,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.546368,2.877,5.311,0.0664,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2381.427246,4.415,8.15,0.1019,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.535263,3.838,5.321,0.0665,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2381.261963,5.887,8.161,0.102,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.361344,37.887,418.159,5.227,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,37.635426,46.561,513.897,6.4237,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,12.794368,2.14,11.818,0.1477,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1133.383667,3.092,17.076,0.2135,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,12.789312,3.211,11.831,0.1479,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1133.621582,4.637,17.085,0.2136,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.786016,4.283,11.842,0.148,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1132.897705,6.187,17.107,0.2138,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.649984,42.125,232.627,2.9078,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,68.897156,50.868,280.913,3.5114,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.755808,1.904,5.265,0.0658,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2405.72168,2.914,8.056,0.1007,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.748672,2.857,5.274,0.0659,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2402.599121,4.376,8.078,0.101,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.741472,3.811,5.283,0.066,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2402.205322,5.836,8.09,0.1011,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.36288,37.727,416.389,5.2049,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,37.985214,46.132,509.164,6.3646,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,12.84784,2.131,11.769,0.1471,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1143.999023,3.064,16.918,0.2115,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,12.842944,3.198,11.781,0.1473,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1143.258179,4.598,16.941,0.2118,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.840768,4.265,11.792,0.1474,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1140.824829,6.144,16.988,0.2124,20,True,,NVIDIA GB200,cudnn 9.26.0, kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.26288,26.039,574.586,7.1823,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,26.631359,32.9,725.988,9.0748,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.772544,1.072,11.83,0.1479,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,560.801575,3.125,34.488,0.4311,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.756352,1.61,11.849,0.1481,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,560.475159,4.69,34.52,0.4315,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.751584,2.147,11.858,0.1482,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,560.293884,6.255,34.543,0.4318,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.156416,21.881,965.509,12.0689,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,14.005152,31.28,1380.256,17.2532,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.738272,0.537,11.858,0.1482,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,285.479218,3.069,67.725,0.8466,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.726592,0.807,11.871,0.1484,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,284.811493,4.614,67.895,0.8487,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.723712,1.076,11.875,0.1484,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,284.934418,6.15,67.878,0.8485,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,26.857952,32.622,719.863,8.9983,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.83376,1.067,11.774,0.1472,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,565.198364,3.1,34.219,0.4277,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.822624,1.601,11.788,0.1473,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,564.968872,4.653,34.245,0.4281,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.817728,2.136,11.796,0.1475,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,564.444397,6.209,34.289,0.4286,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.1552,22.053,973.074,12.1634,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,14.157088,30.945,1365.443,17.068,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.811904,0.534,11.79,0.1474,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,287.494598,3.048,67.25,0.8406,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.806144,0.802,11.797,0.1475,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,287.070648,4.578,67.361,0.842,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.79888,1.07,11.806,0.1476,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,286.759735,6.111,67.446,0.8431,20,True,,NVIDIA GB200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_generation.png b/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_generation.png index 800bf3804..4015a524e 100755 Binary files a/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_generation.png and b/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_generation.png differ diff --git a/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260819_054557.csv b/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260821_054734.csv similarity index 56% rename from benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260819_054557.csv rename to benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260821_054734.csv index 4e2bf98fd..23bbad9ab 100644 --- a/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260819_054557.csv +++ b/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260821_054734.csv @@ -1,33 +1,33 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.636096,43.044,237.706,2.9713,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,66.632004,52.598,290.462,3.6308,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.157185,1.945,5.377,0.0672,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2345.772949,2.988,8.262,0.1033,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.141985,2.919,5.388,0.0673,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2343.797363,4.486,8.28,0.1035,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.13792,3.892,5.396,0.0674,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2343.576904,5.982,8.293,0.1037,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.354816,38.584,425.853,5.3232,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,36.863167,47.537,524.663,6.5583,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,12.81632,2.136,11.798,0.1475,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1117.765869,3.135,17.315,0.2164,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,12.809056,3.206,11.813,0.1477,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1118.273682,4.701,17.319,0.2165,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.803968,4.277,11.825,0.1478,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1115.760986,6.282,17.37,0.2171,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.257344,26.599,586.947,7.3368,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,26.157087,33.497,739.151,9.2394,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.725344,1.076,11.874,0.1484,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,553.244812,3.167,34.959,0.437,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.721536,1.614,11.882,0.1485,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,552.736938,4.755,35.003,0.4375,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.71392,2.154,11.893,0.1487,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,552.290161,6.346,35.043,0.438,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.15328,22.329,985.263,12.3158,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,13.54928,32.333,1426.695,17.8337,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.709504,0.539,11.885,0.1486,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,282.027161,3.107,68.554,0.8569,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.695456,0.809,11.9,0.1487,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,281.535736,4.668,68.685,0.8586,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.690208,1.079,11.907,0.1488,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,281.346924,6.228,68.743,0.8593,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.634208,43.173,238.414,2.9802,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,66.822273,52.448,289.635,3.6204,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.205503,1.941,5.368,0.0671,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2350.959961,2.981,8.244,0.103,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.198433,2.913,5.377,0.0672,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2349.150146,4.476,8.262,0.1033,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.180927,3.886,5.388,0.0673,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2349.692383,5.966,8.271,0.1034,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.351904,38.903,429.377,5.3672,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,36.877502,47.518,524.459,6.5557,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,12.812608,2.137,11.801,0.1475,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1120.248291,3.128,17.277,0.216,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,12.806144,3.207,11.815,0.1477,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1119.706299,4.695,17.297,0.2162,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.8,4.278,11.829,0.1479,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1119.210693,6.263,17.317,0.2165,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.25696,26.639,587.824,7.3478,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,26.131201,33.53,739.883,9.2485,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.737216,1.075,11.863,0.1483,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,554.608582,3.16,34.873,0.4359,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.723776,1.614,11.879,0.1485,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,553.816895,4.746,34.935,0.4367,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.72192,2.152,11.885,0.1486,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,553.230347,6.335,34.984,0.4373,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.150976,22.67,1000.298,12.5037,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,13.55824,32.311,1425.753,17.8219,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.706944,0.539,11.887,0.1486,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,282.15744,3.105,68.522,0.8565,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.697696,0.809,11.898,0.1487,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,281.924072,4.662,68.591,0.8574,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.695168,1.078,11.902,0.1488,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,281.685699,6.221,68.661,0.8583,20,True,,NVIDIA GB300,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_generation.png b/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_generation.png index 85889d9e7..d75c037f7 100755 Binary files a/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_generation.png and b/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_generation.png differ diff --git a/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260820_064856.csv b/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260821_065437.csv similarity index 54% rename from benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260820_064856.csv rename to benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260821_065437.csv index 4ef289e62..e71b61cd6 100644 --- a/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260820_064856.csv +++ b/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260821_065437.csv @@ -1,33 +1,33 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.669408,40.902,225.877,4.7058,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,80.073692,43.768,241.703,5.0355,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.842144,1.899,5.25,0.1094,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2808.916992,2.495,6.9,0.1437,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.823488,2.85,5.26,0.1096,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2810.880615,3.74,6.904,0.1438,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.823809,3.8,5.268,0.1097,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2811.748047,4.986,6.912,0.144,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.382528,35.789,395.002,8.2292,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,42.385761,41.343,456.302,9.5063,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,12.9888,2.108,11.641,0.2425,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1340.370972,2.615,14.439,0.3008,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,12.972224,3.166,11.664,0.243,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1339.683228,3.924,14.457,0.3012,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.9736,4.221,11.671,0.2431,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1340.081299,5.231,14.462,0.3013,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.268928,25.453,561.664,11.7013,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,27.414881,31.96,705.239,14.6925,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.753664,1.073,11.848,0.2468,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,652.695801,2.685,29.632,0.6173,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.757792,1.61,11.848,0.2468,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,652.729797,4.027,29.641,0.6175,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.72288,2.152,11.884,0.2476,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,651.695557,5.378,29.698,0.6187,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.153984,22.227,980.758,20.4325,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,14.853472,29.494,1301.426,27.113,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.697664,0.539,11.896,0.2478,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,330.326904,2.652,58.53,1.2194,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.690144,0.809,11.905,0.248,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,330.246246,3.98,58.554,1.2199,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.655776,1.082,11.939,0.2487,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,330.293457,5.305,58.556,1.2199,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.668896,40.934,226.05,4.7094,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,80.215172,43.691,241.277,5.0266,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.944481,1.892,5.231,0.109,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2809.725586,2.495,6.898,0.1437,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.810783,2.851,5.263,0.1096,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2811.967041,3.739,6.902,0.1438,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.803713,3.802,5.271,0.1098,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2812.984619,4.984,6.909,0.1439,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.382432,35.798,395.101,8.2313,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,42.444736,41.285,455.668,9.4931,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,13.072064,2.095,11.567,0.241,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1339.674194,2.616,14.447,0.301,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,12.983328,3.163,11.654,0.2428,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1339.75061,3.924,14.456,0.3012,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.96704,4.223,11.677,0.2433,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1340.110107,5.23,14.462,0.3013,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.26912,25.435,561.263,11.693,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,27.423361,31.95,705.021,14.6879,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.794624,1.07,11.81,0.246,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,652.920715,2.684,29.622,0.6171,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.72096,1.614,11.882,0.2475,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,653.14093,4.024,29.622,0.6171,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.703552,2.155,11.902,0.248,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,652.315125,5.373,29.67,0.6181,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.153568,22.287,983.415,20.4878,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,14.882816,29.436,1298.86,27.0596,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.731808,0.538,11.864,0.2472,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,330.546661,2.651,58.491,1.2186,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.658272,0.811,11.935,0.2486,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,330.33197,3.979,58.539,1.2196,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.634368,1.084,11.959,0.2492,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,330.612183,5.3,58.5,1.2187,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_generation.png b/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_generation.png index a766fa5e7..e6bffd9f6 100755 Binary files a/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_generation.png and b/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_generation.png differ diff --git a/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260820_072507.csv b/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260821_070405.csv similarity index 57% rename from benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260820_072507.csv rename to benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260821_070405.csv index d09ebbc6f..b3c6f4bf1 100644 --- a/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260820_072507.csv +++ b/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260821_070405.csv @@ -1,33 +1,33 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,1.151872,23.77,131.268,7.3252,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,119.810783,29.252,161.539,9.0144,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,27.359552,2.002,5.534,0.3088,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,1932.781128,3.627,10.027,0.5596,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,27.314337,3.007,5.551,0.3098,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2054.050781,5.119,9.448,0.5273,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,27.33024,4.007,5.555,0.31,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2078.880127,6.743,9.348,0.5217,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.564224,24.264,267.8,14.9442,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,62.386177,28.089,310.016,17.3,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,15.56592,1.759,9.714,0.5421,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,979.33606,3.579,19.762,1.1028,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,15.579072,2.636,9.712,0.542,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1019.31073,5.157,19.001,1.0603,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,15.591872,3.512,9.711,0.5419,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1030.145752,6.804,18.814,1.0499,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.366208,18.692,412.463,23.0169,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,41.304031,21.213,468.091,26.1211,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,13.053472,1.049,11.575,0.6459,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,489.444824,3.58,39.516,2.2051,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,13.053152,1.573,11.58,0.6462,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,495.111389,5.309,39.077,2.1806,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,13.057792,2.097,11.58,0.6462,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,500.069366,7.008,38.703,2.1598,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.25296,13.53,597.016,33.3156,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,21.416384,20.456,902.612,50.369,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,13.138624,0.521,11.496,0.6415,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,245.828186,3.564,78.649,4.3889,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,13.13456,0.782,11.502,0.6419,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,246.982529,5.321,78.295,4.3691,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,13.137088,1.042,11.502,0.6418,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,248.327942,7.057,77.884,4.3462,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,1.1616,23.571,130.169,7.2639,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,119.470116,29.335,161.999,9.0401,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,27.268768,2.008,5.553,0.3099,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,1921.90271,3.647,10.084,0.5627,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,27.286337,3.01,5.557,0.3101,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,1978.372681,5.315,9.81,0.5474,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,27.288383,4.013,5.564,0.3105,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2010.513184,6.973,9.666,0.5394,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.564768,24.24,267.542,14.9298,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,62.387745,28.088,310.008,17.2996,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,15.599552,1.755,9.693,0.5409,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,965.407715,3.63,20.048,1.1187,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,15.615904,2.63,9.689,0.5407,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,974.120422,5.397,19.882,1.1095,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,15.624544,3.505,9.691,0.5408,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,988.40564,7.092,19.608,1.0942,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.365632,18.721,413.113,23.0532,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,41.292671,21.219,468.22,26.1283,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,13.032576,1.05,11.594,0.647,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,485.580414,3.609,39.83,2.2227,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,13.037728,1.575,11.593,0.647,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,488.686249,5.379,39.591,2.2093,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,13.04,2.1,11.595,0.6471,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,491.256989,7.134,39.397,2.1985,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.25184,13.59,599.671,33.4638,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,21.398815,20.472,903.354,50.4104,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,13.156096,0.52,11.481,0.6407,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,244.568542,3.583,79.054,4.4115,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,13.159456,0.78,11.48,0.6406,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,245.939011,5.344,78.627,4.3877,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,13.158432,1.04,11.483,0.6408,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,246.881058,7.098,78.34,4.3717,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_generation.png b/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_generation.png index c07ee5e6c..3b26c9171 100755 Binary files a/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_generation.png and b/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_generation.png differ diff --git a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260819_053557.csv b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260821_053414.csv similarity index 59% rename from benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260819_053557.csv rename to benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260821_053414.csv index 314c6a6cf..0caba9d7d 100644 --- a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260819_053557.csv +++ b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260821_053414.csv @@ -1,148 +1,148 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.09648,712.267,782.519,9.7815,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,0.75744,1451.616,398.698,4.9837,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,11.916,1476.35,101.373,1.2672,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,0.754816,1456.662,377.857,4.7232,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.403264,1567.077,394.543,4.9318,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.470016,1495.918,205.433,2.5679,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,2.849952,1543.2,200.153,2.5019,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.103584,41.464,5183.269,64.7909,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,64,8,128,128,gqa,,16,0.099456,43.185,2699.202,33.74,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,8.922688,61.613,7702.127,96.2766,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.210016,3.887,242.956,3.0369,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,1.261952,6.807,212.74,2.6593,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,84.338844,13.037,814.902,10.1863,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.210016,5.83,242.971,3.0371,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,1.262528,10.206,212.656,2.6582,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,84.329948,19.557,815.037,10.188,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.207936,7.781,243.214,3.0402,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,1.262176,13.611,212.729,2.6591,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,84.35453,26.069,814.85,10.1856,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.226304,303.66,333.611,4.1701,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.167552,941.724,258.652,3.2332,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,13.050848,1347.973,92.558,1.157,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.173568,936.896,243.03,3.0379,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,2.258496,973.667,245.14,3.0643,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.975168,1113.335,152.893,1.9112,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,3.840672,1145.124,148.522,1.8565,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,2.19984,1.952,244.065,3.0508,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,248.644196,2.211,276.394,3.4549,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.23712,3.84,240.012,3.0002,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,250.610489,4.387,274.242,3.428,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.239552,5.753,239.766,2.9971,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,250.605057,6.581,274.264,3.4283,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.228992,7.707,240.917,3.0115,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,250.700989,8.771,274.176,3.4272,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.068896,498.719,547.909,6.8489,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.418432,1313.847,360.859,4.5107,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,5.864704,1499.836,102.986,1.2873,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.720896,762.601,197.818,2.4727,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.375136,799.566,201.307,2.5163,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.747168,1471.572,202.09,2.5261,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.45312,1513.312,196.276,2.4535,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.069664,30.826,3853.523,48.169,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,32,4,128,128,gqa,,16,0.07632,28.138,1758.725,21.9841,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,4.485568,61.281,7660.532,95.7566,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,32,4,128,128,gqa,,16,2.283168,120.393,7525.034,94.0629,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.209056,1.944,121.531,1.5191,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,1.260352,3.408,106.505,1.3313,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,42.363457,12.977,811.169,10.1396,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,32,4,128,128,gqa,,16,32.85968,16.73,522.889,6.5361,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.205408,2.921,121.739,1.5217,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,1.264032,5.097,106.202,1.3275,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,42.388863,19.454,810.733,10.1342,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,32,4,128,128,gqa,,16,32.891041,25.072,522.422,6.5303,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.207808,3.891,121.614,1.5202,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,1.2624,6.804,106.345,1.3293,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,42.401344,25.931,810.543,10.1318,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,32,4,128,128,gqa,,16,32.857185,33.463,522.993,6.5374,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.183584,187.161,205.621,2.5703,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.626976,876.837,240.83,3.0104,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,6.7016,1312.536,90.125,1.1266,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,1.006752,546.069,141.65,1.7706,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.937024,567.629,142.912,1.7864,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,1.016,1082.196,148.617,1.8577,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.9504,1127.473,146.233,1.8279,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,1.893856,1.134,141.749,1.7719,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,124.524063,2.207,275.945,3.4493,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.928608,2.227,139.203,1.74,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,125.620255,4.376,273.554,3.4194,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.936832,3.326,138.62,1.7328,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,125.605507,6.565,273.603,3.42,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.93072,4.449,139.068,1.7383,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,125.618019,8.753,273.592,3.4199,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.057312,299.76,329.327,4.1166,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.2512,1094.259,300.547,3.7568,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.686816,1636.899,112.397,1.405,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.707968,388.263,100.715,1.2589,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.369504,401.427,101.067,1.2633,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.717792,765.899,105.18,1.3148,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.379936,796.785,103.343,1.2918,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.052544,20.435,2554.543,31.9318,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,16,2,128,128,gqa,,16,0.064128,16.744,1046.547,13.0818,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,2.249024,61.11,7639.277,95.491,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,16,2,128,128,gqa,,16,1.170112,117.458,7341.57,91.7696,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.193568,0.979,61.194,0.7649,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,1.253248,1.714,53.554,0.6694,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,21.823008,12.596,787.333,9.8417,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,16.941792,16.225,507.088,6.3386,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.19408,1.468,61.184,0.7648,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,1.251232,2.574,53.644,0.6706,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,21.819263,18.897,787.516,9.8439,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,16.962433,24.308,506.502,6.3313,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.194368,1.957,61.18,0.7647,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,1.2504,3.435,53.683,0.671,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,21.814367,25.202,787.741,9.8468,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,16.94368,32.446,507.094,6.3387,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.15728,109.231,120.005,1.5001,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.362112,759.096,208.492,2.6062,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,3.192224,1377.737,94.602,1.1825,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.929728,295.654,76.693,0.9587,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.784032,308.154,77.584,0.9698,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.928256,592.246,81.333,1.0167,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.788,614.939,79.757,0.997,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,1.736992,0.618,77.275,0.9659,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,62.27824,2.207,275.874,3.4484,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.776192,1.209,75.574,0.9447,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,62.865696,4.372,273.312,3.4164,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.777152,1.813,75.538,0.9442,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,62.856255,6.56,273.37,3.4171,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.774752,2.42,75.645,0.9456,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,62.835838,8.749,273.476,3.4184,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.056288,152.607,167.659,2.0957,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.16352,840.502,230.851,2.8856,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.495552,1470.376,100.963,1.262,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.702144,195.742,50.775,0.6347,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.36352,201.594,50.755,0.6344,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.70928,387.545,53.221,0.6653,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.374016,400.109,51.894,0.6487,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.049824,10.775,1347.001,16.8375,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,8,1,128,128,gqa,,16,0.062112,8.644,540.258,6.7532,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.135904,60.498,7562.663,94.5333,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.861248,0.577,36.06,0.4508,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,1.250816,0.858,26.829,0.3354,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,11.40208,12.054,753.458,9.4182,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,8.510592,16.149,504.723,6.309,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.863136,0.864,36.026,0.4503,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,1.252544,1.286,26.794,0.3349,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,11.409408,18.069,753.02,9.4127,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,8.514144,24.214,504.543,6.3068,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.864672,1.152,35.998,0.45,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,1.244256,1.726,26.974,0.3372,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,11.402304,24.107,753.535,9.4192,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,8.513312,32.288,504.623,6.3078,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.12544,68.478,75.233,0.9404,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.268704,511.488,140.484,1.7561,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.5912,1381.99,94.894,1.1862,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.864864,158.914,41.222,0.5153,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.643104,167.292,42.119,0.5265,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.871488,315.412,43.315,0.5414,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.650592,333.066,43.199,0.54,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,1.58288,0.339,42.399,0.53,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,21.464384,3.202,400.219,5.0027,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.642112,0.654,40.872,0.5109,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,21.783808,6.309,394.375,4.9297,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.642112,0.981,40.875,0.5109,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,21.788095,9.462,394.321,4.929,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.64144,1.308,40.894,0.5112,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,21.795551,12.612,394.21,4.9276,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.093536,734.685,807.149,10.0894,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,0.767328,1432.91,393.56,4.9195,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,11.949184,1472.25,101.091,1.2636,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,0.763072,1440.902,373.769,4.6721,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.422144,1546.273,389.305,4.8663,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.452736,1513.712,207.877,2.5985,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,2.9552,1488.24,193.024,2.4128,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.099744,43.06,5382.817,67.2852,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,64,8,128,128,gqa,,16,0.096288,44.605,2788.009,34.8501,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,8.916128,61.659,7707.793,96.3474,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.204736,3.896,243.538,3.0442,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,1.258176,6.827,213.379,2.6672,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,84.119743,13.071,817.024,10.2128,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.208128,5.835,243.178,3.0397,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,1.257024,10.25,213.587,2.6698,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,84.118942,19.606,817.082,10.2135,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.205216,7.791,243.514,3.0439,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,1.258048,13.656,213.427,2.6678,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,84.150017,26.132,816.83,10.2104,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.230752,297.807,327.18,4.0898,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.17312,937.254,257.425,3.2178,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,13.11616,1341.26,92.097,1.1512,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.18256,929.772,241.182,3.0148,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,2.255584,974.924,245.457,3.0682,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.973536,1114.255,153.02,1.9127,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,3.848,1142.943,148.239,1.853,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,2.195488,1.956,244.549,3.0569,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,249.138809,2.207,275.845,3.4481,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.237216,3.84,240.002,3.0,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,250.762054,4.385,274.076,3.426,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.240384,5.751,239.677,2.996,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,250.721756,6.578,274.137,3.4267,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.234752,7.688,240.296,3.0037,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,250.820923,8.767,274.045,3.4256,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.066336,517.965,569.054,7.1132,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.423232,1298.947,356.766,4.4596,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,6.060192,1451.454,99.663,1.2458,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.71184,772.303,200.335,2.5042,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.371872,801.468,201.786,2.5223,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.768288,1431.119,196.534,2.4567,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.436736,1530.569,198.514,2.4814,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.068224,31.477,3934.859,49.1857,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,32,4,128,128,gqa,,16,0.069344,30.969,1935.653,24.1957,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,4.47856,61.376,7672.519,95.9065,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,32,4,128,128,gqa,,16,2.285184,120.287,7518.396,93.98,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.196864,1.955,122.205,1.5276,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,1.256864,3.417,106.801,1.335,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,42.301121,12.996,812.365,10.1546,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,32,4,128,128,gqa,,16,32.913536,16.703,522.033,6.5254,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.200704,2.927,121.999,1.525,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,1.25664,5.127,106.826,1.3353,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,42.291489,19.499,812.599,10.1575,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,32,4,128,128,gqa,,16,32.90733,25.059,522.164,6.527,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.197952,3.908,122.16,1.527,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,1.261184,6.811,106.448,1.3306,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,42.301697,25.992,812.453,10.1557,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,32,4,128,128,gqa,,16,32.877216,33.443,522.674,6.5334,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.189664,181.161,199.03,2.4879,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.634528,866.401,237.964,2.9746,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,6.74976,1303.171,89.482,1.1185,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,1.008128,545.323,141.457,1.7682,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.945536,565.146,142.287,1.7786,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,1.01968,1078.291,148.081,1.851,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.949504,1127.991,146.3,1.8288,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,1.892032,1.135,141.885,1.7736,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,124.563362,2.207,275.858,3.4482,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.927712,2.228,139.268,1.7408,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,125.48259,4.381,273.854,3.4232,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.935616,3.328,138.708,1.7338,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,125.311203,6.581,274.245,3.4281,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.932,4.446,138.976,1.7372,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,125.425117,8.766,274.013,3.4252,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.053536,320.903,352.555,4.4069,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.253984,1082.265,297.253,3.7157,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.853504,1541.279,105.831,1.3229,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.704288,390.292,101.241,1.2655,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.370816,401.043,100.971,1.2621,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.70688,777.722,106.804,1.335,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.372928,800.852,103.87,1.2984,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.050336,21.331,2666.599,33.3325,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,16,2,128,128,gqa,,16,0.063776,16.836,1052.323,13.154,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,2.248768,61.117,7640.147,95.5018,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,16,2,128,128,gqa,,16,1.173952,117.074,7317.556,91.4694,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.192736,0.979,61.218,0.7652,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,1.250944,1.717,53.653,0.6707,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,21.81872,12.598,787.487,9.8436,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,16.944448,16.222,507.009,6.3376,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.192,1.47,61.242,0.7655,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,1.240512,2.597,54.108,0.6763,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,21.814976,18.901,787.671,9.8459,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,16.957888,24.314,506.638,6.333,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.189792,1.961,61.307,0.7663,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,1.248256,3.441,53.775,0.6722,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,21.814528,25.201,787.735,9.8467,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,16.963009,32.409,506.516,6.3314,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.159104,107.979,118.629,1.4829,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.37856,726.114,199.433,2.4929,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,3.179168,1383.395,94.99,1.1874,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.928384,296.082,76.804,0.96,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.781504,308.591,77.694,0.9712,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.931136,590.414,81.081,1.0135,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.787136,615.237,79.796,0.9975,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,1.739456,0.617,77.165,0.9646,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,62.199615,2.21,276.222,3.4528,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.77936,1.207,75.44,0.943,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,62.670338,4.386,274.164,3.4271,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.777056,1.813,75.542,0.9443,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,62.694366,6.577,274.076,3.4259,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.779712,2.413,75.434,0.9429,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,62.67664,8.771,274.17,3.4271,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.052288,164.281,180.485,2.2561,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.163104,842.646,231.44,2.893,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.537696,1430.077,98.196,1.2274,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.696064,197.452,51.219,0.6402,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.362144,201.798,50.807,0.6351,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.703168,390.914,53.684,0.671,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.372512,400.547,51.951,0.6494,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.044672,12.018,1502.35,18.7794,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,8,1,128,128,gqa,,16,0.061984,8.661,541.373,6.7672,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.132416,60.684,7585.957,94.8245,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.863264,0.576,36.021,0.4503,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,1.247424,0.861,26.902,0.3363,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,11.415712,12.039,752.558,9.407,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,8.504128,16.161,505.107,6.3138,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.860224,0.866,36.082,0.451,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,1.248128,1.29,26.889,0.3361,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,11.415872,18.059,752.593,9.4074,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,8.505504,24.238,505.056,6.3132,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.860096,1.155,36.087,0.4511,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,1.24896,1.719,26.872,0.3359,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,11.415936,24.078,752.635,9.4079,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,8.504384,32.322,505.153,6.3144,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.13408,64.066,70.385,0.8798,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.27232,504.697,138.619,1.7327,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.602688,1372.084,94.214,1.1777,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.867904,158.357,41.078,0.5135,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.652256,166.365,41.886,0.5236,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.87088,315.632,43.346,0.5418,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.650176,333.15,43.209,0.5401,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,1.584736,0.339,42.35,0.5294,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,21.431423,3.206,400.835,5.0104,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.645504,0.653,40.788,0.5099,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,21.758881,6.316,394.827,4.9353,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.648128,0.977,40.726,0.5091,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,21.809889,9.453,393.927,4.9241,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.646624,1.304,40.765,0.5096,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,21.789921,12.615,394.312,4.9289,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_context.png b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_context.png index 6da03a713..1f346d2f7 100755 Binary files a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_context.png and b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_context.png differ diff --git a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_generation.png b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_generation.png index f9d69e0e5..61772f646 100755 Binary files a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_generation.png and b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_generation.png differ diff --git a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260819_053435.csv b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260821_053517.csv similarity index 60% rename from benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260819_053435.csv rename to benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260821_053517.csv index ee4f33ec4..b219f33e6 100644 --- a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260819_053435.csv +++ b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260821_053517.csv @@ -1,151 +1,151 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.083104,826.909,908.47,11.3559,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,0.617184,1781.497,489.303,6.1163,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,9.526112,1846.733,126.805,1.5851,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,0.617088,1781.774,462.191,5.7774,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.195264,1839.78,463.202,5.79,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.215744,1808.788,248.399,3.105,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,2.443008,1800.259,233.493,2.9187,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.106752,40.233,5029.448,62.8681,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,64,8,128,128,gqa,,16,0.095616,44.919,2807.604,35.095,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,9.173312,59.93,7491.697,93.6462,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.186944,3.928,245.519,3.069,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,0.96752,8.878,277.481,3.4685,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,83.289436,13.201,825.169,10.3146,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,64,8,128,128,gqa,,16,50.636673,21.714,678.637,8.483,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.189024,5.886,245.301,3.0663,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,0.9656,13.344,278.05,3.4756,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,83.298592,19.799,825.129,10.3141,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,64,8,128,128,gqa,,16,50.979713,32.351,674.112,8.4264,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.188928,7.849,245.326,3.0666,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,0.965408,17.795,278.122,3.4765,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,83.315613,26.394,825.01,10.3126,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,64,8,128,128,gqa,,16,50.694847,43.378,677.941,8.4743,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.217952,315.296,346.395,4.3299,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.031328,1066.112,292.817,3.6602,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,10.86432,1619.263,111.186,1.3898,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.0192,1078.799,279.84,3.498,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.92176,1144.276,288.094,3.6012,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.688768,1302.146,178.823,2.2353,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,3.23856,1358.025,176.135,2.2017,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,1.855776,2.314,289.315,3.6164,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,215.184189,2.555,319.371,3.9921,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,1.904768,4.51,281.891,3.5236,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,216.191803,5.086,317.902,3.9738,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,1.906656,6.758,281.629,3.5204,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,216.08905,7.632,318.073,3.9759,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,1.89936,9.045,282.728,3.5341,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,215.932632,10.184,318.323,3.979,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.063264,543.117,596.686,7.4586,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.335872,1636.802,449.561,5.6195,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,4.573312,1923.353,132.066,1.6508,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.6032,911.399,236.416,2.9552,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.16608,942.913,237.397,2.9675,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.613536,1792.09,246.106,3.0763,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.18896,1849.535,239.884,2.9986,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.0712,30.161,3770.391,47.1299,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,32,4,128,128,gqa,,16,0.073632,29.165,1822.929,22.7866,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,4.616288,59.545,7443.607,93.0451,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,32,4,128,128,gqa,,16,2.356576,116.643,7290.628,91.1328,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.187968,1.963,122.702,1.5338,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,0.962368,4.463,139.483,1.7435,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,41.872864,13.129,820.673,10.2584,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,32,4,128,128,gqa,,16,24.988928,22.0,687.583,8.5948,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.189536,2.942,122.622,1.5328,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,0.96256,6.693,139.464,1.7433,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,41.867359,19.696,820.831,10.2604,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,32,4,128,128,gqa,,16,25.081247,32.878,685.094,8.5637,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.187616,3.927,122.737,1.5342,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,0.96208,8.929,139.542,1.7443,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,41.869663,26.26,820.836,10.2605,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,32,4,128,128,gqa,,16,25.063072,43.87,685.633,8.5704,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.173696,197.815,217.326,2.7166,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.548416,1002.443,275.329,3.4416,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,5.423424,1621.871,111.365,1.3921,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.8608,638.657,165.667,2.0708,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.621792,677.961,170.69,2.1336,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.879456,1250.218,171.691,2.1461,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.62784,1350.884,175.209,2.1901,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,1.565248,1.372,171.508,2.1438,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,106.57901,2.579,322.407,4.0301,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.618656,2.653,165.859,2.0732,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,107.219681,5.127,320.5,4.0063,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.61584,3.987,166.158,2.077,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,107.010818,7.706,321.145,4.0143,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.615104,5.319,166.244,2.078,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,107.176094,10.259,320.67,4.0084,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.053184,323.027,354.888,4.4361,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.200288,1372.413,376.945,4.7118,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.257664,1948.052,133.762,1.672,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.601792,456.766,118.485,1.4811,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.163616,472.455,118.95,1.4869,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.602816,911.979,125.241,1.5655,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.168768,940.744,122.014,1.5252,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.051776,20.738,2592.435,32.4054,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,16,2,128,128,gqa,,16,0.058816,18.256,1141.066,14.2633,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,2.312544,59.432,7429.444,92.8681,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,16,2,128,128,gqa,,16,1.20496,114.061,7129.248,89.1156,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.185088,0.983,61.432,0.7679,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,0.955968,2.246,70.208,0.8776,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,21.792896,12.613,788.421,9.8553,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,12.837312,21.412,669.22,8.3652,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.184,1.475,61.466,0.7683,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,0.956448,3.368,70.178,0.8772,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,21.796864,18.916,788.325,9.8541,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,12.833824,32.127,669.443,8.368,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.184992,1.966,61.442,0.768,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,0.955808,4.494,70.229,0.8779,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,21.792255,25.227,788.54,9.8567,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,12.837024,42.826,669.316,8.3665,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.1528,112.434,123.523,1.544,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.323648,849.311,233.27,2.9159,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.713248,1620.953,111.302,1.3913,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.79872,344.148,89.272,1.1159,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.474784,372.77,93.852,1.1732,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.797728,689.152,94.641,1.183,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.475872,744.991,96.625,1.2078,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,1.415328,0.759,94.837,1.1855,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,53.005569,2.593,324.134,4.0517,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.467328,1.464,91.482,1.1435,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,53.606625,5.128,320.519,4.0065,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.465984,2.197,91.571,1.1446,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,53.717888,7.676,319.875,3.9984,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.472512,2.917,91.171,1.1396,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,53.675838,10.242,320.145,4.0018,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.052128,164.785,181.039,2.263,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.130848,1050.371,288.493,3.6062,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.209024,1818.842,124.89,1.5611,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.600736,228.784,59.347,0.7418,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.164,236.149,59.455,0.7432,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.60224,456.426,62.681,0.7835,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.166368,471.34,61.133,0.7642,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.048128,11.155,1394.468,17.4309,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,8,1,128,128,gqa,,16,0.06128,8.761,547.593,6.8449,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.169664,58.751,7344.382,91.8048,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.82672,0.588,36.742,0.4593,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,0.961376,1.117,34.907,0.4363,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,11.121056,12.358,772.497,9.6562,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,6.438592,21.346,667.148,8.3393,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.82784,0.881,36.722,0.459,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,0.954656,1.687,35.155,0.4394,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,11.122464,18.535,772.446,9.6556,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,6.445504,31.985,666.473,8.3309,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.830304,1.173,36.674,0.4584,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,0.957376,2.243,35.057,0.4382,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,11.119712,24.72,772.685,9.6586,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,6.448192,42.629,666.236,8.3279,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.120096,71.526,78.58,0.9823,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.22368,614.445,168.762,2.1095,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.344416,1635.672,112.313,1.4039,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.729376,188.434,48.88,0.611,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.329952,206.683,52.036,0.6505,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.733664,374.665,51.452,0.6432,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.337856,410.923,53.297,0.6662,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,1.264672,0.425,53.067,0.6633,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,17.3736,3.955,494.455,6.1807,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.32096,0.813,50.809,0.6351,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,17.483809,7.861,491.368,6.1421,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.313184,1.226,51.113,0.6389,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,17.563135,11.738,489.178,6.1147,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.323712,1.622,50.71,0.6339,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,17.510336,15.698,490.683,6.1335,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.081248,845.799,929.223,11.6153,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,0.618016,1779.099,488.644,6.1081,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,9.578176,1836.695,126.116,1.5764,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,0.628704,1748.854,453.652,5.6706,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.199744,1832.91,461.472,5.7684,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.239808,1773.681,243.578,3.0447,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,2.512896,1750.19,226.999,2.8375,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.103936,41.323,5165.714,64.5714,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,64,8,128,128,gqa,,16,0.093888,45.746,2859.277,35.741,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,9.17904,59.893,7487.022,93.5878,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.190464,3.922,245.125,3.0641,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,0.962464,8.925,278.938,3.4867,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,83.464737,13.173,823.436,10.2929,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,64,8,128,128,gqa,,16,51.126465,21.506,672.136,8.4017,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.189952,5.884,245.197,3.065,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,0.961888,13.395,279.123,3.489,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,83.455772,19.762,823.575,10.2947,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,64,8,128,128,gqa,,16,51.154751,32.241,671.805,8.3976,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.189504,7.846,245.262,3.0658,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,0.964224,17.817,278.463,3.4808,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,83.459808,26.348,823.585,10.2948,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,64,8,128,128,gqa,,16,51.100063,43.034,672.565,8.4071,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.211936,324.246,356.228,4.4528,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.034816,1062.519,291.83,3.6479,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,11.149152,1577.895,108.345,1.3543,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.048448,1048.704,272.033,3.4004,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.931616,1138.437,286.624,3.5828,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.721792,1277.171,175.393,2.1924,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,3.241856,1356.645,175.956,2.1995,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,1.862464,2.306,288.276,3.6035,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,215.71936,2.548,318.579,3.9822,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,1.906432,4.506,281.645,3.5206,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,218.642914,5.029,314.338,3.9292,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,1.925088,6.693,278.932,3.4867,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,218.792603,7.538,314.143,3.9268,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,1.913376,8.979,280.657,3.5082,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,218.472519,10.065,314.622,3.9328,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.061248,560.994,616.326,7.7041,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.337824,1627.344,446.963,5.587,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,4.709088,1867.897,128.258,1.6032,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.605024,908.651,235.704,2.9463,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.172672,937.612,236.063,2.9508,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.614336,1789.756,245.786,3.0723,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.192672,1843.779,239.138,2.9892,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.0688,31.213,3901.916,48.774,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,32,4,128,128,gqa,,16,0.06992,30.713,1919.707,23.9963,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,4.62464,59.438,7430.164,92.8771,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,32,4,128,128,gqa,,16,2.348416,117.048,7315.96,91.4495,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.190528,1.961,122.559,1.532,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,0.960992,4.469,139.683,1.746,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,41.935135,13.11,819.454,10.2432,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,32,4,128,128,gqa,,16,25.3528,21.684,677.715,8.4714,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.189056,2.943,122.649,1.5331,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,0.960544,6.707,139.757,1.747,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,41.941921,19.661,819.372,10.2421,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,32,4,128,128,gqa,,16,25.359327,32.518,677.582,8.4698,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.189504,3.923,122.631,1.5329,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,0.961888,8.93,139.57,1.7446,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,41.940319,26.216,819.453,10.2432,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,32,4,128,128,gqa,,16,25.292225,43.472,679.421,8.4928,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.16832,204.133,224.268,2.8033,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.555776,989.168,271.683,3.396,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,5.630112,1562.33,107.277,1.341,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.877376,626.591,162.537,2.0317,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.624832,676.692,170.371,2.1296,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.904352,1215.801,166.965,2.0871,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.64704,1335.137,173.167,2.1646,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,1.57008,1.368,170.98,2.1372,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,107.059555,2.568,320.96,4.012,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.614304,2.661,166.306,2.0788,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,108.477409,5.068,316.784,3.9598,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.622112,3.972,165.515,2.0689,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,108.540771,7.597,316.619,3.9577,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.615008,5.319,166.254,2.0782,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,108.375427,10.145,317.121,3.964,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.052704,325.969,358.12,4.4765,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.194208,1415.379,388.745,4.8593,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.322688,1893.516,130.017,1.6252,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.602112,456.523,118.422,1.4803,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.165184,471.819,118.79,1.4849,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.604192,909.902,124.956,1.562,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.166016,942.964,122.302,1.5288,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.050016,21.468,2683.66,33.5457,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,16,2,128,128,gqa,,16,0.058688,18.296,1143.555,14.2944,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,2.311776,59.452,7431.913,92.8989,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,16,2,128,128,gqa,,16,1.202784,114.267,7142.146,89.2768,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.182688,0.984,61.499,0.7687,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,0.957408,2.243,70.103,0.8763,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,21.891487,12.556,784.87,9.8109,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,12.88288,21.337,666.853,8.3357,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.183424,1.475,61.482,0.7685,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,0.957248,3.365,70.119,0.8765,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,21.89328,18.833,784.853,9.8107,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,12.856704,32.07,668.251,8.3531,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.186784,1.964,61.392,0.7674,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,0.956992,4.488,70.142,0.8768,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,21.8936,25.11,784.89,9.8111,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,12.8464,42.795,668.828,8.3603,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.154976,110.855,121.789,1.5224,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.32272,851.754,233.941,2.9243,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.723584,1614.801,110.88,1.386,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.808288,340.074,88.215,1.1027,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.473792,373.021,93.916,1.1739,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.803744,683.994,93.932,1.1742,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.4808,742.512,96.304,1.2038,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,1.42688,0.753,94.07,1.1759,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,53.099041,2.588,323.564,4.0445,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.46976,1.461,91.331,1.1416,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,53.837471,5.106,319.145,3.9893,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.46864,2.193,91.406,1.1426,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,53.844769,7.658,319.121,3.989,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.468544,2.925,91.417,1.1427,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,53.867775,10.206,319.005,3.9876,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.0424,202.593,222.575,2.7822,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.128768,1067.338,293.153,3.6644,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.2112,1815.574,124.666,1.5583,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.599392,229.297,59.48,0.7435,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.16304,236.344,59.504,0.7438,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.6008,457.52,62.831,0.7854,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.169792,469.96,60.954,0.7619,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.046304,11.594,1449.399,18.1175,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,8,1,128,128,gqa,,16,0.058688,9.148,571.778,7.1472,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.165728,58.95,7369.18,92.1147,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.828608,0.587,36.704,0.4588,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,0.954432,1.125,35.161,0.4395,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,11.22384,12.245,765.423,9.5678,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,6.455584,21.29,665.392,8.3174,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.828128,0.881,36.716,0.4589,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,0.955072,1.686,35.139,0.4392,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,11.219648,18.375,765.756,9.5719,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,6.447712,31.974,666.245,8.3281,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.829152,1.174,36.697,0.4587,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,0.95952,2.238,34.979,0.4372,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,11.224352,24.489,765.481,9.5685,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,6.469952,42.485,663.995,8.2999,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.126464,67.924,74.623,0.9328,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.22912,599.856,164.755,2.0594,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.367456,1608.113,110.42,1.3803,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.745312,184.405,47.834,0.5979,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.347232,204.032,51.369,0.6421,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.755648,363.764,49.955,0.6244,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.351296,406.836,52.767,0.6596,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,1.273376,0.422,52.705,0.6588,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,17.555328,3.914,489.336,6.1167,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.342272,0.8,50.003,0.625,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,17.836576,7.705,481.65,6.0206,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.335872,1.206,50.245,0.6281,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,17.847744,11.551,481.378,6.0172,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.34144,1.601,50.04,0.6255,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,17.895391,15.36,480.125,6.0016,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_context.png b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_context.png index ce15ae8a7..58d6c934f 100755 Binary files a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_context.png and b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_context.png differ diff --git a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_generation.png b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_generation.png index d0e9a5db5..e18ca7aed 100755 Binary files a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_generation.png and b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_generation.png differ diff --git a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260820_065435.csv b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260821_070026.csv similarity index 55% rename from benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260820_065435.csv rename to benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260821_070026.csv index d0cdfc40c..ddb8e2def 100644 --- a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260820_065435.csv +++ b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260821_070026.csv @@ -1,61 +1,61 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.152,452.102,496.694,10.3478,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.740736,631.636,173.484,3.6143,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,28.950272,607.669,41.725,0.8693,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.7768,618.816,160.52,3.3442,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,3.528064,623.295,156.927,3.2693,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,3.650208,602.438,82.732,1.7236,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,7.777632,565.474,73.342,1.528,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.141472,30.359,3795.123,79.0651,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,15.58352,35.278,4410.022,91.8755,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,1.512672,5.679,354.959,7.395,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,104.216255,10.55,659.474,13.739,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,1.513152,8.515,354.868,7.3931,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,104.842659,15.731,655.573,13.6578,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,1.510624,11.373,355.484,7.4059,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,104.578941,21.027,657.267,13.6931,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.091616,375.041,412.032,8.584,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.905728,606.977,166.711,3.4731,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,14.568608,603.77,41.458,0.8637,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.894464,614.62,159.432,3.3215,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.8024,610.026,153.586,3.1997,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,1.77216,620.436,85.204,1.7751,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,3.777344,582.161,75.506,1.573,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.082752,25.951,3244.053,67.5844,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,7.592032,36.206,4526.039,94.2925,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.51184,2.841,177.577,3.6995,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,52.084545,10.555,659.772,13.7453,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.511712,4.262,177.603,3.7001,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,52.064129,15.839,660.071,13.7515,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.509664,5.69,177.855,3.7053,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,52.253506,21.042,657.719,13.7025,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.063232,271.696,298.494,6.2186,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.491072,559.751,153.74,3.2029,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,7.3968,594.588,40.827,0.8506,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.874752,314.235,81.512,1.6982,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.731936,317.423,79.918,1.6649,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.891584,616.606,84.678,1.7641,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.767616,622.031,80.677,1.6808,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.052704,20.373,2546.788,53.0581,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,3.751968,36.631,4579.175,95.3995,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.511296,1.421,88.821,1.8504,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,25.891745,10.616,663.608,13.8252,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.509792,2.134,88.914,1.8524,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,25.877472,15.933,664.014,13.8336,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.511104,2.842,88.843,1.8509,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,25.88448,21.239,663.875,13.8307,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.043328,198.254,217.808,4.5377,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.283488,484.814,133.158,2.7741,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,3.436256,639.947,43.942,0.9155,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.875616,156.963,40.716,0.8483,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.731712,158.732,39.964,0.8326,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.876032,313.776,43.091,0.8977,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.730272,317.728,41.209,0.8585,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.04288,12.52,1565.134,32.607,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.886144,36.434,4554.508,94.8856,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.508512,0.712,44.492,0.9269,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,12.920704,10.637,664.901,13.8521,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.508,1.068,44.51,0.9273,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,12.921984,15.954,664.875,13.8516,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.506688,1.425,44.552,0.9282,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,12.924384,21.268,664.792,13.8498,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.154304,445.351,489.277,10.1933,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.712736,641.962,176.32,3.6733,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,29.158976,603.32,41.427,0.8631,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.76624,622.515,161.48,3.3642,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,3.775424,582.457,146.645,3.0551,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,3.894976,564.579,77.533,1.6153,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,7.786112,564.858,73.262,1.5263,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.141984,30.25,3781.438,78.78,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,15.424224,35.642,4455.568,92.8243,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,1.526912,5.626,351.649,7.326,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,103.650719,10.608,663.072,13.814,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,1.527104,8.437,351.626,7.3255,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,103.625854,15.916,663.271,13.8182,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,1.525152,11.264,352.097,7.3354,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,103.816513,21.182,662.094,13.7936,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.091456,375.697,412.753,8.599,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.892032,616.296,169.271,3.5265,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,14.57904,603.338,41.428,0.8631,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.890784,617.159,160.091,3.3352,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.756352,626.02,157.613,3.2836,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,1.758048,625.416,85.888,1.7893,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,3.491616,629.801,81.685,1.7018,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.083264,25.791,3224.105,67.1688,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,7.61728,36.086,4511.037,93.9799,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.524352,2.818,176.12,3.6692,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,51.87072,10.599,662.492,13.8019,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.523456,4.229,176.234,3.6715,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,51.888096,15.893,662.31,13.7981,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.52224,5.643,176.385,3.6747,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,52.119167,21.096,659.414,13.7378,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.063584,270.192,296.841,6.1842,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.484384,567.479,155.863,3.2471,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,7.40416,593.997,40.787,0.8497,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.885856,310.296,80.491,1.6769,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.745664,314.926,79.289,1.6519,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.892736,615.81,84.569,1.7618,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.761248,624.28,80.969,1.6869,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.05344,20.092,2511.713,52.3273,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,3.752256,36.628,4578.824,95.3922,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.52224,1.411,88.182,1.8371,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,25.977888,10.581,661.407,13.7793,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.522464,2.116,88.174,1.837,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,25.983105,15.869,661.315,13.7774,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.522336,2.821,88.187,1.8372,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,25.942976,21.191,662.378,13.7995,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.043392,197.961,217.487,4.531,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.283808,484.267,133.008,2.771,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,3.360864,654.303,44.927,0.936,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.881312,155.948,40.453,0.8428,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.743168,157.689,39.701,0.8271,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.88224,311.568,42.787,0.8914,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.743424,315.331,40.898,0.852,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.043072,12.464,1558.157,32.4616,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.88656,36.426,4553.504,94.8647,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.519424,0.707,44.173,0.9203,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,12.90272,10.652,665.827,13.8714,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.5208,1.059,44.135,0.9195,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,13.05584,15.791,658.059,13.7096,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.519744,1.413,44.169,0.9202,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,12.918688,21.278,665.085,13.8559,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_context.png b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_context.png index daeb69198..42c0dbc6e 100755 Binary files a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_context.png and b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_context.png differ diff --git a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_generation.png b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_generation.png index 09d927bc7..22b8fb93c 100755 Binary files a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_generation.png and b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_generation.png differ diff --git a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260820_073350.csv b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260821_071251.csv similarity index 62% rename from benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260820_073350.csv rename to benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260821_071251.csv index a9635e16b..39ad96503 100644 --- a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260820_073350.csv +++ b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260821_071251.csv @@ -1,131 +1,131 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.271744,252.883,277.826,15.5037,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,3.133792,350.857,96.366,5.3775,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,48.740833,360.933,24.783,1.383,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,4.296192,255.927,66.387,3.7046,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,8.565056,256.744,64.64,3.6072,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,6.42064,342.493,47.034,2.6247,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,12.831904,342.743,44.454,2.4807,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.369472,11.625,1453.165,81.0918,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.385216,3.601,225.11,12.562,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,2.706816,3.173,99.182,5.5347,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.38592,5.4,225.058,12.559,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,2.700608,4.771,99.416,5.5478,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.385984,7.2,225.065,12.5594,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,2.70544,6.35,99.245,5.5382,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.322976,212.77,233.756,13.0444,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,3.147104,349.373,95.958,5.3548,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,45.7272,384.72,26.417,1.4741,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,3.5032,313.859,81.415,4.5432,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,7.069952,311.038,78.31,4.37,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,6.55792,335.323,46.05,2.5697,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,13.683232,321.419,41.688,2.3263,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,2.810688,1.528,191.022,10.6597,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.80912,3.058,191.14,10.6663,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.808352,4.588,191.204,10.6699,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.810272,6.113,191.085,10.6632,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.165504,207.607,228.084,12.7279,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,1.664512,330.28,90.714,5.0622,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,24.188929,363.641,24.969,1.3934,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,2.16112,254.385,65.987,3.6823,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,4.295712,255.956,64.442,3.5961,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,4.285888,256.542,35.231,1.966,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,8.551456,257.152,33.353,1.8612,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.1952,11.001,1375.266,76.7447,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,22.369505,12.288,1536.102,85.72,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.376736,1.807,112.957,6.3034,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,2.69264,1.595,49.852,2.7819,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,49.619518,11.079,692.549,38.6467,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.377856,2.709,112.91,6.3008,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,2.697472,2.388,49.766,2.7771,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,49.760929,16.572,690.623,38.5392,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.377248,3.613,112.946,6.3028,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,2.698432,3.183,49.751,2.7763,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,49.834816,22.063,689.641,38.4844,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.18672,184.017,202.168,11.2817,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,1.57024,350.109,96.16,5.3661,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,22.679104,387.85,26.632,1.4861,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,2.270272,242.154,62.815,3.5053,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,4.474336,245.737,61.869,3.4525,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,3.286496,334.554,45.944,2.5638,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,6.572064,334.602,43.398,2.4217,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,2.413728,0.89,111.219,6.2064,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,226.460251,1.214,151.735,8.4673,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.40704,1.784,111.535,6.224,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,132.885223,4.137,258.599,14.4307,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.407232,2.676,111.532,6.2239,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,133.148285,6.193,258.103,14.4031,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.407328,3.568,111.535,6.224,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,133.252899,8.251,257.917,14.3927,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.111392,154.229,169.441,9.4554,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.91984,298.832,82.077,4.5802,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,12.258912,358.763,24.634,1.3747,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,2.15232,127.712,33.129,1.8487,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,4.28576,128.275,32.296,1.8022,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,2.15712,254.856,34.999,1.9531,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,4.292352,256.156,33.223,1.854,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.04736,22.672,2834.162,158.1564,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,11.188352,12.284,1535.608,85.6924,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.356192,0.911,56.971,3.1792,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,2.618688,0.82,25.63,1.4302,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,24.910879,11.034,689.737,38.4898,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,29.918913,9.187,287.142,16.0236,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.354912,1.368,57.005,3.1811,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,2.702112,1.192,24.84,1.3862,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,24.961824,16.518,688.372,38.4136,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,29.996033,13.746,286.421,15.9833,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.356192,1.823,56.978,3.1796,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,2.69984,1.591,24.863,1.3874,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,24.993248,21.996,687.548,38.3676,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,30.003456,18.323,286.368,15.9804,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.125152,137.272,150.812,8.4158,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.800736,343.282,94.285,5.2614,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,11.242368,391.203,26.862,1.499,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,1.112128,247.164,64.114,3.5778,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,2.230688,246.451,62.049,3.4626,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,2.153184,255.322,35.063,1.9567,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,4.255904,258.35,33.508,1.8699,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,2.235712,0.48,60.037,3.3503,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,110.25872,1.247,155.824,8.6955,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.232768,0.962,60.12,3.3549,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,66.247681,4.149,259.36,14.4732,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.225216,1.448,60.328,3.3665,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,66.206078,6.228,259.538,14.4832,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.226304,1.929,60.302,3.3651,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,66.211868,8.303,259.531,14.4828,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.084352,101.834,111.879,6.2432,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.548672,250.494,68.8,3.8393,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,6.290112,349.6,24.005,1.3396,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,2.15136,63.885,16.572,0.9248,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,4.285472,64.142,16.149,0.9012,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,2.15152,127.76,17.545,0.9791,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,4.286144,128.264,16.636,0.9283,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.036992,14.513,1814.256,101.242,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,5.599904,12.272,1534.037,85.6047,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,2.35024,0.457,28.558,1.5936,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,2.697728,0.398,12.44,0.6942,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,13.0576,10.526,657.93,36.7148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,15.89616,8.646,270.222,15.0794,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,2.34768,0.686,28.59,1.5954,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,2.694016,0.598,12.457,0.6952,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,13.092704,15.746,656.206,36.6186,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,15.894464,12.97,270.267,15.0819,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,2.347936,0.915,28.589,1.5954,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,2.69504,0.797,12.453,0.6949,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,13.151456,20.901,653.314,36.4573,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,15.879648,17.31,270.536,15.0969,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.082528,104.085,114.351,6.3812,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.445472,308.524,84.739,4.7287,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,5.351584,410.911,28.215,1.5745,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,1.04288,131.788,34.186,1.9077,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,2.0408,134.691,33.911,1.8924,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,1.090208,252.133,34.625,1.9322,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,2.050336,268.13,34.776,1.9406,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,2.046336,0.262,32.797,1.8302,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,42.770241,1.607,200.851,11.2082,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,2.040608,0.526,32.891,1.8354,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,22.514721,6.104,381.572,21.2931,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,2.104224,0.765,31.898,1.78,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,22.51424,9.157,381.603,21.2948,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,2.101536,1.022,31.941,1.7824,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,22.518784,12.207,381.55,21.2918,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.272032,252.615,277.532,15.4873,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,3.135776,350.635,96.305,5.3741,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,48.528385,362.513,24.892,1.3891,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,4.295264,255.982,66.402,3.7055,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,8.56544,256.732,64.637,3.607,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,6.424128,342.307,47.009,2.6233,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,12.84416,342.416,44.411,2.4783,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.368576,11.653,1456.697,81.2889,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.378976,3.611,225.701,12.5949,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,2.701632,3.18,99.373,5.5453,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.379136,5.416,225.699,12.5948,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,2.704512,4.764,99.273,5.5398,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.379936,7.219,225.637,12.5914,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,2.701472,6.359,99.391,5.5464,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.323776,212.244,233.178,13.0122,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,3.156384,348.345,95.676,5.3391,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,45.55619,386.165,26.516,1.4797,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,3.510752,313.184,81.24,4.5335,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,7.082336,310.494,78.173,4.3623,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,6.578208,334.289,45.908,2.5618,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,13.696,321.119,41.649,2.3242,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,2.810624,1.528,191.027,10.66,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.811456,3.055,190.982,10.6575,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.814016,4.579,190.82,10.6484,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.816288,6.1,190.677,10.6405,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.17632,194.871,214.092,11.9471,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,1.664832,330.217,90.697,5.0612,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,24.234528,362.957,24.922,1.3908,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,2.15888,254.649,66.056,3.6861,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,4.292192,256.166,64.495,3.599,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,4.282496,256.746,35.259,1.9676,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,8.55104,257.164,33.354,1.8613,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.196256,10.942,1367.866,76.3318,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,22.380224,12.282,1535.366,85.6789,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.47632,1.734,108.414,6.0499,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,2.819104,1.524,47.616,2.6571,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,49.649601,11.073,692.129,38.6233,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.373664,2.714,113.11,6.3119,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,2.699648,2.386,49.726,2.7749,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,49.748482,16.576,690.796,38.5489,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.3744,3.618,113.082,6.3104,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,2.696096,3.186,49.794,2.7787,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,49.807297,22.075,690.022,38.5057,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.195136,176.081,193.448,10.7951,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,1.624096,338.5,92.972,5.1882,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,22.700895,387.478,26.606,1.4847,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,2.277984,241.334,62.602,3.4934,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,4.48304,245.26,61.749,3.4458,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,3.293696,333.823,45.844,2.5582,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,6.582048,334.094,43.332,2.4181,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,2.5312,0.848,106.057,5.9184,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,226.591003,1.213,151.647,8.4624,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.405984,1.785,111.584,6.2268,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,132.860321,4.138,258.647,14.4334,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.407328,2.676,111.528,6.2237,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,132.900665,6.205,258.584,14.4299,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.40672,3.569,111.563,6.2256,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,133.049606,8.264,258.311,14.4147,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.111872,153.567,168.714,9.4148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.918208,299.363,82.223,4.5883,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,12.260352,358.721,24.631,1.3745,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,2.151296,127.773,33.144,1.8496,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,4.284992,128.298,32.302,1.8025,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,2.155648,255.03,35.023,1.9544,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,4.292384,256.154,33.223,1.854,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.047296,22.703,2837.997,158.3704,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,11.18992,12.282,1535.392,85.6804,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.355776,0.912,56.981,3.1797,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,2.451584,0.876,27.377,1.5277,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,24.902624,11.038,689.966,38.5026,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,29.891745,9.196,287.403,16.0381,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.354176,1.368,57.023,3.1821,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,2.699488,1.193,24.864,1.3875,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,24.933088,16.537,689.165,38.4579,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,29.908833,13.786,287.257,16.0299,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.353632,1.825,57.04,3.183,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,2.698208,1.592,24.878,1.3883,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,24.964001,22.022,688.354,38.4126,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,29.937857,18.363,286.996,16.0154,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.124416,138.084,151.704,8.4656,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.804384,341.725,93.858,5.2376,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,11.255232,390.756,26.831,1.4973,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,1.167904,235.36,61.052,3.4069,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,2.231552,246.356,62.025,3.4612,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,2.15472,255.14,35.038,1.9553,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,4.262976,257.921,33.452,1.8668,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,2.332576,0.46,57.544,3.2112,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,110.499138,1.244,155.485,8.6766,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.325216,0.924,57.73,3.2215,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,66.223969,4.151,259.452,14.4784,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.319104,1.389,57.885,3.2302,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,66.178497,6.23,259.646,14.4892,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.320448,1.851,57.855,3.2285,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,66.190338,8.306,259.616,14.4875,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.08416,102.067,112.134,6.2575,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.549696,250.027,68.672,3.8321,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,6.297216,349.206,23.978,1.3381,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,2.151136,63.891,16.573,0.9249,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,4.283744,64.168,16.155,0.9015,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,2.252032,122.058,16.762,0.9354,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,4.286048,128.266,16.636,0.9284,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.038752,13.854,1731.858,96.6439,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,5.597888,12.276,1534.589,85.6356,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,2.453856,0.438,27.352,1.5263,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,2.819904,0.381,11.901,0.6641,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,13.064768,10.52,657.569,36.6947,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,15.899008,8.644,270.174,15.0766,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,2.454784,0.656,27.343,1.5258,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,2.815712,0.572,11.919,0.6651,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,13.108352,15.727,655.422,36.5749,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,15.898144,12.967,270.205,15.0784,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,2.45488,0.875,27.344,1.5259,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,2.81888,0.762,11.906,0.6644,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,13.113408,20.962,655.21,36.563,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,15.892448,17.296,270.318,15.0847,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.087296,98.4,108.106,6.0327,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.473952,289.985,79.647,4.4446,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,5.396128,407.519,27.982,1.5615,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,1.090112,126.078,32.705,1.825,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,2.138976,128.509,32.355,1.8055,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,1.116672,246.158,33.805,1.8864,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,2.07584,264.835,34.349,1.9168,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,2.136672,0.251,31.41,1.7528,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,42.720257,1.609,201.086,11.2213,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,2.130944,0.504,31.496,1.7576,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,22.519487,6.103,381.491,21.2886,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,2.131328,0.756,31.493,1.7574,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,22.508257,9.159,381.705,21.3005,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,2.131136,1.008,31.497,1.7577,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,22.520096,12.206,381.527,21.2906,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], diff --git a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_context.png b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_context.png index 3a7a26ea0..c19d8d5c6 100755 Binary files a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_context.png and b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_context.png differ diff --git a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_generation.png b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_generation.png index 47759d2d3..3445b57e7 100755 Binary files a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_generation.png and b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_generation.png differ diff --git a/benchmark/attention_inference/results/qwen35/gb200/qwen35_20260819_060531.csv b/benchmark/attention_inference/results/qwen35/gb200/qwen35_20260821_060434.csv similarity index 61% rename from benchmark/attention_inference/results/qwen35/gb200/qwen35_20260819_060531.csv rename to benchmark/attention_inference/results/qwen35/gb200/qwen35_20260821_060434.csv index 5632ee270..2465a436a 100644 --- a/benchmark/attention_inference/results/qwen35/gb200/qwen35_20260819_060531.csv +++ b/benchmark/attention_inference/results/qwen35/gb200/qwen35_20260821_060434.csv @@ -1,121 +1,121 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.083808,819.963,850.792,10.6349,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.64832,1695.94,439.926,5.4991,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,10.784896,1631.187,105.782,1.3223,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.646496,1700.725,233.559,2.9195,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.236128,1778.961,230.731,2.8841,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.276448,1722.768,131.437,1.643,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.448544,1796.188,123.334,1.5417,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.106208,40.439,2527.759,31.597,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,8.524704,64.49,4031.1,50.3887,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.549824,1.311,40.994,0.5124,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,191.447678,5.743,179.517,2.244,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.552544,1.966,40.982,0.5123,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,191.323608,8.62,179.655,2.2457,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.549984,2.623,41.003,0.5125,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,191.444839,11.486,179.564,2.2445,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.216448,317.487,329.424,4.1178,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,1.014848,1083.425,281.04,3.513,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,11.584416,1518.608,98.481,1.231,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.831936,1321.63,181.498,2.2687,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.546496,1421.939,184.425,2.3053,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.439744,1527.371,116.529,1.4566,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.812096,1563.974,107.39,1.3424,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,1.464704,2.932,183.292,2.2911,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,105.461441,5.213,325.844,4.073,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,1.482784,5.793,181.079,2.2635,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,105.358788,10.436,326.201,4.0775,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,1.48272,8.69,181.109,2.2639,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,106.243134,15.524,323.525,4.0441,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,1.482272,11.59,181.186,2.2648,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,105.328285,20.878,326.375,4.0797,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.059232,580.087,601.897,7.5237,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.350752,1567.363,406.573,5.0822,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,5.179936,1698.108,110.122,1.3765,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.58528,939.304,128.994,1.6124,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.12832,974.468,126.388,1.5799,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.642848,1710.376,130.491,1.6311,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.19568,1839.14,126.284,1.5785,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.09584,22.407,1400.606,17.5076,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.278784,64.242,4015.619,50.1952,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.547936,0.656,20.503,0.2563,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,96.535583,5.695,178.008,2.2251,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.546944,0.984,20.508,0.2564,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,96.343231,8.559,178.385,2.2298,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.545632,1.312,20.515,0.2564,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,96.393539,11.406,178.313,2.2289,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.152384,225.481,233.959,2.9245,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.526016,1045.131,271.106,3.3888,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,5.315936,1654.665,107.305,1.3413,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.652768,842.192,115.657,1.4457,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.195776,919.496,119.258,1.4907,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.709984,1548.643,118.152,1.4769,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.28928,1705.621,117.116,1.4639,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,1.142656,1.879,117.476,1.4684,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,34.655617,7.932,495.792,6.1974,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,1.188608,3.613,112.948,1.4118,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,35.104286,15.661,489.515,6.1189,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,1.185824,5.433,113.227,1.4153,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,35.186878,23.436,488.425,6.1053,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,1.185344,7.247,113.286,1.4161,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,35.077343,31.345,490.01,6.1251,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.05568,308.547,338.979,4.2372,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.19968,1376.592,378.092,4.7262,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.270016,1937.452,133.034,1.6629,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.583584,471.017,122.181,1.5273,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.126816,487.884,122.835,1.5354,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.583648,941.93,129.354,1.6169,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.127392,975.27,126.492,1.5812,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.093344,11.503,1437.97,17.9746,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.276544,32.138,4017.477,50.2185,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.543584,0.328,20.514,0.2564,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,52.671646,5.219,326.209,4.0776,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.5432,0.492,20.516,0.2565,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,52.660671,7.83,326.297,4.0787,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.542592,0.656,20.519,0.2565,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,52.654976,10.441,326.352,4.0794,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.128736,133.45,146.613,1.8327,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.296864,925.939,254.317,3.179,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.55328,1722.508,118.275,1.4784,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.648416,423.922,109.965,1.3746,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.18832,462.633,116.477,1.456,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.6536,841.12,115.51,1.4439,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.193152,921.519,119.521,1.494,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,1.142176,0.94,117.518,1.469,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,18.151199,7.572,946.544,11.8318,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,1.186176,1.81,113.165,1.4146,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,18.152639,15.143,946.527,11.8316,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,1.18608,2.716,113.181,1.4148,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,18.341856,22.48,936.82,11.7102,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,1.184736,3.625,113.317,1.4165,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,18.324991,30.0,937.739,11.7217,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.054464,157.718,192.526,2.4066,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.122624,1120.816,342.046,4.2756,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.191232,1846.008,140.839,1.7605,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.581632,236.299,118.986,1.4873,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.1248,244.379,121.19,1.5149,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.584256,470.475,122.041,1.5255,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.127488,487.593,122.761,1.5345,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.094624,5.674,1418.476,17.7309,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.278048,16.063,4015.942,50.1993,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.549088,0.164,20.495,0.2562,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,30.41264,4.519,564.927,7.0616,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.547744,0.246,20.5,0.2563,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,30.402624,6.781,565.13,7.0641,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.547424,0.328,20.502,0.2563,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,30.388737,9.045,565.406,7.0676,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.127232,67.514,82.414,1.0302,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.231296,594.212,181.339,2.2667,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.347072,1632.447,124.546,1.5568,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.646176,212.696,107.101,1.3388,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.189248,231.136,114.623,1.4328,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.648992,423.546,109.868,1.3733,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.192736,460.92,116.046,1.4506,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,1.142304,0.47,117.501,1.4688,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,9.717088,7.072,1768.06,22.1007,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,1.184768,0.906,113.293,1.4162,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,9.716,14.146,1768.312,22.1039,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,1.190208,1.353,112.779,1.4097,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,9.64752,21.369,1780.918,22.2615,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,1.188,1.808,112.992,1.4124,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,9.723712,28.269,1767.017,22.0877,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.080096,857.964,890.221,11.1278,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.647264,1698.707,440.643,5.508,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,10.849088,1621.536,105.156,1.3145,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.650944,1689.103,231.963,2.8995,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.244448,1767.067,229.188,2.8649,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.271136,1729.967,131.986,1.6498,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.475328,1776.753,122.0,1.525,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.103936,41.323,2583.015,32.2877,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,8.721056,63.038,3940.341,49.2543,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,7.042688,1.22,38.125,0.4766,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,192.266052,5.719,178.753,2.2344,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,7.043904,1.829,38.123,0.4765,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,191.70755,8.603,179.296,2.2412,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,7.039808,2.44,38.15,0.4769,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,192.058075,11.45,178.99,2.2374,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.21472,320.042,332.075,4.1509,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,1.020896,1077.007,279.375,3.4922,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,11.655072,1509.402,97.884,1.2236,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.83568,1315.709,180.685,2.2586,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.538336,1429.482,185.403,2.3175,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.442848,1524.085,116.278,1.4535,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.784864,1579.268,108.44,1.3555,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,1.46272,2.936,183.54,2.2943,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,106.508064,5.162,322.642,4.033,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,1.483648,5.79,180.974,2.2622,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,106.32515,10.341,323.236,4.0405,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,1.483968,8.683,180.957,2.262,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,105.669662,15.608,325.281,4.066,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,1.483488,11.581,181.037,2.263,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,105.910271,20.763,324.582,4.0573,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.05616,611.819,634.822,7.9353,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.348,1579.758,409.788,5.1224,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,5.46784,1608.696,104.324,1.304,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.578208,950.792,130.571,1.6321,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.124608,977.684,126.805,1.5851,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.637152,1725.666,131.658,1.6457,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.231488,1785.664,122.612,1.5326,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.093376,22.998,1437.565,17.9696,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.275008,64.299,4019.166,50.2396,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.54048,0.657,20.526,0.2566,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,97.128319,5.66,176.921,2.2115,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.537984,0.985,20.536,0.2567,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,97.16317,8.487,176.879,2.211,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.539456,1.314,20.534,0.2567,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,97.035233,11.331,177.134,2.2142,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.14848,231.41,240.11,3.0014,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.530496,1036.305,268.817,3.3602,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,5.546304,1585.938,102.848,1.2856,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.656128,837.879,115.065,1.4383,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.196128,919.226,119.223,1.4903,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.708512,1551.86,118.398,1.48,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.275552,1723.978,118.376,1.4797,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,1.144384,1.877,117.298,1.4662,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,35.34288,7.777,486.151,6.0769,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,1.187488,3.617,113.054,1.4132,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,35.60957,15.438,482.569,6.0321,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,1.186688,5.429,113.144,1.4143,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,35.684319,23.109,481.617,6.0202,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,1.183136,7.26,113.498,1.4187,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,35.445473,31.02,484.921,6.0615,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.051328,334.708,367.721,4.5965,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.19488,1410.498,387.405,4.8426,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.253952,1951.26,133.982,1.6748,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.58064,473.405,122.801,1.535,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.122112,489.93,123.35,1.5419,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.580928,946.341,129.96,1.6245,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.122592,979.44,127.033,1.5879,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.094912,11.313,1414.214,17.6777,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.273568,32.16,4020.275,50.2534,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.541024,0.328,20.522,0.2565,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,52.697311,5.216,326.05,4.0756,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.539776,0.493,20.527,0.2566,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,52.689728,7.825,326.117,4.0765,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.53888,0.657,20.531,0.2566,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,52.658272,10.44,326.332,4.0791,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.129184,132.988,146.105,1.8263,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.299136,918.906,252.385,3.1548,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.555584,1720.956,118.169,1.4771,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.650592,422.504,109.597,1.37,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.191552,461.378,116.161,1.452,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.653408,841.367,115.544,1.4443,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.197824,917.924,119.054,1.4882,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,1.1424,0.94,117.495,1.4687,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,18.175392,7.562,945.285,11.8161,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,1.188864,1.806,112.91,1.4114,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,18.212769,15.093,943.402,11.7925,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,1.186304,2.715,113.16,1.4145,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,18.2096,22.643,943.624,11.7953,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,1.186208,3.621,113.176,1.4147,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,18.240959,30.139,942.059,11.7757,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.051776,165.906,202.522,2.5315,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.120192,1143.495,348.967,4.3621,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.154016,1905.54,145.381,1.8173,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.577408,238.027,119.856,1.4982,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.124864,244.365,121.183,1.5148,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.580864,473.222,122.754,1.5344,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.12336,489.385,123.213,1.5402,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.093664,5.732,1433.014,17.9127,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.272032,16.086,4021.598,50.27,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.355968,0.169,21.118,0.264,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,30.926336,4.444,555.543,6.9443,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.35344,0.254,21.127,0.2641,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,30.933632,6.665,555.429,6.9429,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.348768,0.338,21.143,0.2643,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,30.96944,8.876,554.804,6.935,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.130592,65.777,80.294,1.0037,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.235008,584.827,178.475,2.2309,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.351168,1627.498,124.168,1.5521,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.65008,211.419,106.458,1.3307,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.190944,230.807,114.46,1.4307,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.651008,422.234,109.527,1.3691,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.193984,460.438,115.925,1.4491,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,1.150048,0.467,116.71,1.4589,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,9.682048,7.098,1774.459,22.1807,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,1.193056,0.9,112.506,1.4063,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,9.71984,14.14,1767.613,22.0952,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,1.194304,1.349,112.392,1.4049,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,9.527968,21.637,1803.264,22.5408,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,1.190336,1.804,112.77,1.4096,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,9.601312,28.629,1789.544,22.3693,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/qwen35/gb200/qwen35_context.png b/benchmark/attention_inference/results/qwen35/gb200/qwen35_context.png index 4acd51d54..58ea37314 100755 Binary files a/benchmark/attention_inference/results/qwen35/gb200/qwen35_context.png and b/benchmark/attention_inference/results/qwen35/gb200/qwen35_context.png differ diff --git a/benchmark/attention_inference/results/qwen35/gb200/qwen35_generation.png b/benchmark/attention_inference/results/qwen35/gb200/qwen35_generation.png index 9b1ed896c..ddebb5961 100755 Binary files a/benchmark/attention_inference/results/qwen35/gb200/qwen35_generation.png and b/benchmark/attention_inference/results/qwen35/gb200/qwen35_generation.png differ diff --git a/benchmark/attention_inference/results/qwen35/gb300/qwen35_20260819_060443.csv b/benchmark/attention_inference/results/qwen35/gb300/qwen35_20260821_060629.csv similarity index 59% rename from benchmark/attention_inference/results/qwen35/gb300/qwen35_20260819_060443.csv rename to benchmark/attention_inference/results/qwen35/gb300/qwen35_20260821_060629.csv index 274a59ee5..0924345f1 100644 --- a/benchmark/attention_inference/results/qwen35/gb300/qwen35_20260819_060443.csv +++ b/benchmark/attention_inference/results/qwen35/gb300/qwen35_20260821_060629.csv @@ -1,121 +1,121 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.078272,877.957,910.966,11.3871,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.567456,1937.616,502.616,6.2827,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,9.0648,1940.714,125.855,1.5732,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.58352,1884.274,258.766,3.2346,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.231008,1786.36,231.69,2.8961,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.152544,1907.973,145.567,1.8196,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.290336,1920.263,131.854,1.6482,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.101984,42.114,2632.454,32.9057,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,8.600672,63.92,3995.494,49.9437,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.741088,1.274,39.831,0.4979,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,186.664963,5.89,184.117,2.3015,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.738176,1.912,39.853,0.4982,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,187.686234,8.787,183.137,2.2892,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.7392,2.549,39.851,0.4981,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,186.383865,11.798,184.439,2.3055,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.20016,343.323,356.231,4.4529,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.920064,1195.038,309.992,3.8749,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,9.832704,1789.15,116.026,1.4503,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.783008,1404.215,192.84,2.4105,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.469664,1496.276,194.067,2.4258,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.360352,1616.51,123.33,1.5416,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.630272,1672.088,114.813,1.4352,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,1.43952,2.984,186.498,2.3312,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,100.87133,5.45,340.671,4.2584,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,1.46048,5.882,183.844,2.2981,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,100.641281,10.925,341.491,4.2686,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,1.459904,8.826,183.939,2.2992,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,100.581474,16.397,341.736,4.2717,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,1.454624,11.811,184.63,2.3079,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,101.075104,21.756,340.109,4.2514,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.056832,604.584,627.315,7.8414,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.302688,1816.246,471.133,5.8892,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,4.361184,2016.905,130.796,1.6349,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.576896,952.955,130.868,1.6359,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.116096,985.141,127.772,1.5972,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.582656,1887.068,143.972,1.7996,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.202592,1828.57,125.558,1.5695,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.093152,23.054,1441.022,18.0128,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.212288,65.256,4079.01,50.9876,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.475392,0.663,20.732,0.2592,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,94.890015,5.794,181.095,2.2637,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.476416,0.995,20.732,0.2591,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,94.982109,8.682,180.941,2.2618,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.474208,1.327,20.741,0.2593,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,94.874977,11.589,181.167,2.2646,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.139296,246.667,255.941,3.1993,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.471968,1164.816,302.153,3.7769,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,4.688576,1876.069,121.663,1.5208,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.644032,853.616,117.226,1.4653,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.185888,927.163,120.253,1.5032,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.6504,1690.516,128.976,1.6122,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.198656,1834.574,125.97,1.5746,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,1.137696,1.888,117.988,1.4748,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,31.387903,8.757,547.407,6.8426,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,1.179904,3.64,113.781,1.4223,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,31.414207,17.5,547.016,6.8377,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,1.1816,5.452,113.631,1.4204,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,31.421568,26.244,546.954,6.8369,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,1.17792,7.292,114.0,1.425,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,31.415648,34.999,547.124,6.8391,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.051872,331.197,363.864,4.5483,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.170432,1612.83,442.977,5.5372,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.048896,2146.544,147.392,1.8424,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.576384,476.901,123.708,1.5463,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.116192,492.528,124.004,1.55,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.577408,952.11,130.752,1.6344,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.116352,984.915,127.743,1.5968,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.091712,11.708,1463.559,18.2945,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.208576,32.657,4082.359,51.0295,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.476672,0.332,20.726,0.2591,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,51.652447,5.322,332.646,4.1581,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.47104,0.498,20.745,0.2593,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,51.629951,7.986,332.811,4.1601,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.473504,0.663,20.738,0.2592,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,51.54781,10.665,333.362,4.167,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.124448,138.049,151.665,1.8958,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.266144,1032.816,283.672,3.5459,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.36,1863.579,127.962,1.5995,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.639808,429.626,111.445,1.3931,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.182528,464.899,117.048,1.4631,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.647104,849.563,116.67,1.4584,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.181376,930.704,120.712,1.5089,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,1.138112,0.943,117.937,1.4742,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,16.204384,8.482,1060.264,13.2533,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,1.176256,1.826,114.12,1.4265,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,16.326401,16.836,1052.404,13.155,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,1.178208,2.734,113.938,1.4242,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,16.190529,25.467,1061.3,13.2663,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,1.178048,3.646,113.96,1.4245,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,16.225376,33.882,1059.086,13.2386,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.051712,166.111,202.772,2.5347,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.105472,1303.085,397.67,4.9709,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.045184,2103.958,160.519,2.0065,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.575456,238.835,120.263,1.5033,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.115104,246.504,122.244,1.5281,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.575936,477.272,123.804,1.5475,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.115808,492.697,124.046,1.5506,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.092576,5.799,1449.856,18.1232,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.20848,16.329,4082.328,51.0291,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.238304,0.172,21.516,0.269,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,30.444256,4.514,564.34,7.0543,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.234272,0.258,21.531,0.2691,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,30.363647,6.79,565.856,7.0732,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.238656,0.344,21.517,0.269,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,30.456863,9.025,564.141,7.0518,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.124448,69.024,84.258,1.0532,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.213152,644.793,196.775,2.4597,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.212352,1813.849,138.386,1.7298,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.641056,214.395,107.956,1.3495,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.17776,233.39,115.741,1.4468,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.638944,430.207,111.595,1.3949,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.182336,464.974,117.067,1.4633,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,1.139232,0.471,117.818,1.4727,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,8.207808,8.372,2093.177,26.1647,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,1.181728,0.909,113.584,1.4198,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,8.373696,16.413,2051.772,25.6472,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,1.18224,1.362,113.539,1.4192,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,8.421088,24.481,2040.288,25.5036,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,1.179648,1.82,113.792,1.4224,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,8.372128,32.833,2052.282,25.6535,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.075392,911.496,945.766,11.8221,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.569888,1929.347,500.471,6.2559,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,9.125728,1927.757,125.015,1.5627,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.585344,1878.402,257.959,3.2245,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.128896,1947.941,252.647,3.1581,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.152064,1908.768,145.627,1.8203,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.34768,1873.359,128.633,1.6079,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.101248,42.42,2651.59,33.1449,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,8.588768,64.009,4001.032,50.0129,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.737184,1.275,39.854,0.4982,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,187.593918,5.861,183.205,2.2901,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.735904,1.913,39.866,0.4983,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,187.889084,8.778,182.939,2.2867,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.736256,2.55,39.869,0.4984,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,186.754593,11.775,184.073,2.3009,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.2064,332.943,345.461,4.3183,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.926752,1186.414,307.755,3.8469,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,9.90032,1776.931,115.234,1.4404,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.7888,1393.904,191.424,2.3928,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.477376,1488.466,193.054,2.4132,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.366368,1609.393,122.787,1.5348,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.63392,1669.772,114.654,1.4332,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,1.44064,2.981,186.353,2.3294,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,100.05677,5.494,343.444,4.2931,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,1.459136,5.887,184.014,2.3002,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,100.042496,10.99,343.535,4.2942,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,1.467552,8.78,182.981,2.2873,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,100.700096,16.378,341.334,4.2667,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,1.463104,11.742,183.559,2.2945,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,100.605186,21.858,341.697,4.2712,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.05312,646.832,671.152,8.3894,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.30048,1829.592,474.595,5.9324,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,4.554464,1931.313,125.245,1.5656,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.57504,956.031,131.291,1.6411,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.111008,989.652,128.358,1.6045,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.58272,1886.861,143.956,1.7995,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.127232,1950.817,133.952,1.6744,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.091296,23.522,1470.318,18.379,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.21024,65.288,4080.995,51.0124,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.479264,0.663,20.72,0.259,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,95.220734,5.773,180.466,2.2558,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.480224,0.994,20.719,0.259,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,95.121346,8.669,180.676,2.2585,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.480512,1.326,20.721,0.259,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,95.078911,11.564,180.779,2.2597,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.145184,236.663,245.561,3.0695,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.478496,1148.925,298.03,3.7254,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,4.89184,1798.115,116.608,1.4576,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.645664,851.458,116.93,1.4616,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.187744,925.714,120.065,1.5008,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.654816,1679.115,128.106,1.6013,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.209952,1817.447,124.794,1.5599,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,1.13424,1.893,118.347,1.4793,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,31.728449,8.663,541.532,6.7691,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,1.179584,3.641,113.812,1.4226,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,31.728449,17.327,541.598,6.77,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,1.178912,5.465,113.89,1.4236,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,31.745344,25.977,541.376,6.7672,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,1.180736,7.275,113.728,1.4216,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,31.721855,34.661,541.843,6.773,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.05008,343.048,376.884,4.7111,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.167424,1641.807,450.936,5.6367,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.066272,2128.493,146.152,1.8269,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.573152,479.59,124.405,1.5551,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.112768,494.043,124.385,1.5548,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.5736,958.431,131.62,1.6453,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.113696,987.264,128.048,1.6006,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.089696,11.971,1496.454,18.7057,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.20976,32.648,4081.211,51.0151,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.481152,0.331,20.711,0.2589,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,51.690079,5.318,332.404,4.155,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.481088,0.497,20.713,0.2589,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,51.676064,7.979,332.514,4.1564,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.478368,0.663,20.723,0.259,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,51.601727,10.654,333.013,4.1627,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.127296,134.96,148.271,1.8534,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.267264,1028.488,282.483,3.531,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.400032,1832.495,125.827,1.5728,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.639424,429.884,111.512,1.3939,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.183296,464.597,116.972,1.4621,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.64768,848.808,116.566,1.4571,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.18672,926.513,120.168,1.5021,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,1.136512,0.945,118.103,1.4763,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,16.319263,8.422,1052.8,13.16,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,1.180544,1.819,113.705,1.4213,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,16.466656,16.693,1043.44,13.043,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,1.180192,2.729,113.746,1.4218,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,16.393984,25.15,1048.129,13.1016,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,1.182016,3.634,113.578,1.4197,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,16.320543,33.685,1052.91,13.1614,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.049856,172.295,210.321,2.629,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.102464,1341.339,409.344,5.1168,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.052352,2089.627,159.426,1.9928,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.572832,239.929,120.814,1.5102,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.110752,247.47,122.723,1.534,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.573248,479.51,124.384,1.5548,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.112352,494.228,124.432,1.5554,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.091936,5.84,1459.948,18.2494,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.207136,16.334,4083.631,51.0454,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.269888,0.171,21.408,0.2676,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,30.475391,4.51,563.764,7.047,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.271168,0.257,21.404,0.2676,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,30.531008,6.752,562.754,7.0344,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.263008,0.343,21.433,0.2679,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,30.50688,9.01,563.216,7.0402,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.125856,68.252,83.316,1.0414,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.213632,643.344,196.333,2.4542,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.224128,1796.4,137.054,1.7132,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.642944,213.765,107.639,1.3455,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.182176,232.519,115.308,1.4414,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.645408,425.898,110.478,1.381,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.180736,465.604,117.225,1.4653,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,1.140416,0.471,117.695,1.4712,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,8.35536,8.225,2056.212,25.7027,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,1.186336,0.905,113.143,1.4143,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,8.375872,16.409,2051.239,25.6405,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,1.178272,1.367,113.921,1.424,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,8.382432,24.594,2049.697,25.6212,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,1.182816,1.816,113.487,1.4186,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,8.39568,32.74,2046.525,25.5816,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/qwen35/gb300/qwen35_context.png b/benchmark/attention_inference/results/qwen35/gb300/qwen35_context.png index b722966bd..9232fe8d0 100755 Binary files a/benchmark/attention_inference/results/qwen35/gb300/qwen35_context.png and b/benchmark/attention_inference/results/qwen35/gb300/qwen35_context.png differ diff --git a/benchmark/attention_inference/results/qwen35/gb300/qwen35_generation.png b/benchmark/attention_inference/results/qwen35/gb300/qwen35_generation.png index fac57855d..9d2df32e7 100755 Binary files a/benchmark/attention_inference/results/qwen35/gb300/qwen35_generation.png and b/benchmark/attention_inference/results/qwen35/gb300/qwen35_generation.png differ diff --git a/benchmark/attention_inference/results/qwen35/h200/qwen35_20260820_070011.csv b/benchmark/attention_inference/results/qwen35/h200/qwen35_20260821_070612.csv similarity index 55% rename from benchmark/attention_inference/results/qwen35/h200/qwen35_20260820_070011.csv rename to benchmark/attention_inference/results/qwen35/h200/qwen35_20260821_070612.csv index 4d5bbef93..900cccce5 100644 --- a/benchmark/attention_inference/results/qwen35/h200/qwen35_20260820_070011.csv +++ b/benchmark/attention_inference/results/qwen35/h200/qwen35_20260821_070612.csv @@ -1,61 +1,61 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.154368,445.167,461.904,9.623,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,1.626848,675.854,175.316,3.6524,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,26.520864,663.334,43.017,0.8962,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,1.514144,726.161,99.723,2.0776,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,3.110208,707.034,91.702,1.9105,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,3.180416,691.426,52.752,1.099,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,6.582048,668.188,45.881,0.9559,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.104544,41.083,2567.993,53.4998,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,9.367232,58.689,3668.526,76.4276,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,2.82496,3.041,95.046,1.9801,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,97.68486,11.256,351.827,7.3297,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,2.82144,4.567,95.176,1.9828,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,98.016228,16.826,350.68,7.3058,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,2.82528,6.081,95.058,1.9804,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,98.469955,22.332,349.107,7.2731,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.099712,344.59,357.546,7.4489,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.873184,629.599,163.318,3.4025,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,13.428096,655.051,42.48,0.885,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,1.403936,391.582,53.776,1.1203,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,2.792032,393.803,51.076,1.0641,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,1.508128,729.057,55.623,1.1588,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,3.153792,697.263,47.877,0.9974,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.087776,24.465,1529.28,31.86,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.668512,58.879,3680.395,76.6749,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,2.82064,1.523,47.596,0.9916,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,48.525471,11.329,354.125,7.3776,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,2.823776,2.282,47.549,0.9906,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,48.691616,16.936,352.959,7.3533,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,2.824256,3.041,47.546,0.9906,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,48.660416,22.596,353.229,7.3589,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.063136,272.109,298.948,6.2281,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.499136,550.707,151.256,3.1512,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,6.854816,641.6,44.055,0.9178,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,1.404288,195.742,50.775,1.0578,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,2.784704,197.42,49.704,1.0355,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,1.402432,392.002,53.833,1.1215,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,2.789696,394.133,51.119,1.065,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.08736,12.291,1536.469,32.0098,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.65184,29.545,3693.359,76.945,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,2.822272,0.761,47.562,0.9909,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,24.493504,11.222,701.491,14.6144,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,2.821632,1.142,47.576,0.9912,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,24.704737,16.69,695.535,14.4903,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,2.821632,1.522,47.579,0.9912,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,24.721664,22.238,695.101,14.4813,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.062208,138.084,168.56,3.5117,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.315136,436.126,133.095,2.7728,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,3.47984,631.932,48.213,1.0044,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,1.403264,97.942,49.318,1.0275,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,2.788448,98.577,48.886,1.0184,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,1.403584,195.84,50.801,1.0583,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,2.791136,196.965,49.59,1.0331,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.087072,6.166,1541.504,32.1147,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.65296,14.769,3692.358,76.9241,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,2.821472,0.381,47.573,0.9911,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,12.635488,10.877,1359.735,28.3278,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,2.82064,0.571,47.588,0.9914,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,12.809664,16.094,1341.288,27.9435,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,2.820832,0.761,47.587,0.9914,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,12.863968,21.368,1335.666,27.8264,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.1544,445.074,461.808,9.621,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,1.6008,686.851,178.169,3.7119,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,26.673569,659.536,42.771,0.8911,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,1.461952,752.085,103.283,2.1517,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,3.00976,730.631,94.763,1.9742,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,2.951904,744.951,56.835,1.1841,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,6.584384,667.951,45.865,0.9555,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.104384,41.146,2571.929,53.5819,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,9.374144,58.646,3665.821,76.3713,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,2.844416,3.02,94.396,1.9666,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,97.800735,11.242,351.41,7.321,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,2.844704,4.529,94.398,1.9666,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,97.961823,16.836,350.875,7.3099,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,2.845824,6.037,94.372,1.9661,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,98.184029,22.397,350.123,7.2942,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.10064,341.412,354.249,7.3802,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.861248,638.325,165.581,3.4496,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,13.419136,655.489,42.508,0.8856,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,1.414048,388.782,53.391,1.1123,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,2.810112,391.27,50.748,1.0572,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,1.448512,759.063,57.912,1.2065,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,3.057376,719.252,49.387,1.0289,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.087712,24.483,1530.396,31.8833,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.67856,58.753,3672.49,76.5102,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,2.843584,1.51,47.212,0.9836,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,48.276512,11.388,355.951,7.4156,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,2.845248,2.264,47.19,0.9831,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,48.509312,16.999,354.286,7.381,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,2.843712,3.021,47.221,0.9838,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,48.696095,22.579,352.97,7.3535,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.063456,270.737,297.44,6.1967,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.493824,556.631,152.883,3.1851,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,6.875584,639.662,43.922,0.915,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,1.415584,194.18,50.37,1.0494,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,2.813664,195.388,49.193,1.0249,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,1.415392,388.412,53.34,1.1113,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,2.811584,391.065,50.721,1.0567,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.08784,12.224,1528.073,31.8349,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.67888,29.374,3672.015,76.5003,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,2.844704,0.755,47.187,0.9831,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,24.706881,11.126,695.432,14.4882,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,2.843648,1.133,47.208,0.9835,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,24.516031,16.818,700.889,14.6019,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,2.844896,1.51,47.19,0.9831,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,24.60976,22.339,698.262,14.5471,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.062752,136.887,167.098,3.4812,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.31616,434.713,132.664,2.7638,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,3.219296,683.076,52.115,1.0857,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,1.413568,97.228,48.958,1.02,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,2.809472,97.84,48.52,1.0108,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,1.41344,194.474,50.447,1.051,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,2.810112,195.635,49.255,1.0261,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.088032,6.099,1524.694,31.7644,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.678752,14.688,3672.003,76.5001,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,2.843296,0.378,47.208,0.9835,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,12.830464,10.712,1339.072,27.8973,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,2.843552,0.566,47.205,0.9834,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,12.76432,16.151,1346.052,28.0428,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,2.843808,0.755,47.202,0.9834,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,12.833664,21.419,1338.82,27.8921,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/qwen35/h200/qwen35_context.png b/benchmark/attention_inference/results/qwen35/h200/qwen35_context.png index 3a3deb84e..e54ef7894 100755 Binary files a/benchmark/attention_inference/results/qwen35/h200/qwen35_context.png and b/benchmark/attention_inference/results/qwen35/h200/qwen35_context.png differ diff --git a/benchmark/attention_inference/results/qwen35/h200/qwen35_generation.png b/benchmark/attention_inference/results/qwen35/h200/qwen35_generation.png index cb0f1ce42..5e165a7b9 100755 Binary files a/benchmark/attention_inference/results/qwen35/h200/qwen35_generation.png and b/benchmark/attention_inference/results/qwen35/h200/qwen35_generation.png differ diff --git a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260820_074342.csv b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260821_072245.csv similarity index 61% rename from benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260820_074342.csv rename to benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260821_072245.csv index bd30b3a0f..c53c0aede 100644 --- a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260820_074342.csv +++ b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260821_072245.csv @@ -1,121 +1,121 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.550912,124.738,129.428,7.2225,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,7.248672,151.685,39.347,2.1957,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,111.480675,157.805,10.234,0.5711,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,8.349408,131.687,18.085,1.0092,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,16.782688,131.029,16.994,0.9484,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,14.774528,148.839,11.356,0.6337,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,29.718847,147.988,10.162,0.5671,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.214656,20.009,1250.691,69.793,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,22.327135,24.623,1539.111,85.8879,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.98272,1.23,38.452,2.1458,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,167.731033,6.555,204.9,11.4342,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.981632,1.846,38.463,2.1464,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,168.653122,9.779,203.805,11.373,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.981568,2.461,38.468,2.1466,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,169.589401,12.967,202.704,11.3116,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.380352,180.673,187.466,10.4613,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,3.4032,323.082,83.807,4.6767,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,50.721985,346.836,22.492,1.2551,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,5.860096,187.627,25.767,1.4379,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,11.70064,187.94,24.376,1.3603,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,8.666944,253.725,19.358,1.0802,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,17.327265,253.822,17.429,0.9726,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,6.814432,0.63,39.397,2.1985,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,403.080505,1.364,85.253,4.7574,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.06032,1.417,44.305,2.4724,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,244.314713,4.5,140.672,7.85,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.059872,2.126,44.313,2.4728,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,244.963806,6.733,140.316,7.8301,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.06224,2.834,44.302,2.4722,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,245.091171,8.972,140.26,7.827,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.31696,108.404,112.48,6.2768,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,3.77536,145.617,37.773,2.1079,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,56.042912,156.953,10.178,0.568,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,4.359936,126.093,17.316,0.9663,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,8.743072,125.758,16.311,0.9102,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,8.758912,125.531,9.577,0.5344,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,16.757055,131.23,9.011,0.5028,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.082816,25.931,1620.872,90.4504,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,12.103264,22.711,1419.614,79.2195,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.016512,0.714,22.314,1.2452,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,84.516449,6.505,203.322,11.3461,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.027776,1.069,22.275,1.243,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,84.829666,9.721,202.596,11.3056,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.010528,1.429,22.341,1.2467,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,85.040802,12.929,202.118,11.2789,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.209728,163.83,169.99,9.486,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,1.69408,324.516,84.179,4.6975,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,25.22896,348.651,22.61,1.2617,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,2.873216,191.338,26.276,1.4663,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,5.701728,192.838,25.011,1.3957,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,5.658976,194.295,14.824,0.8272,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,11.304512,194.526,13.357,0.7454,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,6.346624,0.338,21.15,1.1803,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,174.968536,1.571,98.2,5.4799,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,5.698112,0.754,23.561,1.3148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,97.772224,5.623,175.756,9.8078,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,5.699936,1.13,23.556,1.3145,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,97.936287,8.42,175.483,9.7926,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,5.699008,1.507,23.563,1.3149,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,97.525375,11.274,176.244,9.835,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.193568,88.754,97.508,5.4413,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,2.034176,135.13,37.115,2.0711,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,28.450687,154.585,10.615,0.5923,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,3.260928,84.294,21.866,1.2202,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,6.6632,82.506,20.773,1.1592,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,4.332704,126.885,17.425,0.9724,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,8.717184,126.132,16.359,0.9129,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.082912,12.95,1618.896,90.3402,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,12.067936,11.389,1423.683,79.4466,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.03248,0.356,22.252,1.2417,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,44.521633,6.174,385.924,21.5359,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.020576,0.535,22.297,1.2443,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,44.587135,9.247,385.381,21.5056,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.038912,0.711,22.231,1.2406,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,44.693535,12.301,384.487,21.4557,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.143808,119.464,131.247,7.3241,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,1.101536,249.541,68.538,3.8247,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,12.486368,352.228,24.186,1.3496,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,2.870336,95.765,24.841,1.3862,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,5.69392,96.551,24.309,1.3565,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,2.874336,191.264,26.266,1.4657,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,5.691456,193.186,25.056,1.3982,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,6.37584,0.168,21.052,1.1748,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,86.38176,1.591,198.895,11.0991,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,5.697088,0.377,23.562,1.3148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,51.928898,5.293,330.875,18.464,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,5.698112,0.565,23.559,1.3147,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,51.948929,7.937,330.767,18.458,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,5.697184,0.754,23.564,1.315,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,52.066784,10.559,330.039,18.4173,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.14496,59.257,72.336,4.0366,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,1.179552,116.518,35.558,1.9843,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,14.668576,149.914,11.438,0.6383,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,3.194496,43.024,21.664,1.2089,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,6.634048,41.434,20.548,1.1466,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,3.244704,84.716,21.975,1.2263,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,6.66272,82.512,20.774,1.1593,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.082752,6.488,1621.977,90.5121,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,12.086496,5.686,1421.454,79.3222,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.017664,0.178,22.305,1.2447,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,23.63232,5.816,727.009,40.5697,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.0232,0.267,22.285,1.2436,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,23.636992,8.722,726.888,40.5629,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.036448,0.356,22.237,1.2409,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,23.667521,11.614,725.972,40.5119,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.13808,62.21,75.94,4.2377,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.618144,222.341,67.853,3.7865,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,5.99168,367.013,28.001,1.5625,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,2.870304,47.883,24.111,1.3455,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,5.692544,48.287,23.946,1.3363,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,2.874688,95.62,24.804,1.3841,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,5.69136,96.595,24.32,1.3571,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,6.495136,0.083,20.665,1.1532,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,39.754047,1.729,432.167,24.1165,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,5.699872,0.188,23.549,1.3141,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,16.983904,8.092,1011.6,56.4509,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,5.695328,0.283,23.568,1.3152,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,16.982847,12.139,1011.694,56.4561,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,5.696256,0.377,23.565,1.315,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,16.983232,16.185,1011.702,56.4566,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.579136,118.659,123.12,6.8705,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,7.245216,151.757,39.366,2.1967,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,111.429283,157.878,10.238,0.5713,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,8.34992,131.679,18.083,1.0091,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,16.770432,131.125,17.007,0.949,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,14.754048,149.045,11.371,0.6346,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,29.763617,147.766,10.146,0.5662,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.214048,20.065,1254.243,69.9912,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,22.324224,24.626,1539.311,85.8991,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.98208,1.23,38.456,2.146,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,167.387878,6.569,205.32,11.4576,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.980896,1.846,38.467,2.1466,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,168.255524,9.802,204.286,11.3999,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.981216,2.461,38.47,2.1468,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,168.949478,13.016,203.472,11.3545,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.37984,180.917,187.719,10.4754,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,3.403648,323.039,83.796,4.6761,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,50.71619,346.875,22.495,1.2553,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,5.853984,187.823,25.794,1.4394,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,11.688864,188.13,24.4,1.3616,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,8.658624,253.969,19.376,1.0813,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,17.316256,253.984,17.44,0.9732,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,6.799136,0.632,39.486,2.2034,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,403.131683,1.364,85.242,4.7568,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.057376,1.418,44.326,2.4736,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,243.366852,4.518,141.219,7.8805,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.05872,2.127,44.322,2.4733,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,243.411484,6.776,141.211,7.8801,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.059296,2.835,44.323,2.4734,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,243.632538,9.026,141.1,7.8739,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.312256,110.037,114.174,6.3713,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,3.774144,145.664,37.785,2.1085,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,56.005505,157.058,10.185,0.5684,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,4.35168,126.332,17.349,0.9681,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,8.742144,125.771,16.313,0.9103,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,8.737376,125.84,9.601,0.5358,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,16.745344,131.321,9.017,0.5032,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.08256,26.011,1625.898,90.7309,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,12.084544,22.746,1421.813,79.3423,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.040512,0.711,22.225,1.2402,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,84.230591,6.527,204.012,11.3846,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.027488,1.069,22.276,1.2431,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,84.514397,9.757,203.352,11.3478,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.043264,1.421,22.22,1.24,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,84.678978,12.984,202.981,11.3271,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.208832,164.533,170.719,9.5267,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,1.690208,325.259,84.372,4.7083,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,25.162176,349.576,22.67,1.2651,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,2.8736,191.313,26.273,1.4661,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,5.6944,193.086,25.043,1.3975,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,5.6456,194.756,14.859,0.8292,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,11.289888,194.778,13.374,0.7463,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,6.392448,0.336,20.999,1.1718,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,175.024261,1.571,98.169,5.4782,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,5.697728,0.754,23.562,1.3148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,97.088799,5.662,176.993,9.8769,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,5.697088,1.131,23.568,1.3152,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,97.194786,8.484,176.822,9.8673,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,5.697664,1.508,23.568,1.3152,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,97.461372,11.282,176.36,9.8415,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.195264,87.983,96.661,5.394,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,2.029376,135.449,37.202,2.076,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,28.414656,154.781,10.628,0.5931,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,3.262016,84.266,21.859,1.2198,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,6.671904,82.399,20.746,1.1577,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,4.329184,126.988,17.439,0.9732,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,8.71936,126.1,16.355,0.9127,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.083424,12.871,1608.96,89.7857,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,12.06592,11.391,1423.921,79.4599,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.049248,0.355,22.19,1.2383,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,44.441601,6.185,386.619,21.5747,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.039808,0.533,22.226,1.2403,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,44.56736,9.252,385.552,21.5152,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.021888,0.713,22.294,1.2441,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,44.674591,12.306,384.65,21.4648,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.144608,118.803,130.521,7.2835,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,1.179264,233.093,64.021,3.5726,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,12.47888,352.439,24.2,1.3505,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,2.873888,95.647,24.811,1.3845,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,5.694048,96.549,24.308,1.3565,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,2.872512,191.385,26.283,1.4667,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,5.692608,193.147,25.051,1.3979,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,6.65232,0.161,20.177,1.126,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,86.414017,1.59,198.821,11.0949,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,5.956128,0.361,22.537,1.2577,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,51.830208,5.303,331.505,18.4992,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,5.954464,0.541,22.545,1.2581,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,51.683998,7.978,332.463,18.5526,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,5.9544,0.721,22.546,1.2582,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,51.788864,10.615,331.81,18.5162,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.153376,56.006,68.366,3.8151,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,1.23664,111.139,33.917,1.8927,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,14.676384,149.834,11.431,0.6379,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,3.192352,43.053,21.679,1.2097,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,6.637088,41.415,20.538,1.1461,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,3.243328,84.752,21.985,1.2268,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,6.667872,82.448,20.758,1.1584,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.085856,6.253,1563.337,87.2398,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,12.075104,5.691,1422.795,79.397,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.30016,0.17,21.305,1.1889,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,23.597216,5.824,728.091,40.6301,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.316352,0.255,21.251,1.1859,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,23.638784,8.721,726.833,40.5599,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.30944,0.34,21.275,1.1872,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,23.66032,11.618,726.193,40.5242,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.1432,59.986,73.225,4.0862,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.651392,210.993,64.39,3.5932,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,6.078336,361.78,27.602,1.5403,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,3.001952,45.783,23.054,1.2865,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,5.973344,46.017,22.821,1.2735,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,2.999232,91.649,23.774,1.3267,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,5.693664,96.556,24.31,1.3566,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,6.602336,0.081,20.329,1.1345,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,39.737438,1.729,432.348,24.1266,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,5.97664,0.18,22.458,1.2533,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,16.975201,8.096,1012.119,56.4798,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,5.975328,0.27,22.464,1.2536,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,16.974527,12.145,1012.19,56.4838,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,5.975712,0.359,22.463,1.2535,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,16.977793,16.19,1012.026,56.4747,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], diff --git a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_context.png b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_context.png index b2bd9594f..816c1ae88 100755 Binary files a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_context.png and b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_context.png differ diff --git a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_generation.png b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_generation.png index f9cd7973c..559719b66 100755 Binary files a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_generation.png and b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_generation.png differ diff --git a/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260819_053502.csv b/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260821_053322.csv similarity index 55% rename from benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260819_053502.csv rename to benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260821_053322.csv index 3e1459da5..ff42dd111 100644 --- a/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260819_053502.csv +++ b/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260821_053322.csv @@ -1,13 +1,13 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.423072,944.859,213.866,2.6733,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.171008,1033.706,134.457,1.6807,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.884992,1051.051,86.378,1.0797,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,6.445888,992.244,56.148,0.7018,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,11.471104,1001.801,42.292,0.5286,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,20.019457,999.028,31.969,0.3996,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.62304,641.602,145.224,1.8153,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.544672,783.647,101.931,1.2741,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,3.54368,855.685,70.323,0.879,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,7.065184,905.269,51.226,0.6403,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,12.40432,926.433,39.11,0.4889,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,21.213024,942.817,30.17,0.3771,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.434912,919.136,208.044,2.6005,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.20704,1002.848,130.443,1.6305,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.98576,1015.578,83.463,1.0433,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,6.339296,1008.928,57.092,0.7136,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,11.662816,985.334,41.596,0.52,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,20.2728,986.544,31.569,0.3946,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.625504,639.074,144.652,1.8082,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.559456,776.218,100.965,1.2621,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,3.549984,854.165,70.198,0.8775,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,7.124992,897.671,50.796,0.635,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,12.367008,929.228,39.228,0.4903,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,21.403551,934.424,29.902,0.3738,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_context.png b/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_context.png index 912c49297..e66b4f29e 100755 Binary files a/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_context.png and b/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_context.png differ diff --git a/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260819_053407.csv b/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260821_053445.csv similarity index 59% rename from benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260819_053407.csv rename to benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260821_053445.csv index 5fd6f2cb1..56ca6ea31 100644 --- a/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260819_053407.csv +++ b/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260821_053445.csv @@ -1,13 +1,13 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.346912,1152.291,260.817,3.2602,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,0.98592,1227.764,159.699,1.9962,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.338528,1296.659,106.563,1.332,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,4.85376,1317.72,74.565,0.9321,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,8.681728,1323.673,55.879,0.6985,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,14.99008,1334.216,42.695,0.5337,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.532832,750.224,169.811,2.1226,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.283456,943.139,122.677,1.5335,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.890272,1049.131,86.22,1.0778,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,5.786624,1105.29,62.545,0.7818,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,9.943936,1155.656,48.787,0.6098,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,17.176064,1164.411,37.261,0.4658,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.341728,1169.771,264.774,3.3097,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,0.963648,1256.141,163.39,2.0424,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.319168,1307.483,107.453,1.3432,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,4.856288,1317.034,74.527,0.9316,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,8.71424,1318.734,55.671,0.6959,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,14.95264,1337.556,42.802,0.535,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.53504,747.128,169.11,2.1139,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.29328,935.975,121.745,1.5218,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.898816,1046.039,85.966,1.0746,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,5.709216,1120.276,63.393,0.7924,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,9.950688,1154.871,48.753,0.6094,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,17.361792,1151.955,36.863,0.4608,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_context.png b/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_context.png index c7b5e4a1d..6d79ea395 100755 Binary files a/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_context.png and b/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_context.png differ diff --git a/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260820_070037.csv b/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260821_070639.csv similarity index 59% rename from benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260820_070037.csv rename to benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260821_070639.csv index e8d362442..4a836b458 100644 --- a/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260820_070037.csv +++ b/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260821_070639.csv @@ -1,7 +1,7 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.059968,377.128,85.362,1.7784,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,3.149952,384.284,49.985,1.0414,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,8.217472,369.003,30.326,0.6318,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,16.873119,379.058,21.45,0.4469,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,30.32736,378.924,15.996,0.3333,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,53.068321,376.873,12.06,0.2512,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.066784,374.718,84.816,1.767,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,3.014016,401.616,52.239,1.0883,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,8.285632,365.968,30.076,0.6266,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,17.000128,376.226,21.289,0.4435,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,30.178017,380.799,16.076,0.3349,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,52.924641,377.896,12.093,0.2519,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_context.png b/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_context.png index 24588746a..fd33ef6fc 100755 Binary files a/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_context.png and b/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_context.png differ diff --git a/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260820_074424.csv b/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260821_072326.csv similarity index 51% rename from benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260820_074424.csv rename to benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260821_072326.csv index 5d60396ca..26942228b 100644 --- a/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260820_074424.csv +++ b/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260821_072326.csv @@ -1,13 +1,13 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.696032,235.693,53.348,2.977,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,5.016896,241.28,31.384,1.7513,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,12.300192,246.522,20.26,1.1306,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,25.908319,246.866,13.969,0.7795,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,46.474815,247.269,10.439,0.5825,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,80.731842,247.734,7.927,0.4424,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.164,343.422,77.733,4.3378,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,3.406912,355.3,46.215,2.579,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,8.34928,363.178,29.847,1.6656,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,17.364864,368.324,20.842,1.1631,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,30.899488,371.908,15.7,0.8761,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,53.261951,375.503,12.016,0.6705,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.719264,232.508,52.628,2.9368,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,5.016928,241.279,31.384,1.7513,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,12.364096,245.248,20.155,1.1247,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,25.955296,246.42,13.944,0.7781,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,46.544064,246.901,10.423,0.5816,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,80.715172,247.785,7.929,0.4425,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.168672,342.049,77.422,4.3204,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,3.406976,355.294,46.214,2.5789,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,8.381856,361.766,29.731,1.6591,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,17.307552,369.544,20.911,1.1669,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,30.831585,372.727,15.735,0.8781,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,53.192768,375.991,12.032,0.6714,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], diff --git a/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_context.png b/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_context.png index 24ce871d4..bee910336 100755 Binary files a/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_context.png and b/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_context.png differ diff --git a/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052837.csv b/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052837.csv deleted file mode 100644 index ad0aba304..000000000 --- a/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052837.csv +++ /dev/null @@ -1,31 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.209,1348.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.159,1772.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.162,1744.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.731,386.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.731,386.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.218,1293.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.213,1379.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.159,1849.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.162,1811.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.731,402.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.730,402.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.216,1359.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.389,1510.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.297,1977.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.298,1968.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.742,791.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.733,801.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.394,1491.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.718,1635.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.564,2080.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.583,2013.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.768,1528.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.734,1600.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,1.029,1141.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.446,1624.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.118,2101.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.155,2033.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.522,1543.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.453,1616.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,2.003,1172.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260821_052755.csv b/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260821_052755.csv new file mode 100644 index 000000000..0a4187249 --- /dev/null +++ b/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260821_052755.csv @@ -0,0 +1,31 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.210,1346.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.160,1768.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.162,1743.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.732,386.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.730,387.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.216,1306.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.211,1389.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.159,1842.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.162,1812.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.732,401.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.730,402.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.217,1356.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.387,1517.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.297,1974.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.300,1958.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.736,797.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.731,803.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.396,1484.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.722,1625.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.564,2082.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.585,2007.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.774,1516.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.735,1598.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,1.028,1143.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.413,1662.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.119,2098.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.158,2028.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.525,1540.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.452,1618.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,2.017,1164.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_no_mask.png b/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_no_mask.png index 4d4a6d513..3df392424 100755 Binary files a/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_no_mask.png and b/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_no_mask.png differ diff --git a/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052715.csv b/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052715.csv deleted file mode 100644 index 47471669d..000000000 --- a/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052715.csv +++ /dev/null @@ -1,31 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.155,1819.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.109,2600.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.116,2438.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.596,474.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2540.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.494,572.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.191,1475.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.157,1872.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.109,2699.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.116,2531.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.593,495.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.493,595.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.191,1540.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.295,1991.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.205,2863.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.217,2700.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.597,983.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.494,1189.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.357,1645.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.562,2090.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.387,3033.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.418,2806.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.624,1880.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.494,2376.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.557,2109.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.115,2107.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.768,3056.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.833,2819.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.240,1894.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.980,2397.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.105,2124.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260821_052854.csv b/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260821_052854.csv new file mode 100644 index 000000000..d9e8678e5 --- /dev/null +++ b/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260821_052854.csv @@ -0,0 +1,31 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.158,1786.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.110,2569.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.117,2406.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.602,469.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.494,572.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.193,1460.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.158,1862.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.110,2666.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.117,2502.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.597,491.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.493,595.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.193,1519.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.299,1964.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.209,2811.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.221,2659.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.603,973.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.497,1182.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.362,1623.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.569,2063.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.395,2976.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.426,2758.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.629,1866.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.501,2343.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.563,2087.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.127,2085.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.782,3001.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.844,2781.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.247,1883.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.988,2376.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.111,2114.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_no_mask.png b/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_no_mask.png index 2066a19e0..df3949d3e 100755 Binary files a/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_no_mask.png and b/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_no_mask.png differ diff --git a/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260820_052459.csv b/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260821_052709.csv similarity index 68% rename from benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260820_052459.csv rename to benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260821_052709.csv index 5df344ddf..bb3dbc0e4 100644 --- a/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260820_052459.csv +++ b/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260821_052709.csv @@ -1,16 +1,16 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.718,393.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.510,554.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.639,442.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.721,407.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.510,576.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.642,458.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.473,399.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.021,575.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.330,441.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.254,521.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,1.547,759.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.052,572.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,3.831,613.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,2.598,904.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,3.495,672.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.508,556.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.643,439.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.716,410.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.509,577.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.638,460.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.464,401.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.016,578.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.310,448.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.212,531.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,1.519,773.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,1.990,590.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,3.717,632.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,2.538,925.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,3.355,700.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_no_mask.png b/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_no_mask.png index 9c9ece71d..6c15484b0 100755 Binary files a/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_no_mask.png and b/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_no_mask.png differ diff --git a/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260820_064429.csv b/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260821_062316.csv similarity index 66% rename from benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260820_064429.csv rename to benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260821_062316.csv index 7db6d569c..18f3b1219 100644 --- a/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260820_064429.csv +++ b/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260821_062316.csv @@ -1,21 +1,21 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,1.996,141.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,1.268,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.994,284.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,1.987,142.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.997,147.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.268,232.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.993,295.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.988,148.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.845,318.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.277,460.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,2.036,288.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.998,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.699,317.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.543,462.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.547,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.990,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.345,320.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,5.032,467.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.045,333.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.941,296.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,1.998,141.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,1.268,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,933.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.992,285.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,1.985,142.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.996,147.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.267,232.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,933.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.993,296.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.988,148.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.844,318.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.276,460.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,933.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,2.038,288.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,2.136,275.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.709,317.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.545,461.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,889.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.554,330.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.990,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.366,319.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,5.034,467.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,889.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.103,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.938,296.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 diff --git a/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_no_mask.png b/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_no_mask.png index bec5d8c8c..e0fd33974 100755 Binary files a/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_no_mask.png and b/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_no_mask.png differ diff --git a/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260819_052447.csv b/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260821_052309.csv similarity index 51% rename from benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260819_052447.csv rename to benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260821_052309.csv index 6391d6f9f..26e78cae2 100644 --- a/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260819_052447.csv +++ b/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260821_052309.csv @@ -1,9 +1,9 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,3.319,165.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,1.651,165.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,0.829,163.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,0.419,159.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,6.695,164.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,3.246,168.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,1.638,165.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,0.816,163.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,3.385,162.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,1.686,162.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,0.841,161.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,0.422,158.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,6.775,162.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,3.286,166.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,1.625,167.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,0.832,160.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_top_left.png b/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_top_left.png index d46ab6ebf..fd7daa692 100755 Binary files a/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_top_left.png and b/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_top_left.png differ diff --git a/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260819_052317.csv b/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260821_052358.csv similarity index 51% rename from benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260819_052317.csv rename to benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260821_052358.csv index 57374bc7f..42a445e3e 100644 --- a/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260819_052317.csv +++ b/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260821_052358.csv @@ -1,9 +1,9 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,3.092,177.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,1.506,181.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,0.744,182.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,0.368,181.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,6.200,177.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,3.006,181.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,1.482,183.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,0.730,183.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,3.118,176.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,1.517,180.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,0.753,180.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,0.376,177.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,6.251,175.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,3.029,180.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,1.495,181.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,0.737,181.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_top_left.png b/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_top_left.png index b9c5b9e60..a0a4869e7 100755 Binary files a/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_top_left.png and b/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_top_left.png differ diff --git a/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260820_052740.csv b/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260821_053022.csv similarity index 67% rename from benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260820_052740.csv rename to benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260821_053022.csv index 39253dbab..6404dd18c 100644 --- a/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260820_052740.csv +++ b/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260821_053022.csv @@ -1,9 +1,9 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,9.889,55.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,4.872,56.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,2.377,57.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,1.177,57.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,19.848,55.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,9.735,56.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,4.729,57.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,2.339,57.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,9.938,55.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,4.899,56.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,2.390,57.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,1.183,56.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,19.893,55.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,9.784,56.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,4.762,57.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,2.349,57.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_top_left.png b/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_top_left.png index 2549b1e0f..b7296482b 100755 Binary files a/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_top_left.png and b/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_top_left.png differ diff --git a/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260820_055712.csv b/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260820_055712.csv deleted file mode 100644 index 46698ff85..000000000 --- a/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260820_055712.csv +++ /dev/null @@ -1,9 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,9.450,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,455.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,4.714,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,2.348,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,1.197,56.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,18.845,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,456.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,9.393,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,4.670,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,2.309,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 diff --git a/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260821_053932.csv b/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260821_053932.csv new file mode 100644 index 000000000..cc6059b6f --- /dev/null +++ b/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260821_053932.csv @@ -0,0 +1,9 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,9.447,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,4.713,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,2.518,54.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,1.258,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,18.840,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,449.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,9.382,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,4.662,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,2.405,55.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 diff --git a/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_top_left.png b/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_top_left.png index 86716eeb4..0eead84fe 100755 Binary files a/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_top_left.png and b/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_top_left.png differ diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_20260819_054000.csv b/benchmark/attention_training/results/dsv3/gb200/dsv3_20260819_054000.csv deleted file mode 100644 index a670f9e0f..000000000 --- a/benchmark/attention_training/results/dsv3/gb200/dsv3_20260819_054000.csv +++ /dev/null @@ -1,131 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,54.527,1613.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,206.815,1106.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,209.620,1091.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,114.865,1532.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,418.462,1093.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,414.584,1103.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,34.847,2524.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,False,121.353,1885.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,True,121.973,1875.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,69.525,2530.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,233.177,1962.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,234.969,1947.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,fwd,False,40.460,2174.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,False,140.254,1631.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,True,140.361,1629.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,76.959,2286.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,284.477,1608.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,283.137,1615.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,61.808,1423.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,124.142,1417.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,52.579,1673.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,210.398,1087.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,212.466,1076.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,110.435,1593.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,397.750,1150.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,397.369,1151.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,13.783,1596.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,51.369,1113.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,49.281,1160.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,26.039,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,106.657,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,102.765,1113.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,8.577,2564.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,False,28.196,2028.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,True,29.506,1938.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,17.559,2505.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,54.953,2081.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,54.993,2079.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,fwd,False,9.106,2415.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,False,33.638,1700.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,True,34.596,1653.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,18.367,2395.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,64.688,1768.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,64.224,1780.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,14.011,1570.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,29.166,1508.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,13.681,1607.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,52.526,1089.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,53.692,1065.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,23.911,1839.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,101.607,1125.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,103.043,1110.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.481,1579.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,13.374,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,12.749,1121.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,6.301,1745.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,25.578,1118.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,25.108,1139.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,2.304,2387.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,False,7.435,1923.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,True,7.300,1958.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,4.268,2576.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,13.257,2156.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,13.310,2148.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,fwd,False,2.461,2234.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,False,8.558,1670.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,True,9.249,1546.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,4.617,2381.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,15.803,1809.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,15.924,1795.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.827,1437.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,6.996,1572.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.617,1520.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,13.340,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,13.309,1074.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,6.193,1775.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,24.152,1184.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,23.913,1195.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.998,1377.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.463,1032.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.302,1082.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.699,1618.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.997,1192.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,5.993,1193.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.666,2064.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.088,1712.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.090,1710.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.141,2410.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,3.496,2044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,3.503,2040.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.708,1942.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.557,1398.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.545,1405.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.145,2401.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,4.188,1706.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,4.179,1710.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,1.062,1294.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.764,1559.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,1.013,1357.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.710,964.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.840,931.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.608,1710.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,6.084,1175.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,6.269,1140.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.322,1068.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.054,848.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,0.987,906.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.471,1458.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.667,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.638,1090.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.210,1638.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.676,1322.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.675,1325.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.320,2151.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,0.992,1802.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.002,1782.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.222,1551.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.848,1054.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.846,1056.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.337,2038.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.175,1521.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.201,1488.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.340,1012.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.496,1385.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.323,1065.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.193,749.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,1.237,722.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.471,1459.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.755,1018.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.804,990.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_20260821_053908.csv b/benchmark/attention_training/results/dsv3/gb200/dsv3_20260821_053908.csv new file mode 100644 index 000000000..a8d43ab03 --- /dev/null +++ b/benchmark/attention_training/results/dsv3/gb200/dsv3_20260821_053908.csv @@ -0,0 +1,141 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,51.625,1704.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,212.837,1075.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,206.302,1109.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,105.708,1664.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,414.793,1103.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,406.258,1126.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,34.942,2517.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,False,121.943,1876.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,True,119.423,1915.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,72.147,2438.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,235.344,1944.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,233.158,1962.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,fwd,False,37.429,2350.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,False,137.520,1663.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,True,141.320,1618.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,71.725,2453.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,279.355,1637.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,279.301,1638.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,56.724,1551.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,123.755,1422.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,mxfp8,top_left,2,32768,32768,128,128,192,128,fwd,False,35.959,2446.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,mxfp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,73.758,2385.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,52.165,1686.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,207.987,1100.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,211.768,1080.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,107.241,1640.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,391.136,1169.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,397.389,1151.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,11.944,1841.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,50.674,1128.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,50.811,1125.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,23.858,1843.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,98.477,1161.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,100.721,1135.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,8.365,2629.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,False,29.661,1928.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,True,29.693,1926.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,16.623,2646.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,55.967,2043.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,57.708,1982.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,fwd,False,8.964,2453.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,False,34.289,1668.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,True,33.885,1687.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,17.370,2532.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,64.404,1775.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,66.614,1717.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,13.502,1629.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,29.905,1471.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,mxfp8,top_left,2,16384,16384,128,128,192,128,fwd,False,8.815,2495.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,mxfp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,16.635,2644.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,13.611,1616.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,52.031,1099.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,51.529,1110.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,22.753,1933.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,104.117,1098.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,104.705,1092.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.287,1673.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,11.957,1196.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,11.630,1229.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.961,1845.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,24.525,1166.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,23.816,1200.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,2.200,2499.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,False,7.149,2000.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,True,7.101,2013.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,4.088,2690.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,13.119,2179.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,13.614,2100.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,fwd,False,2.391,2300.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,False,8.447,1692.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,True,8.768,1630.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,4.256,2583.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,16.600,1722.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,15.984,1788.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.502,1570.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,6.552,1678.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,mxfp8,top_left,2,8192,8192,128,128,192,128,fwd,False,2.377,2313.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,mxfp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,4.133,2660.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.332,1650.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,12.878,1110.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,13.290,1076.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.918,1858.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,23.715,1205.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,24.941,1146.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.961,1431.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.225,1108.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.148,1135.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.536,1789.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.778,1237.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,5.772,1238.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.634,2169.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.061,1734.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.062,1733.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.079,2547.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,3.359,2128.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,3.356,2130.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.671,2048.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.493,1434.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.493,1434.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.102,2494.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,4.023,1777.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,4.021,1777.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.989,1390.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.643,1673.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn_oss,mxfp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.703,1957.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn_oss,mxfp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.112,2472.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.938,1465.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.710,963.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.844,930.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.513,1817.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,6.091,1173.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,6.299,1135.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.306,1124.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.017,879.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,0.979,913.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.444,1548.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.650,1083.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.616,1105.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.198,1735.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.669,1336.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.668,1338.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.304,2257.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,0.966,1850.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,0.967,1847.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.211,1628.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.834,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.836,1070.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.320,2146.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.170,1528.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.170,1527.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.320,1074.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.463,1484.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn_oss,mxfp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.244,1408.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn_oss,mxfp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.329,2091.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,299.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.306,1122.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.193,749.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,1.241,720.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.445,1545.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.759,1016.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.808,988.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,149.000 diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask.png b/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask.png index 704fb7a27..f3e5b4947 100755 Binary files a/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask.png and b/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask.png differ diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask_det_overhead.png b/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask_det_overhead.png index 2600ba897..5d94cdbb7 100755 Binary files a/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask_det_overhead.png and b/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left.png b/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left.png index af38b376a..fe5dee0f3 100755 Binary files a/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left.png and b/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left.png differ diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left_det_overhead.png b/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left_det_overhead.png index 5554c280b..55f0d967d 100755 Binary files a/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left_det_overhead.png and b/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_20260819_053824.csv b/benchmark/attention_training/results/dsv3/gb300/dsv3_20260819_053824.csv deleted file mode 100644 index 9ec037eb6..000000000 --- a/benchmark/attention_training/results/dsv3/gb300/dsv3_20260819_053824.csv +++ /dev/null @@ -1,131 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,42.710,2060.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,171.146,1336.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,172.219,1328.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,88.301,1992.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,360.136,1270.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,347.818,1315.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,25.932,3392.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,False,89.690,2550.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,True,91.080,2511.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,50.511,3483.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,184.480,2479.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,182.464,2507.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,fwd,False,28.480,3089.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,False,115.705,1977.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,True,115.849,1974.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,56.801,3097.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,223.500,2047.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,225.941,2024.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,48.876,1800.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,103.721,1696.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,44.367,1983.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,181.850,1258.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,190.938,1198.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,87.228,2017.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,353.198,1295.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,365.503,1251.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,10.642,2066.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,42.932,1332.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,40.796,1402.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,20.248,2172.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,87.126,1312.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,83.826,1364.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,6.456,3407.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,False,23.126,2472.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,True,22.639,2526.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,12.460,3530.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,44.072,2595.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,42.966,2661.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,fwd,False,7.230,3042.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,False,28.457,2009.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,True,28.648,1996.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,14.052,3130.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,55.100,2075.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,53.516,2137.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,12.048,1825.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,23.498,1872.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,11.408,1928.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,46.057,1241.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,47.925,1193.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,20.290,2168.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,89.876,1272.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,91.583,1249.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,2.849,1930.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,10.934,1307.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,10.611,1347.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.117,2149.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,20.999,1361.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,20.412,1400.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,1.706,3224.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,False,6.048,2364.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,True,6.051,2363.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,3.142,3499.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,11.013,2596.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,10.945,2612.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,fwd,False,1.903,2890.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,False,7.699,1857.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,True,7.701,1856.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,3.503,3139.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,13.705,2086.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,13.680,2090.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.186,1726.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.885,1868.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.041,1808.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,12.228,1169.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,12.843,1113.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.163,2129.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,22.006,1299.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,22.709,1259.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.812,1693.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.095,1155.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,2.967,1205.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.346,2042.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.576,1282.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,5.545,1289.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.476,2885.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,False,1.763,2027.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,True,1.762,2029.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,0.807,3407.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,2.901,2464.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,2.905,2460.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.529,2597.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.282,1566.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.287,1563.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,0.902,3049.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,3.638,1965.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,3.634,1967.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.887,1550.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.511,1820.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.864,1591.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.545,1008.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.716,962.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.363,2016.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.900,1211.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,6.119,1168.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.254,1353.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,0.970,922.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,0.922,969.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.370,1855.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.594,1121.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.540,1161.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.145,2369.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.581,1538.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.581,1537.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.221,3109.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,0.835,2139.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,0.833,2145.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.161,2130.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.768,1164.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.768,1164.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.248,2772.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.052,1698.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.051,1700.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.272,1262.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.404,1702.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.271,1269.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.142,783.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,1.198,746.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.381,1805.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.703,1049.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.757,1017.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_20260821_053959.csv b/benchmark/attention_training/results/dsv3/gb300/dsv3_20260821_053959.csv new file mode 100644 index 000000000..cbaeca477 --- /dev/null +++ b/benchmark/attention_training/results/dsv3/gb300/dsv3_20260821_053959.csv @@ -0,0 +1,141 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,44.064,1996.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,176.105,1299.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,184.827,1237.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,91.725,1918.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,373.123,1226.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,370.589,1234.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,26.397,3332.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,False,93.816,2438.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,True,92.282,2478.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,51.800,3396.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,198.337,2306.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,197.431,2317.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,fwd,False,28.921,3041.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,False,122.003,1875.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,True,122.640,1865.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,57.868,3040.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,233.270,1961.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,237.759,1924.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,48.684,1807.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,103.385,1702.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,mxfp8,top_left,2,32768,32768,128,128,192,128,fwd,False,30.608,2874.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,mxfp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,62.762,2803.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,46.131,1907.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,184.988,1236.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,196.805,1162.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,92.921,1893.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,357.398,1280.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,368.862,1240.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,10.736,2048.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,43.690,1309.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,41.511,1377.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,21.438,2052.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,87.987,1300.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,90.141,1269.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,6.482,3393.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,False,22.718,2517.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,True,22.794,2508.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,12.642,3479.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,43.472,2630.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,45.606,2507.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,fwd,False,7.239,3038.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,False,28.908,1978.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,True,28.903,1978.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,14.093,3121.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,55.403,2064.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,55.623,2056.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,11.977,1836.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,23.703,1855.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,mxfp8,top_left,2,16384,16384,128,128,192,128,fwd,False,7.811,2816.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,mxfp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,15.078,2917.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,11.474,1917.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,45.962,1244.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,48.453,1180.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,20.455,2150.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,88.294,1295.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,92.324,1239.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,2.857,1925.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,11.028,1296.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,10.615,1347.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.214,2109.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,21.146,1352.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,20.474,1396.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,1.707,3221.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,False,6.069,2355.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,True,6.093,2346.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,3.155,3485.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,11.045,2588.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,10.970,2606.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,fwd,False,1.906,2885.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,False,7.730,1849.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,True,7.762,1842.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,3.518,3125.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,13.664,2092.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,13.865,2062.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.166,1737.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.873,1872.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,mxfp8,top_left,2,8192,8192,128,128,192,128,fwd,False,2.150,2558.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,mxfp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,3.805,2890.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.044,1806.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,12.263,1166.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,12.895,1109.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.215,2108.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,22.003,1299.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,23.059,1240.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.812,1694.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.098,1154.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,2.970,1203.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.339,2054.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.600,1276.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,5.591,1278.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.477,2884.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,False,1.778,2010.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,True,1.777,2012.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,0.805,3413.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,2.911,2455.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,2.910,2456.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.533,2582.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.301,1553.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.299,1555.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,0.901,3050.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,3.655,1956.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,3.655,1955.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.880,1562.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.503,1828.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,mxfp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.643,2138.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn_oss,mxfp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.004,2738.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.863,1592.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.563,1003.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.730,958.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.355,2028.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.934,1204.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,6.151,1162.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.253,1357.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,0.974,918.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,0.927,964.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.369,1863.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.596,1119.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.549,1153.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.146,2347.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.587,1523.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.587,1522.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.221,3104.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,0.843,2120.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,0.841,2124.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.162,2121.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.774,1155.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.774,1155.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.249,2765.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.063,1681.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.065,1677.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.271,1268.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.399,1724.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,cudnn_oss,mxfp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.222,1551.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,cudnn_oss,mxfp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.288,2390.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,299.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.274,1254.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.150,777.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,1.205,741.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.379,1812.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.714,1043.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.766,1011.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,149.000 diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask.png b/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask.png index 8a8cf33d7..6d6d5ac8a 100755 Binary files a/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask.png and b/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask.png differ diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask_det_overhead.png b/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask_det_overhead.png index ad936c2c1..da5646e34 100755 Binary files a/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask_det_overhead.png and b/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left.png b/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left.png index 87ea839b4..5f9bb1190 100755 Binary files a/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left.png and b/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left.png differ diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left_det_overhead.png b/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left_det_overhead.png index 24181b444..d46c3db3a 100755 Binary files a/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left_det_overhead.png and b/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/h200/dsv3_20260820_054000.csv b/benchmark/attention_training/results/dsv3/h200/dsv3_20260821_054358.csv similarity index 59% rename from benchmark/attention_training/results/dsv3/h200/dsv3_20260820_054000.csv rename to benchmark/attention_training/results/dsv3/h200/dsv3_20260821_054358.csv index 2a8db4978..6a493a1a4 100644 --- a/benchmark/attention_training/results/dsv3/h200/dsv3_20260820_054000.csv +++ b/benchmark/attention_training/results/dsv3/h200/dsv3_20260821_054358.csv @@ -1,67 +1,67 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,146.696,600.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,456.748,501.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,283.865,620.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,1438.840,318.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,143.816,612.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,285.046,617.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,130.209,676.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,468.108,489.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,527.921,433.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,276.034,637.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,932.262,491.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,973.394,470.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,36.083,609.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,115.401,495.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,69.954,629.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,357.644,320.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,35.377,622.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,70.153,627.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,31.894,690.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,121.612,470.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,138.332,413.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,67.035,656.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,235.523,486.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,247.941,461.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,8.643,636.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,28.747,497.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,35.546,402.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,16.645,661.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,90.402,316.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,91.392,313.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,9.059,607.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,17.440,630.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,7.806,704.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,29.594,483.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,35.041,408.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,15.847,694.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,56.948,502.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,62.811,455.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,2.256,609.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,7.337,487.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,9.029,396.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,4.125,666.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,22.347,320.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,22.292,321.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,2.352,584.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,4.399,625.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,2.067,665.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,7.326,488.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,9.626,371.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,3.953,695.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,14.136,506.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,16.664,429.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.628,547.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,2.164,413.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,2.528,354.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,146.892,599.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,453.173,505.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,284.168,619.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,1428.847,320.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,142.283,618.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,284.620,618.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,129.075,681.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,464.109,493.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,526.909,434.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,273.818,642.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,927.956,493.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,969.468,472.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,35.939,612.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,114.497,499.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,70.852,621.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,352.443,324.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,35.385,621.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,70.150,627.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,31.551,697.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,121.215,472.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,136.999,417.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,67.001,656.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,231.137,495.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,246.910,463.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,8.936,615.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,28.632,499.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,35.147,407.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,17.047,645.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,88.972,321.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,90.874,315.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,9.150,601.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,17.314,635.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,7.702,714.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,29.504,485.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,34.882,410.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,15.796,696.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,57.244,499.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,62.779,455.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,2.262,608.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,7.455,479.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,9.017,396.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,4.084,673.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,22.345,320.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,22.118,323.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,2.350,585.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,4.436,620.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,2.070,664.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,7.312,489.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,9.628,371.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,3.910,703.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,13.693,522.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,16.604,430.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.629,547.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,2.160,414.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,2.529,353.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.069,643.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,5.916,302.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,5.644,317.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.648,531.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.142,602.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,5.948,300.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,5.678,315.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.651,528.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.153,596.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.606,567.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,2.156,415.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,2.864,312.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.038,662.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,3.720,480.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,4.691,381.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,2.153,415.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,2.868,312.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.037,663.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,3.731,479.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,4.724,378.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask.png b/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask.png index c6ca1bfef..386c6cc6d 100755 Binary files a/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask.png and b/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask.png differ diff --git a/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask_det_overhead.png b/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask_det_overhead.png index 09f990e7c..e683503ff 100755 Binary files a/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask_det_overhead.png and b/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/h200/dsv3_top_left.png b/benchmark/attention_training/results/dsv3/h200/dsv3_top_left.png index e421455ee..effea5cb4 100755 Binary files a/benchmark/attention_training/results/dsv3/h200/dsv3_top_left.png and b/benchmark/attention_training/results/dsv3/h200/dsv3_top_left.png differ diff --git a/benchmark/attention_training/results/dsv3/h200/dsv3_top_left_det_overhead.png b/benchmark/attention_training/results/dsv3/h200/dsv3_top_left_det_overhead.png index a7508a56b..f4a03736c 100755 Binary files a/benchmark/attention_training/results/dsv3/h200/dsv3_top_left_det_overhead.png and b/benchmark/attention_training/results/dsv3/h200/dsv3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260820_054555.csv b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260820_054555.csv deleted file mode 100644 index 358ea563b..000000000 --- a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260820_054555.csv +++ /dev/null @@ -1,41 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,387.010,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,776.002,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,221.052,398.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,437.820,402.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,252.639,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,1425.478,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,473.856,371.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,2906.651,157.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,99.786,220.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,197.184,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,56.106,392.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,109.909,400.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,63.849,344.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,361.592,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,117.886,373.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,723.681,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,24.161,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,48.595,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,13.740,400.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,27.343,402.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,15.850,347.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,455.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,92.939,154.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,29.084,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,182.998,156.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,6.237,220.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,12.055,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,3.661,376.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,6.654,413.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,4.149,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,24.481,146.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,7.219,381.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,455.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,46.888,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.675,205.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,3.105,221.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.085,317.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.895,363.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.185,290.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,6.793,132.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,455.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.930,356.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,12.298,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,462.000 diff --git a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260821_052822.csv b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260821_052822.csv new file mode 100644 index 000000000..785e841c9 --- /dev/null +++ b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260821_052822.csv @@ -0,0 +1,41 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,387.462,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,460.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,775.969,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,220.669,399.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,436.942,403.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,252.936,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,458.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,1425.592,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,471.573,373.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,2893.757,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,99.281,222.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,196.681,224.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,55.970,393.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,109.765,401.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,63.817,345.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,458.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,361.730,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,117.559,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,723.164,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,24.259,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,48.494,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,13.773,399.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,27.376,402.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,15.816,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,448.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,92.957,154.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,29.303,375.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,182.972,156.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,6.240,220.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,12.042,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,3.658,376.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,6.645,414.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,4.149,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,24.502,146.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,7.224,381.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,46.879,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.803,191.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,3.102,222.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.088,316.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.917,359.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.256,274.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,6.790,132.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.989,345.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,12.298,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,448.000 diff --git a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_no_mask.png b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_no_mask.png index 87263753b..1a8c069bc 100755 Binary files a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_no_mask.png and b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_no_mask.png differ diff --git a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_top_left.png b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_top_left.png index eb3c21fd2..1a9f8343a 100755 Binary files a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_top_left.png and b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_top_left.png differ diff --git a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260819_051737.csv b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260821_051601.csv similarity index 51% rename from benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260819_051737.csv rename to benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260821_051601.csv index 04d57def2..8532bae7a 100644 --- a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260819_051737.csv +++ b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260821_051601.csv @@ -1,66 +1,66 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.611,170.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,3.651,188.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2306.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,4.727,145.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.499,183.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,False,91.317,8.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,True,91.340,8.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.559,176.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,False,106.970,6.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,True,106.949,6.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,4.430,62.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.818,151.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,4.677,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,28.625,24.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.809,169.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,1.812,189.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,2.375,144.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.751,182.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,False,23.589,15.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,True,23.591,15.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.782,175.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,False,27.627,12.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,4221.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,True,27.612,12.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,2.211,62.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.913,150.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,2.321,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.908,43.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.407,167.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2362.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,0.899,190.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.194,143.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.378,180.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,False,6.291,27.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,True,6.290,27.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.394,173.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,False,7.360,23.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,True,7.338,23.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,1.108,62.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.459,149.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.157,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,2.328,73.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.207,164.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.459,184.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.608,139.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.192,176.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.770,48.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.770,48.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.200,169.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,False,2.066,41.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,True,2.067,41.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.555,61.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.232,146.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.579,146.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.762,111.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.106,156.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.233,178.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.312,133.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.099,168.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.547,76.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.549,76.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.104,160.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.640,65.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.641,65.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.280,59.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.119,140.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.292,142.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.324,129.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.610,170.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,3.658,187.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,4.739,145.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.499,183.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,False,91.142,8.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,True,91.083,8.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.557,176.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,False,106.742,6.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,True,106.787,6.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,4.607,60.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.824,150.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,4.667,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,28.810,24.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.809,169.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,1.824,188.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,2.384,144.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.752,182.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,False,23.535,15.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,True,23.535,15.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.782,175.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,False,27.587,12.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,True,27.585,12.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,2.310,59.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.915,150.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,2.325,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.972,43.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.407,167.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2241.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,0.904,189.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.200,142.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.378,180.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,False,6.279,27.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,True,6.279,27.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.395,173.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,False,7.350,23.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,True,7.351,23.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,1.154,59.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.461,148.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.157,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,2.353,72.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.207,164.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.461,183.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.610,139.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.192,177.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.773,48.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.773,48.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.200,169.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,False,2.068,41.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,True,2.071,41.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.577,59.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.233,145.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.580,146.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.768,110.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.107,156.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.234,177.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.314,133.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.099,168.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.550,76.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.550,76.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.104,160.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.643,65.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.643,65.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.291,57.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.120,139.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.292,142.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.324,128.000,0.000,10,128,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left.png b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left.png index c81060675..1bbf24dc4 100755 Binary files a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left.png and b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left.png differ diff --git a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left_det_overhead.png b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left_det_overhead.png index d7c58dbf5..41f7813f7 100755 Binary files a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left_det_overhead.png and b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260819_051623.csv b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260821_051700.csv similarity index 52% rename from benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260819_051623.csv rename to benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260821_051700.csv index 058e38790..ec212b9e9 100644 --- a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260819_051623.csv +++ b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260821_051700.csv @@ -1,66 +1,66 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.423,193.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2194.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,3.449,199.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,4.335,158.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.265,217.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,False,74.869,9.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,True,74.899,9.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.302,211.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,False,94.871,7.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,True,94.867,7.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,4.384,63.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.681,163.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,4.357,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,27.971,25.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.715,192.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,1.720,199.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,2.177,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.635,216.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,False,19.391,18.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,True,19.392,18.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5060.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.654,209.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,False,24.459,14.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,True,24.465,14.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,2.194,62.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.843,162.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,2.178,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.722,44.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.360,189.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,0.862,198.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.098,155.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.320,213.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,False,5.201,33.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,True,5.201,33.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.330,207.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,False,6.522,26.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,True,6.520,26.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,1.099,62.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.424,161.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.088,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,2.266,75.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.183,185.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.442,192.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.562,151.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.162,209.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.485,57.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.484,57.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.168,201.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.843,46.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.844,46.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.551,61.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.215,158.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.547,155.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.740,114.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.095,175.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.228,183.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.289,144.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.084,198.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.468,89.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.468,89.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.087,191.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.574,72.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.575,72.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.277,60.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.111,150.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.277,150.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.309,135.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.425,193.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,3.457,198.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2474.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,4.342,158.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.265,217.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,False,74.816,9.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,True,74.799,9.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.304,210.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,False,94.708,7.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,True,94.706,7.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,4.470,61.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2456.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.681,163.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,4.357,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,28.004,24.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.716,191.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,1.721,199.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,2.180,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.635,216.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,False,19.396,18.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,True,19.395,18.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.654,209.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,False,24.458,14.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,True,24.468,14.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,2.238,61.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.843,162.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,2.180,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.724,44.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.361,189.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,0.865,197.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.100,155.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.320,213.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,False,5.206,33.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,True,5.206,33.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.330,207.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,False,6.523,26.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,True,6.522,26.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,1.123,61.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.424,161.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.089,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,2.266,75.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.185,183.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.442,191.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.563,150.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.163,208.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.489,57.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.489,57.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.169,200.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.848,46.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.847,46.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.565,60.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.215,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.549,154.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.741,114.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.095,175.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.228,182.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.290,143.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.084,197.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.470,88.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.471,88.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.088,190.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.577,72.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.577,72.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.285,58.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.111,149.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.278,150.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.311,134.000,0.000,10,128,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left.png b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left.png index 02efc321e..630288007 100755 Binary files a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left.png and b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left.png differ diff --git a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left_det_overhead.png b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left_det_overhead.png index 5aeeb4b09..20263e880 100755 Binary files a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left_det_overhead.png and b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260820_052044.csv b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260821_052207.csv similarity index 66% rename from benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260820_052044.csv rename to benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260821_052207.csv index 599d3891b..d9150dc14 100644 --- a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260820_052044.csv +++ b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260821_052207.csv @@ -1,34 +1,34 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.030,135.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,7.261,94.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,2.958,93.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.699,102.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,6.008,114.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,6.227,110.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.020,134.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,3.632,94.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,1.462,94.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.358,101.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,3.002,114.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,3.116,110.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.043,134.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,7.288,94.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,2.974,92.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.698,102.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,6.020,114.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,6.251,110.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.028,133.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,3.645,94.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,1.473,93.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.366,100.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,3.011,114.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,3.120,110.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.518,132.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.819,94.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,16.638,10.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.732,93.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.683,100.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.499,114.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.555,110.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.264,128.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.914,92.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,4.623,18.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.372,91.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.824,93.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,16.721,10.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.735,93.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.686,99.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.504,113.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.556,110.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.263,129.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.917,92.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,4.637,18.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.371,91.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.352,96.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.753,112.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.755,112.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.784,108.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.135,124.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.464,90.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,1.321,32.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.134,124.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.463,90.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,1.324,31.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.190,88.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.179,93.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.387,108.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.178,93.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.386,108.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.404,103.000,0.000,10,128,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left.png b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left.png index e40778cb2..fbf7b8a5c 100755 Binary files a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left.png and b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left.png differ diff --git a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left_det_overhead.png b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left_det_overhead.png index f64108ba3..88a9bc913 100755 Binary files a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left_det_overhead.png and b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260820_053226.csv b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260820_053226.csv deleted file mode 100644 index 6dfcfc10e..000000000 --- a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260820_053226.csv +++ /dev/null @@ -1,44 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.960,93.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,14.277,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,3.007,91.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.918,94.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,14.608,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,14.612,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,112.398,2.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,12.965,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.483,92.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,7.095,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,1.509,91.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.477,93.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,7.225,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.233,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,27.883,5.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,6.441,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.757,90.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.519,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,31.290,5.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.762,89.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,904.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.759,90.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.591,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,3.602,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,7.018,10.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.190,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,463.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.397,85.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.759,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,8.344,10.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.391,87.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,904.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.399,85.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.798,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,1.810,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,1.862,18.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.584,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.211,79.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.884,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,2.373,18.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.212,78.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,904.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.215,77.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.898,46.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.901,46.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.546,30.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.797,52.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 diff --git a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260821_051507.csv b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260821_051507.csv new file mode 100644 index 000000000..a0f4d7edc --- /dev/null +++ b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260821_051507.csv @@ -0,0 +1,44 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.961,93.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,14.279,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,3.003,91.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,933.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.919,94.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,448.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,12.936,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,12.962,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,112.723,2.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,12.972,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,462.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.489,92.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,7.098,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,1.510,91.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,892.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.477,93.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,6.388,54.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,6.389,54.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,27.728,5.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,455.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,6.442,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.763,89.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.526,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,31.275,5.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.810,84.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,889.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.759,90.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.181,54.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,3.194,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,7.017,10.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.191,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.397,85.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.769,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,8.338,10.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.416,81.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,889.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.401,84.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.596,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,1.604,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,2.004,17.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.587,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.212,79.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.890,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,2.455,17.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.215,77.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,892.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.216,77.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.809,51.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.816,51.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.578,29.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.798,52.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 diff --git a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left.png b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left.png index a1b577200..118cdc250 100755 Binary files a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left.png and b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left.png differ diff --git a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left_det_overhead.png b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left_det_overhead.png index 75c027a40..89f37c62e 100755 Binary files a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left_det_overhead.png and b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260819_052610.csv b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260819_052610.csv deleted file mode 100644 index 33825452c..000000000 --- a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260819_052610.csv +++ /dev/null @@ -1,131 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,26.666,1649.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,101.670,1125.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,96.032,1191.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,1746.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,50.706,1735.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,209.828,1090.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,202.150,1131.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,16.432,2677.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,False,59.456,1923.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,4762.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,True,57.824,1978.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,35.239,2496.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,4968.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,112.962,2025.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,114.972,1989.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,fwd,False,17.820,2468.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,False,66.848,1711.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,True,68.123,1679.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,36.537,2407.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,138.103,1656.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,138.804,1648.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,25.829,1703.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,57.760,1523.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,27.044,1626.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,104.626,1093.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,106.922,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,49.908,1762.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,198.655,1151.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,203.649,1123.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.126,1795.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,22.533,1269.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,22.475,1272.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,12.412,1772.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,48.551,1178.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,47.074,1215.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,4.149,2650.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,False,13.474,2122.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,True,14.161,2019.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,8.093,2717.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,26.296,2174.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,26.627,2147.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,fwd,False,4.489,2450.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,False,15.665,1825.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,True,15.681,1823.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,8.453,2601.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,31.253,1829.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,30.918,1849.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.693,1643.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,14.461,1521.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.225,1766.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,24.763,1154.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,25.663,1114.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,12.577,1748.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,48.063,1190.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,48.432,1181.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.705,1613.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,5.751,1243.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,5.651,1265.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.086,1781.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,11.519,1241.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,11.570,1235.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,1.125,2444.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.584,1994.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.580,1996.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.035,2701.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,6.342,2254.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,6.344,2253.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,fwd,False,1.209,2274.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,False,4.226,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,True,4.226,1692.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.194,2506.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,7.589,1883.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,7.588,1884.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.800,1528.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.249,1692.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.719,1599.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,6.451,1108.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,6.681,1070.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.008,1828.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,11.776,1214.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,12.039,1187.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.498,1381.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.656,1079.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.613,1108.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.814,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.927,1221.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.881,1240.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.327,2101.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.056,1692.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.058,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.562,2447.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.738,2056.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.712,2087.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.348,1974.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.277,1400.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.282,1394.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.589,2334.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.032,1759.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.045,1747.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.519,1326.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.864,1592.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.495,1389.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.890,945.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.959,912.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.813,1690.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,3.079,1161.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,3.163,1130.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.163,1053.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.520,860.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.511,875.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.231,1484.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.856,1044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.836,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.105,1630.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.348,1282.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.350,1277.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.161,2140.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.511,1748.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.512,1745.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.113,1515.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.435,1026.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.436,1026.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.169,2031.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.617,1448.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.617,1448.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.171,1004.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.243,1415.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.163,1053.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.619,722.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.643,695.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.232,1479.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.913,979.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.935,955.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260821_052530.csv b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260821_052530.csv new file mode 100644 index 000000000..4afcabe1c --- /dev/null +++ b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260821_052530.csv @@ -0,0 +1,141 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,26.622,1652.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,103.211,1108.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,101.447,1127.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,53.957,1630.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,214.117,1068.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,201.046,1138.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,16.896,2603.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,False,55.914,2045.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,True,54.646,2093.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,35.445,2482.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,3773.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,115.677,1977.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,116.704,1960.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,fwd,False,18.523,2374.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,False,66.646,1716.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,True,66.414,1722.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,35.448,2481.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,136.984,1670.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,136.828,1671.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,29.551,1488.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,59.184,1486.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,top_left,2,32768,32768,64,64,192,128,fwd,False,17.336,2537.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,35.881,2451.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,27.181,1618.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,104.689,1092.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,106.118,1078.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,48.663,1808.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,200.966,1138.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,204.050,1121.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.176,1780.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,23.461,1219.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,24.279,1178.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,13.032,1687.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,49.565,1154.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,48.865,1170.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,4.162,2642.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,False,13.607,2101.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,True,13.614,2100.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,8.095,2716.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,26.237,2179.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,26.330,2171.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,fwd,False,4.489,2449.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,False,16.210,1764.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,True,16.245,1760.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,8.715,2523.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,32.371,1766.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,32.853,1740.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.641,1656.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,15.300,1437.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,top_left,2,16384,16384,64,64,192,128,fwd,False,4.408,2495.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,8.291,2652.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.227,1766.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,24.289,1177.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,25.303,1130.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,11.912,1846.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,48.152,1187.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,48.183,1187.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.638,1678.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,5.737,1246.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,5.698,1254.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.017,1822.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,11.176,1279.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,11.317,1263.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,1.135,2422.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.586,1993.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.588,1992.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.112,2603.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,6.584,2171.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,6.472,2209.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,fwd,False,1.209,2274.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,False,4.233,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,True,4.227,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.126,2586.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,7.650,1869.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,7.721,1851.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.747,1574.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.250,1692.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,top_left,2,8192,8192,64,64,192,128,fwd,False,1.207,2277.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.114,2601.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.670,1646.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,6.463,1106.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,6.688,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.953,1862.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,11.421,1252.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,12.688,1127.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.498,1381.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.654,1081.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.583,1129.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.816,1685.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.888,1237.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.893,1235.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.328,2097.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.057,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.056,1692.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.564,2438.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.730,2066.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.687,2118.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.349,1971.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.278,1398.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.283,1393.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.593,2320.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,4239.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.032,1759.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.039,1752.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.515,1334.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.868,1583.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.373,1843.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.591,2327.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.495,1389.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.889,946.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.957,913.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.816,1685.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,3.086,1158.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,3.163,1130.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.162,1060.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.518,862.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.510,876.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.232,1484.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.861,1037.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.841,1062.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.106,1620.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.349,1279.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.349,1280.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.160,2144.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.511,1749.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.509,1757.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.112,1530.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.434,1029.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.435,1027.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.170,2019.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.617,1448.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.616,1449.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.170,1012.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.242,1418.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.137,1255.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.179,1917.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.162,1059.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.619,722.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.642,696.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.234,1466.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.912,980.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.935,955.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask.png b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask.png index 1e608c1d7..cfbd51c68 100755 Binary files a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask.png and b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask_det_overhead.png b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask_det_overhead.png index 823b204b9..e16c63a16 100755 Binary files a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left.png b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left.png index d66095745..b5fac14b9 100755 Binary files a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left.png and b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left_det_overhead.png b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left_det_overhead.png index 5f6437d81..6d12358e1 100755 Binary files a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left_det_overhead.png and b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260819_052447.csv b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260819_052447.csv deleted file mode 100644 index 3c7005eb1..000000000 --- a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260819_052447.csv +++ /dev/null @@ -1,131 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,20.703,2124.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2465.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,82.079,1393.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,887.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,86.623,1320.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,42.858,2052.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,187.010,1223.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,179.692,1273.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,12.637,3480.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,False,45.083,2537.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,True,45.379,2520.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,25.447,3457.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,91.473,2500.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,89.252,2562.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,fwd,False,14.126,3113.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,False,56.391,2028.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,True,57.028,2005.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,28.631,3072.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,111.575,2050.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,115.225,1985.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,23.464,1874.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,49.784,1767.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,23.050,1908.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,90.084,1269.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,95.219,1201.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,44.091,1995.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,175.537,1303.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,183.027,1250.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,5.290,2079.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,21.249,1345.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,20.429,1399.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,10.064,2185.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,42.850,1334.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,42.074,1359.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,3.201,3435.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,False,11.219,2548.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,True,11.174,2558.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,6.193,3551.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,21.518,2657.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,21.514,2658.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,fwd,False,3.584,3068.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,False,14.394,1986.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,True,14.122,2024.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,6.910,3182.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,27.157,2105.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,27.057,2113.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.009,1830.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,11.611,1894.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,5.645,1948.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,22.721,1258.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,23.974,1192.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,10.119,2173.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,43.601,1311.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,45.239,1264.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.410,1949.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,5.430,1316.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,5.216,1370.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.559,2148.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,10.381,1377.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,10.121,1412.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,0.847,3244.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.024,2364.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.023,2365.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,1.562,3521.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,5.407,2644.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,5.420,2637.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,fwd,False,0.946,2906.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.851,1856.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.854,1855.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,1.742,3156.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,6.787,2106.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,6.829,2093.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.592,1727.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.947,1866.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.516,1814.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,6.162,1160.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,6.488,1102.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.556,2151.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,10.971,1303.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,11.455,1248.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.408,1684.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.542,1159.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.495,1196.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.680,2022.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.812,1271.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.787,1282.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.240,2870.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,False,0.893,2001.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,True,0.893,2001.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.413,3324.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.452,2462.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.455,2457.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.265,2595.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.161,1540.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.160,1540.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.464,2965.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.825,1958.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.827,1956.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.443,1552.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.762,1803.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.434,1584.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.807,989.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.894,944.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.693,1983.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.979,1199.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,3.082,1159.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.132,1306.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.492,909.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.473,944.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.188,1827.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.830,1076.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.797,1121.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.076,2247.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.301,1485.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.301,1485.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.114,3024.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.434,2058.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.434,2058.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.084,2040.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.397,1126.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.397,1127.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.128,2684.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.549,1626.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.549,1626.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.139,1236.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.202,1700.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.141,1221.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.592,754.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.621,720.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.192,1793.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.881,1014.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.910,981.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260821_052626.csv b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260821_052626.csv new file mode 100644 index 000000000..b1aca42b0 --- /dev/null +++ b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260821_052626.csv @@ -0,0 +1,141 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,20.614,2134.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,85.446,1338.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,87.746,1303.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,43.000,2046.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,182.239,1255.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,175.118,1306.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,12.686,3467.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,False,44.378,2577.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,True,45.345,2522.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,4948.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,25.133,3500.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,88.073,2597.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,88.717,2578.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,fwd,False,14.265,3083.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,False,56.983,2007.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,True,56.441,2026.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,28.489,3088.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,111.469,2052.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,112.266,2037.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,23.649,1860.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,49.682,1770.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,top_left,2,32768,32768,64,64,192,128,fwd,False,14.925,2947.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,30.394,2894.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5006.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,22.155,1985.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,91.814,1245.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,96.344,1187.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,42.605,2065.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,179.184,1276.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,184.807,1237.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,5.326,2064.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,21.301,1342.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,20.639,1385.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,10.341,2126.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,42.988,1330.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,40.049,1428.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,3.236,3398.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,False,11.332,2523.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,True,11.212,2550.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,6.223,3534.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,21.711,2633.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,21.119,2707.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,fwd,False,3.614,3042.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,False,14.379,1988.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,True,14.301,1999.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,6.947,3165.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,27.527,2077.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,27.404,2086.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.069,1812.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,11.765,1869.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,top_left,2,16384,16384,64,64,192,128,fwd,False,3.905,2816.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,7.485,2938.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,5.677,1937.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,22.849,1251.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,24.046,1189.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,10.375,2119.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,43.284,1321.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,44.403,1288.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.427,1926.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,5.477,1305.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,5.276,1355.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.574,2136.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,10.588,1350.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,10.281,1390.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,0.853,3225.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.055,2340.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.051,2343.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,1.576,3489.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,5.449,2623.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,5.452,2622.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,fwd,False,0.951,2891.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.887,1839.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.889,1838.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,1.758,3126.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,6.823,2095.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,6.858,2084.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.594,1724.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.964,1855.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,top_left,2,8192,8192,64,64,192,128,fwd,False,1.085,2533.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,1.920,2863.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.522,1806.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,6.181,1156.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,6.499,1100.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.581,2130.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,10.993,1300.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,11.478,1245.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.409,1679.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.552,1151.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.505,1188.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.680,2021.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.846,1256.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.816,1269.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.240,2859.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,False,0.903,1980.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,True,0.901,1984.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.413,3325.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.465,2440.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.466,2438.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.267,2572.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.170,1527.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,1270.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.172,1525.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.466,2947.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.844,1938.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.845,1937.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.441,1560.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.759,1812.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.331,2078.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.524,2622.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.437,1572.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.815,985.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.899,941.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.701,1959.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.992,1194.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,3.093,1155.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.132,1300.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.496,901.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.478,935.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.188,1830.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.836,1068.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.805,1110.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.077,2225.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.303,1474.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.303,1473.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.114,3007.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.440,2029.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.440,2030.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.085,2017.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.400,1116.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.400,1118.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.129,2658.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.558,1602.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.555,1609.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.139,1234.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.202,1705.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.121,1420.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,mxfp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.152,2261.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.142,1213.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.597,749.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.624,716.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.193,1781.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.887,1008.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.913,978.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask.png b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask.png index 9f35fd735..2af406326 100755 Binary files a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask.png and b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask_det_overhead.png b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask_det_overhead.png index 217a7df02..552384010 100755 Binary files a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left.png b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left.png index 8c78f7b0d..a69be8d2b 100755 Binary files a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left.png and b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left_det_overhead.png b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left_det_overhead.png index feb639604..cd67bbbf3 100755 Binary files a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left_det_overhead.png and b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/h200/kimiK26_20260820_055407.csv b/benchmark/attention_training/results/kimiK26/h200/kimiK26_20260821_055845.csv similarity index 60% rename from benchmark/attention_training/results/kimiK26/h200/kimiK26_20260820_055407.csv rename to benchmark/attention_training/results/kimiK26/h200/kimiK26_20260821_055845.csv index 3e79569af..c2a52fcde 100644 --- a/benchmark/attention_training/results/kimiK26/h200/kimiK26_20260820_055407.csv +++ b/benchmark/attention_training/results/kimiK26/h200/kimiK26_20260821_055845.csv @@ -1,69 +1,69 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,72.560,606.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,228.076,501.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,140.790,625.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,719.265,318.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,70.828,621.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,143.006,615.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,64.174,685.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,236.371,484.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,266.157,430.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,135.465,649.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,465.516,491.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,486.763,470.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,18.073,608.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,55.997,511.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,70.613,405.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,34.382,640.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,180.028,318.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,181.448,315.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,17.653,623.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,35.272,623.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,15.807,696.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,58.848,486.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,67.350,424.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,33.481,657.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,115.579,495.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,121.680,470.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,4.317,637.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,14.151,505.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,17.613,406.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,8.251,666.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,43.967,325.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,44.483,321.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,4.483,613.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,8.722,630.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,3.855,713.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,14.240,502.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,17.337,412.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,7.845,701.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,27.961,511.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,31.221,458.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.150,598.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,3.709,482.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,4.576,391.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,2.077,662.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,11.260,317.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,10.919,327.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,1.165,590.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,2.237,614.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.038,663.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,3.697,483.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,4.823,371.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,1.983,693.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,6.756,529.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,8.181,437.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.328,524.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,72.281,608.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,225.389,507.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,142.630,617.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,717.516,319.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,70.233,626.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,141.946,620.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,64.944,677.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,233.684,489.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,265.231,431.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,136.761,643.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,465.086,492.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,486.852,470.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,18.136,606.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,56.681,504.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,70.792,404.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,35.310,623.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,178.100,321.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,180.594,317.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,17.541,627.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,35.259,624.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,15.755,698.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,56.606,505.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,67.418,424.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,33.876,649.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,116.742,490.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,122.874,465.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,4.318,637.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,13.564,527.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,17.404,411.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,8.020,685.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,44.030,325.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,44.320,323.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,4.465,616.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,8.791,625.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,3.827,718.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,13.677,523.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,17.231,415.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,7.717,712.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,28.249,506.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,31.383,455.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.152,597.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,3.703,483.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,4.582,390.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,2.058,668.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,11.339,315.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,10.970,326.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,1.166,590.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,2.258,609.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.037,663.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,3.691,484.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,4.838,369.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,1.964,700.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,6.718,532.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,8.236,434.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.329,522.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,1.116,401.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,1.309,341.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.551,623.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,3.041,294.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,2.871,311.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.325,528.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.586,587.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.310,555.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,1.107,404.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,1.449,308.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.534,643.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,1.311,341.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.545,630.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,3.039,294.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,2.883,310.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.327,526.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.585,588.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.309,556.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,1.109,403.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,1.443,310.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.528,651.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,1.898,471.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,2.316,386.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,2.324,384.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask.png b/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask.png index 170cb259d..f64471f6c 100755 Binary files a/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask.png and b/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask.png differ diff --git a/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask_det_overhead.png b/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask_det_overhead.png index 95db96d1f..e4004a548 100755 Binary files a/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left.png b/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left.png index 3c816d038..ba14b3f75 100755 Binary files a/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left.png and b/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left.png differ diff --git a/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left_det_overhead.png b/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left_det_overhead.png index 183467222..7a993bdcc 100755 Binary files a/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left_det_overhead.png and b/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260820_063243.csv b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260820_063243.csv deleted file mode 100644 index b8792d180..000000000 --- a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260820_063243.csv +++ /dev/null @@ -1,41 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,193.843,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,386.736,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,109.967,400.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,218.094,403.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,125.945,349.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,460.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,714.136,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,234.985,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,1442.505,159.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,48.880,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,96.892,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,27.239,404.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,54.304,405.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,31.502,349.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,460.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,181.067,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,58.466,376.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,363.040,157.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,12.207,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,23.920,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,6.702,410.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,13.002,423.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,7.978,345.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,46.557,154.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,14.172,388.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,91.964,155.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,3.205,214.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,6.105,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.871,367.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,3.337,412.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,2.150,320.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,12.368,144.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,456.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,3.641,377.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,23.546,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,459.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.891,193.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,1.579,218.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.623,276.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.969,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.667,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,3.458,129.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.987,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,6.221,144.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,455.000 diff --git a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260821_061139.csv b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260821_061139.csv new file mode 100644 index 000000000..7775ec8b7 --- /dev/null +++ b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260821_061139.csv @@ -0,0 +1,41 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,193.519,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,463.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,386.250,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,110.091,400.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,217.952,404.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,448.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,126.413,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,456.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,714.269,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,234.710,375.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,1441.591,159.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,49.047,224.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,97.762,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,27.618,398.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,54.665,402.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,31.621,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,181.124,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,58.736,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,455.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,362.883,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,12.205,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,23.903,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,458.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,6.755,407.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,13.194,417.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,448.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,7.972,345.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,46.561,154.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,14.175,388.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,91.960,155.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,3.199,215.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,6.103,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.873,367.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,3.345,411.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,2.301,299.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,12.369,144.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,3.674,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,23.583,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.947,181.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,1.690,203.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.630,273.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.980,351.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.675,255.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,3.686,121.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,1.051,327.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,6.212,144.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 diff --git a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_no_mask.png b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_no_mask.png index 43c27dc74..89e07d42d 100755 Binary files a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_no_mask.png and b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_no_mask.png differ diff --git a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_top_left.png b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_top_left.png index 7ba7cc7db..fd6d7416e 100755 Binary files a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_top_left.png and b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_top_left.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260819_052144.csv b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260819_052144.csv deleted file mode 100644 index a5e70a854..000000000 --- a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260819_052144.csv +++ /dev/null @@ -1,121 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,37.207,1773.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,160.134,1071.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,153.937,1114.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,81.276,1623.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,307.124,1117.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,300.337,1142.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,25.680,2569.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,False,88.571,1937.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,True,88.423,1940.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,50.614,2607.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,169.029,2030.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,173.050,1982.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,fwd,False,27.080,2436.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,False,101.852,1684.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,True,102.592,1672.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,54.897,2403.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,203.193,1688.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,202.912,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,39.439,1673.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,154.031,1114.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,156.070,1099.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,79.776,1654.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,288.242,1190.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,299.464,1146.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,9.460,1743.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,31.963,1342.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,34.377,1247.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,19.022,1734.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,73.686,1164.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,75.515,1136.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,6.277,2627.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,False,20.869,2055.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,True,20.726,2069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,12.625,2613.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,38.113,2250.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,40.744,2105.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,fwd,False,6.837,2412.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,False,24.864,1725.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,True,24.019,1785.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,13.296,2481.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,45.680,1877.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,45.848,1871.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,9.373,1760.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,37.467,1145.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,38.421,1116.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,17.558,1879.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,73.865,1161.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,74.679,1148.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.554,1615.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,8.683,1235.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,8.579,1250.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,4.523,1823.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,18.369,1167.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,16.769,1279.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.668,2472.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,False,5.380,1993.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,True,5.381,1993.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.086,2672.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,9.611,2231.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,9.575,2239.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.793,2300.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,False,6.349,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,True,6.350,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.286,2509.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,12.069,1776.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,11.497,1865.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.576,1601.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,9.645,1112.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,9.968,1076.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,4.474,1843.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,17.873,1200.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,18.221,1177.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.737,1398.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.479,1081.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.399,1118.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.219,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.332,1237.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.375,1225.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.483,2136.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.566,1712.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.566,1711.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.830,2484.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.547,2104.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.547,2104.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.513,2009.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.897,1413.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.898,1413.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.877,2350.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,3.062,1750.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,3.063,1750.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.735,1403.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.803,956.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.901,924.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.227,1680.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.591,1167.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.763,1125.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.236,1092.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.768,873.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.744,901.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.348,1480.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.275,1051.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.220,1099.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.154,1675.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.511,1312.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.510,1313.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.241,2135.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.745,1799.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.745,1798.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.164,1572.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.635,1055.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.637,1053.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.254,2030.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.902,1486.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.903,1483.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.237,1087.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.906,740.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.940,713.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.356,1446.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.352,991.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.388,965.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260821_052008.csv b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260821_052008.csv new file mode 100644 index 000000000..fd4004292 --- /dev/null +++ b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260821_052008.csv @@ -0,0 +1,121 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,38.751,1702.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,158.205,1084.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,155.186,1105.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,79.233,1665.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,310.156,1106.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,303.029,1132.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,26.307,2508.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,False,91.882,1867.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,True,88.529,1938.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,51.391,2567.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,170.096,2017.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,171.423,2001.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,fwd,False,27.314,2415.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,False,99.467,1724.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,True,104.622,1640.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,53.356,2473.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,202.844,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,202.958,1690.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,38.509,1713.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,154.260,1112.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,156.062,1099.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,81.851,1612.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,289.271,1186.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,295.888,1159.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,9.340,1766.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,35.903,1194.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,33.899,1265.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,17.505,1884.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,77.833,1102.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,71.017,1208.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,6.510,2533.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,False,20.453,2097.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,True,21.560,1989.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,12.729,2591.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,38.412,2233.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,39.153,2190.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,fwd,False,6.943,2376.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,False,23.973,1789.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,True,25.220,1700.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,12.832,2571.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,48.125,1782.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,46.817,1832.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,9.470,1742.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,37.932,1131.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,37.706,1137.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,18.098,1823.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,73.363,1169.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,74.676,1148.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.618,1575.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,8.729,1228.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,8.679,1235.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,4.595,1795.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,17.129,1252.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,17.350,1236.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.730,2383.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,False,5.408,1983.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,True,5.406,1983.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.136,2629.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,9.556,2244.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,9.703,2210.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.828,2255.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,False,6.399,1676.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,True,6.355,1687.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.276,2517.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,11.693,1834.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,11.499,1865.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.553,1615.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,9.635,1113.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,9.991,1073.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,4.685,1760.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,17.689,1212.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,17.863,1200.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.749,1377.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.469,1086.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.419,1108.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.226,1681.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.431,1210.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.400,1218.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.492,2095.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.573,1704.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.575,1702.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.842,2449.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.576,2081.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.580,2077.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.520,1982.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.906,1407.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.907,1406.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.876,2353.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,3.086,1737.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,3.086,1737.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.746,1381.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.799,958.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.900,924.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.251,1648.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.594,1167.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.759,1126.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.240,1076.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.778,862.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.753,890.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.353,1461.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.270,1055.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.233,1087.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.157,1641.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.512,1310.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.512,1309.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.245,2105.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.749,1789.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.749,1790.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.167,1548.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.638,1050.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.639,1048.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.257,2007.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.908,1477.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.908,1476.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,5135.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.240,1073.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.906,740.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.942,712.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.360,1433.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.348,994.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.384,968.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600,2568.000 diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask.png b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask.png index 82eccc889..409d5e57c 100755 Binary files a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask.png and b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask_det_overhead.png b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask_det_overhead.png index eef6e73cc..3eff403a5 100755 Binary files a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left.png b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left.png index 7b17b8618..42424ef8c 100755 Binary files a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left.png and b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left_det_overhead.png b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left_det_overhead.png index 9b6c6113a..19c66b336 100755 Binary files a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260819_052012.csv b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260819_052012.csv deleted file mode 100644 index 84214fe1b..000000000 --- a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260819_052012.csv +++ /dev/null @@ -1,121 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,32.690,2018.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,134.621,1274.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,128.157,1338.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,63.940,2064.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,272.241,1260.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,264.532,1297.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,19.088,3456.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,False,69.538,2467.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,True,68.788,2494.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,37.854,3486.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,138.369,2479.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,138.286,2481.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,fwd,False,21.512,3067.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,False,85.655,2003.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,True,85.465,2007.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,42.132,3132.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,171.514,2000.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,171.442,2001.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,34.358,1920.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,137.701,1246.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,143.286,1197.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,64.737,2038.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,267.039,1285.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,276.013,1243.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,8.118,2032.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,31.984,1341.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,30.726,1396.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,15.149,2177.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,65.819,1303.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,60.311,1422.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,4.847,3403.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,False,17.075,2512.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,True,17.225,2490.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,9.366,3522.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,32.348,2651.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,32.741,2619.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,fwd,False,5.429,3038.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,False,21.252,2018.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,True,21.487,1996.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,10.491,3144.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,40.387,2123.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,40.713,2106.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,8.542,1931.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,34.341,1249.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,35.820,1197.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,15.840,2082.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,65.511,1309.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,68.706,1248.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.150,1918.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,8.201,1307.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,7.901,1357.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.891,2119.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,15.939,1345.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,15.411,1391.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.281,3219.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,False,4.556,2353.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,True,4.558,2352.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,2.381,3464.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,8.228,2606.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,8.213,2611.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.432,2880.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,False,5.801,1848.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,True,5.803,1848.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,2.661,3099.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,10.332,2075.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,10.207,2101.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.286,1804.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,9.192,1166.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,9.656,1110.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.933,2097.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,16.443,1304.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,17.071,1256.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.615,1678.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.325,1153.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.240,1197.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.032,1999.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.227,1268.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.174,1284.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.357,2886.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.337,2005.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.337,2005.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.623,3309.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.198,2439.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.198,2439.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.398,2593.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.727,1552.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.730,1549.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.698,2952.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.756,1945.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.758,1943.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.652,1580.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.675,1002.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.803,956.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.051,1962.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.458,1202.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.623,1159.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.194,1331.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.730,919.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.699,959.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.287,1795.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.232,1087.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.163,1152.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.112,2310.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.443,1514.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.443,1513.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.174,2966.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.641,2089.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.641,2091.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.123,2092.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.584,1148.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.584,1149.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.195,2645.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.812,1650.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.811,1652.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.206,1251.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.866,774.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.907,739.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.299,1725.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.308,1024.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.350,993.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260821_052051.csv b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260821_052051.csv new file mode 100644 index 000000000..40c70c94a --- /dev/null +++ b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260821_052051.csv @@ -0,0 +1,121 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,31.925,2066.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,134.749,1273.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,129.965,1320.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,64.495,2046.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,274.467,1250.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,264.821,1295.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,19.198,3436.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,False,68.221,2514.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,True,69.588,2465.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,38.185,3455.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,138.797,2472.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,135.874,2525.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,fwd,False,21.910,3011.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,False,86.262,1988.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,True,85.821,1999.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,42.850,3079.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,173.006,1983.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,170.105,2017.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,34.414,1917.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,138.329,1240.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,145.447,1179.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,65.536,2013.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,269.946,1271.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,278.066,1234.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,8.082,2041.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,32.237,1330.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,31.278,1371.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,15.522,2125.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,65.422,1311.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,62.474,1373.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,4.880,3380.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,False,17.198,2494.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,True,17.201,2493.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,9.455,3489.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,33.239,2580.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,32.798,2615.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,fwd,False,5.444,3030.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,False,22.105,1940.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,True,21.871,1961.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,10.532,3132.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,41.259,2079.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,40.369,2124.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,8.570,1925.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,34.618,1239.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,36.102,1188.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,15.429,2138.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,67.517,1270.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,69.853,1228.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.159,1910.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,8.226,1303.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,7.987,1342.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.893,2118.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,16.145,1328.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,15.598,1375.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.284,3212.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,False,4.593,2334.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,True,4.594,2334.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,2.381,3464.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,8.207,2612.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,8.245,2600.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.434,2875.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,False,5.838,1836.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,True,5.835,1837.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,2.665,3094.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,10.318,2078.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,10.346,2072.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.290,1801.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,9.251,1159.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,9.703,1105.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.962,2081.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,16.629,1289.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,17.184,1248.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.612,1685.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.331,1150.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.244,1195.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.015,2031.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.255,1260.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.202,1276.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.359,2873.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.350,1985.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.349,1987.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.619,3332.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.204,2432.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.204,2432.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.403,2557.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.747,1534.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.745,1536.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.697,2957.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.774,1932.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.777,1930.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.653,1578.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.693,996.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.820,951.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.059,1947.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.490,1194.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.654,1152.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.195,1323.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.736,910.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.706,950.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.284,1816.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.246,1075.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.173,1142.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.114,2268.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.450,1488.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.450,1489.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.174,2966.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.647,2070.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.646,2073.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.125,2055.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.592,1133.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.592,1133.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.196,2629.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.823,1628.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.822,1631.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,5155.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.209,1235.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.873,768.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.914,734.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.299,1723.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.321,1015.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.359,986.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600,2578.000 diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask.png b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask.png index 828024a30..2ab177768 100755 Binary files a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask.png and b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask_det_overhead.png b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask_det_overhead.png index 38cd5e858..bf7a7018f 100755 Binary files a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left.png b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left.png index 1d281574b..4dbeaf1f1 100755 Binary files a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left.png and b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left_det_overhead.png b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left_det_overhead.png index dc88d376c..3c4c6d7e1 100755 Binary files a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260820_051537.csv b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260821_051609.csv similarity index 61% rename from benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260820_051537.csv rename to benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260821_051609.csv index f237057d1..8e4a5004b 100644 --- a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260820_051537.csv +++ b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260821_051609.csv @@ -1,67 +1,67 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,109.732,601.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,344.187,498.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,212.208,622.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,1080.527,317.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,107.017,616.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,213.524,618.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,98.116,672.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,350.509,489.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,398.790,430.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,204.558,645.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,699.128,491.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,730.406,470.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,27.171,607.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,85.970,499.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,52.325,630.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,269.545,318.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,26.455,623.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,52.528,628.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,23.502,702.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,88.478,485.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,103.513,414.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,50.467,654.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,176.151,487.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,184.547,465.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,6.463,638.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,22.002,487.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,26.811,400.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,13.100,629.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,67.010,320.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,67.287,319.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,6.958,593.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,13.105,629.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,5.787,713.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,21.664,495.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,26.204,409.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,12.250,673.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,42.381,506.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,47.162,455.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,1.703,605.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,5.506,487.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,6.780,395.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,3.107,664.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,16.827,319.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,16.659,322.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,110.122,599.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,336.447,510.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,212.427,621.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,1072.344,320.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,106.595,619.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,212.206,622.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,100.587,656.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,350.726,489.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,394.141,435.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,205.057,643.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,694.861,494.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,725.632,473.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,27.025,610.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,85.235,503.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,52.058,634.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,267.693,320.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,26.198,630.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,52.533,628.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,23.697,696.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,87.893,488.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,103.125,416.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,50.166,658.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,173.597,494.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,184.915,464.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,6.469,637.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,21.641,495.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,26.426,406.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,12.367,667.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,65.824,326.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,65.987,325.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,6.762,610.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,13.036,633.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,5.760,716.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,21.297,503.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,26.238,409.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,12.171,678.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,42.423,505.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,47.209,454.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,1.707,604.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,5.502,487.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,6.789,395.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,3.065,673.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,16.909,317.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,16.264,330.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,1.760,586.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,3.299,625.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,1.550,665.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,3.320,621.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,1.553,664.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,5.507,487.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,7.215,372.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,2.977,692.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,10.149,528.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,12.304,436.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.476,541.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,1.637,410.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,1.915,350.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.814,633.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,4.508,297.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,4.268,314.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,7.220,371.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,2.943,700.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,10.055,533.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,12.406,432.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.477,540.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,1.638,409.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,1.920,349.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.809,637.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,4.514,297.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,4.286,313.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.487,529.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.867,595.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.869,593.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.456,566.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,1.629,411.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,2.148,312.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.790,652.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,2.805,478.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,3.488,384.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,1.635,410.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,2.156,311.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.787,655.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,2.828,474.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,3.506,382.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask.png b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask.png index 2678d733b..61c0fbe0f 100755 Binary files a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask.png and b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask.png differ diff --git a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask_det_overhead.png b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask_det_overhead.png index f6ac489e8..94080c64f 100755 Binary files a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left.png b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left.png index 4d8899078..60666c7a2 100755 Binary files a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left.png and b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left.png differ diff --git a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left_det_overhead.png b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left_det_overhead.png index cc6b604ef..ce46dc659 100755 Binary files a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260820_061939.csv b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260820_061939.csv deleted file mode 100644 index b8a16437d..000000000 --- a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260820_061939.csv +++ /dev/null @@ -1,31 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,291.651,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,585.249,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,189.314,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,1071.068,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,354.526,372.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,2166.153,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,74.794,221.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,147.825,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,47.927,344.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,270.956,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,88.269,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,543.826,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,18.281,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,36.210,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,11.844,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,69.657,154.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,21.429,385.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,137.848,156.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,4.738,218.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,9.137,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,3.156,327.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,18.438,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,5.430,380.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,35.303,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,1.292,199.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,2.352,219.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.957,270.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,5.129,131.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,1.505,342.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,9.273,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 diff --git a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260821_055953.csv b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260821_055953.csv new file mode 100644 index 000000000..8b6b360d9 --- /dev/null +++ b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260821_055953.csv @@ -0,0 +1,31 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,290.989,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,583.473,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,462.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,189.870,347.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,459.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,1071.125,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,352.873,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,2161.326,159.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,74.586,221.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,147.682,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,47.889,344.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,271.019,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,88.091,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,543.821,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,18.236,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,35.980,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,11.851,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,69.668,154.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,21.688,380.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,449.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,137.786,156.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,4.726,218.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,9.128,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,3.155,327.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,18.438,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,5.433,379.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,35.254,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,462.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,1.374,188.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,2.359,218.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.967,267.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,5.450,123.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,1.542,334.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,9.267,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 diff --git a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_no_mask.png b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_no_mask.png index 57e8007c8..008293911 100755 Binary files a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_no_mask.png and b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_no_mask.png differ diff --git a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_top_left.png b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_top_left.png index d2ca95370..48bd70ae3 100755 Binary files a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_top_left.png and b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_top_left.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260819_052235.csv b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260819_052235.csv deleted file mode 100644 index 3abd5ffb5..000000000 --- a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260819_052235.csv +++ /dev/null @@ -1,141 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,22.112,1591.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,69.824,1260.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,85.096,1034.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,45.807,1536.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,140.325,1254.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,167.870,1048.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,16.344,2153.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,53.589,1641.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,53.029,1659.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,33.166,2122.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,102.344,1719.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,101.599,1732.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,17.077,2060.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,False,64.253,1369.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,True,61.526,1430.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,33.630,2092.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,121.763,1445.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,125.481,1402.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,24.097,1460.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,48.135,1462.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,21.830,1612.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,41.165,1709.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,22.906,1536.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,67.684,1300.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,72.269,1217.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,59.415,1184.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,127.272,1382.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,135.115,1302.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.587,1575.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,16.041,1371.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,20.035,1098.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,11.056,1591.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,30.562,1439.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,38.890,1131.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.095,2148.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,13.654,1611.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,13.468,1633.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,7.959,2210.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,23.956,1836.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,24.111,1824.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.341,2026.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,False,15.641,1406.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,True,15.560,1413.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.221,2140.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,29.887,1472.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,29.609,1485.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.697,1544.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,11.679,1506.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,5.755,1529.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,10.335,1702.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.428,1620.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,16.411,1340.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,16.481,1334.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,14.203,1239.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,29.723,1480.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,31.213,1409.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.450,1516.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,4.298,1279.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.916,1118.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2287.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.640,1666.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,8.118,1354.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,9.380,1172.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.076,2044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.807,1444.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.805,1445.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.013,2184.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.196,1774.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,6.181,1779.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.137,1935.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,False,4.359,1261.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,True,4.369,1259.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.076,2118.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.476,1471.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.540,1458.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.578,1394.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.769,1589.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.597,1377.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.628,1674.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.467,1499.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,4.206,1307.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.419,1244.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,3.506,1254.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.437,1478.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.909,1390.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.420,1310.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.228,1119.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.356,1014.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.724,1519.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.164,1270.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.397,1147.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.308,1784.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.179,1166.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.178,1167.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.541,2033.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.746,1574.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.747,1573.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.325,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.343,1024.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.347,1021.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.563,1951.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.071,1327.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.066,1331.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.456,1206.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.755,1457.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.496,1108.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.710,1548.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.415,1325.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.266,1086.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.447,950.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.923,1191.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.036,1350.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.225,1235.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.133,1030.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.397,866.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.441,779.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.198,1388.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.632,1087.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.699,983.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.098,1400.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.418,822.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.419,821.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.151,1824.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.565,1215.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.565,1215.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.104,1323.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.470,731.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.471,730.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.159,1734.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.651,1056.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.650,1057.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.155,887.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.210,1306.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.183,752.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.199,1385.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.134,1025.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.423,813.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.547,629.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.255,1076.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.617,1114.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.724,949.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260821_052057.csv b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260821_052057.csv new file mode 100644 index 000000000..31bf63073 --- /dev/null +++ b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260821_052057.csv @@ -0,0 +1,141 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,23.048,1527.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,70.513,1247.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,83.834,1049.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,45.100,1560.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,145.938,1205.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,164.071,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,16.576,2123.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,53.226,1653.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,51.625,1704.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,32.864,2141.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,100.064,1758.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,100.810,1745.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,17.274,2037.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,False,63.421,1387.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,True,62.383,1410.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,33.850,2079.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,125.676,1400.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,124.938,1408.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,23.658,1487.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,48.231,1459.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,21.818,1613.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,41.217,1707.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,22.536,1561.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,66.597,1321.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,68.820,1278.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,58.568,1201.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,122.064,1441.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,133.802,1315.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.212,1688.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,16.890,1302.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,18.136,1213.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,10.629,1655.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,33.109,1328.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,39.228,1121.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.085,2153.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,13.482,1631.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,13.481,1631.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,7.991,2201.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,24.344,1807.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,23.796,1848.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.332,2030.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,False,15.938,1380.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,True,15.610,1409.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.241,2135.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,29.944,1469.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,29.975,1467.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.613,1567.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,10.870,1618.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,5.743,1532.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,10.377,1695.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.253,1675.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,15.775,1394.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,16.357,1344.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,14.021,1255.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,29.096,1512.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,32.379,1358.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.425,1543.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,4.075,1349.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.647,1183.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2269.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.500,1759.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.642,1439.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,9.249,1189.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.076,2043.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.790,1451.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.792,1450.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.999,2200.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.181,1779.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,6.193,1775.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.140,1929.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,False,4.320,1273.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,True,4.321,1272.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.067,2128.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.397,1486.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.387,1488.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.550,1419.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.670,1647.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.593,1381.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.624,1676.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.395,1577.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,3.992,1377.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.268,1288.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,3.494,1259.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.169,1534.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.647,1438.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.399,1379.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.182,1163.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.318,1043.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.683,1609.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.017,1363.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.310,1190.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.302,1820.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.173,1172.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.171,1174.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.534,2058.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.737,1582.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.738,1581.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.319,1722.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.330,1033.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.332,1032.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.553,1988.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.048,1342.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.040,1347.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.445,1235.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.720,1526.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.494,1113.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.708,1552.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.400,1373.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.190,1156.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.450,948.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.908,1211.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.945,1413.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.185,1258.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.126,1090.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.385,894.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.426,807.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.191,1441.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.606,1135.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.676,1017.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.095,1446.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.416,826.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.415,828.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.146,1883.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.561,1226.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.561,1225.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.100,1370.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.466,738.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.466,738.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.153,1799.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.645,1066.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.646,1064.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.150,914.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.202,1360.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.184,748.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.197,1394.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,5135.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.128,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.411,837.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.548,627.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.247,1111.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.596,1153.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.724,949.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask.png b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask.png index 917c012f5..1e59ae3fa 100755 Binary files a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask.png and b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask_det_overhead.png b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask_det_overhead.png index ab0211e42..a359f8cb6 100755 Binary files a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask_det_overhead.png and b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left.png b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left.png index 06e33e3a8..864484460 100755 Binary files a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left.png and b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left_det_overhead.png b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left_det_overhead.png index 4aba170a3..7ad560837 100755 Binary files a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left_det_overhead.png and b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260819_052115.csv b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260819_052115.csv deleted file mode 100644 index 125e18813..000000000 --- a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260819_052115.csv +++ /dev/null @@ -1,141 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,17.480,2013.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,57.269,1536.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,65.595,1341.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,34.799,2022.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,121.379,1449.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,131.096,1342.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,11.293,3116.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,42.105,2089.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,41.894,2100.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,22.113,3182.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,79.534,2212.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,79.503,2213.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,12.176,2890.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,False,55.353,1589.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,True,53.731,1637.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,24.495,2873.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,105.818,1662.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,104.817,1678.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,18.877,1864.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,36.859,1909.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,17.294,2035.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,27.729,2538.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,17.285,2036.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,56.186,1566.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,58.777,1497.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,34.951,2013.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,115.771,1520.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,124.400,1414.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.413,1993.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,14.640,1502.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,16.659,1320.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.195,2147.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2194.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,28.735,1531.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,32.033,1373.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,2.858,3078.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,10.791,2038.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,10.767,2042.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,5.503,3197.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,20.529,2142.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,19.762,2225.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,3.094,2843.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,False,14.044,1566.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,True,14.029,1568.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,4724.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,5.922,2971.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,25.754,1708.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,25.931,1696.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.700,1872.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.977,1960.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2521.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.636,1897.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,7.002,2512.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.405,1997.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,13.786,1595.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,14.842,1482.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.075,2179.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,27.611,1593.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,29.476,1492.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.156,1902.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,3.821,1439.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.243,1296.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.056,2140.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.113,1546.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.915,1389.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,0.757,2904.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.093,1778.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.091,1779.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.389,3167.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,5.194,2117.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,5.192,2118.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,0.822,2677.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.915,1405.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.914,1405.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.500,2932.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.724,1635.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,6.725,1635.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.303,1687.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.263,1944.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.337,1645.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.801,2441.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.163,1891.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,3.740,1470.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.148,1326.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.036,2160.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.904,1593.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.387,1488.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.319,1726.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.101,1249.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.210,1136.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.542,2030.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.918,1433.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.190,1255.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.216,2549.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,0.986,1394.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,0.987,1394.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.370,2971.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.492,1842.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.491,1844.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.234,2345.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.204,1142.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.204,1142.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.400,2750.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.878,1464.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.877,1464.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.366,1504.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.586,1876.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.438,1254.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.497,2211.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.332,1656.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.126,1220.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.412,974.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.541,2032.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.891,1454.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.155,1275.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.096,1427.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.356,966.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.387,889.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.147,1871.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.565,1216.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.632,1088.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.069,1985.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.364,943.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.365,942.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.101,2719.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.494,1392.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.495,1388.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.075,1823.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.424,811.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.424,811.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.111,2473.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.595,1154.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.595,1156.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.123,1122.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.158,1743.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.173,793.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.145,1892.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.106,1292.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.393,875.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.532,646.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.145,1898.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.571,1203.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.709,969.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260821_052150.csv b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260821_052150.csv new file mode 100644 index 000000000..f9f03b493 --- /dev/null +++ b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260821_052150.csv @@ -0,0 +1,141 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,17.614,1998.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,58.574,1502.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,72.098,1220.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,35.782,1967.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,122.454,1437.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,146.139,1204.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,11.499,3060.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,41.908,2099.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,41.856,2102.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,23.013,3058.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,80.954,2173.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,80.955,2173.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,12.395,2839.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,False,54.835,1604.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,True,53.530,1643.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,24.988,2816.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,108.642,1619.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,110.163,1597.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,19.749,1782.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,39.272,1792.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,17.329,2030.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,27.959,2517.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,17.294,2035.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,55.967,1572.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,59.314,1483.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,34.883,2017.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,119.063,1478.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,128.523,1369.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.440,1981.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,15.037,1463.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,16.660,1320.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.308,2117.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,29.545,1489.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2474.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,33.049,1331.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,2.883,3051.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,4911.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,10.831,2030.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,10.888,2020.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,5.543,3174.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,19.921,2208.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,20.092,2189.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,3.112,2826.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,False,14.096,1560.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,True,14.061,1564.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,6.017,2924.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,4948.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,26.763,1643.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,25.979,1693.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.884,1801.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,9.000,1955.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.654,1890.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,7.035,2501.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.413,1993.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,13.636,1613.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,14.802,1486.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.170,2153.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,27.469,1601.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,28.478,1544.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.162,1893.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,3.829,1436.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.261,1290.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.055,2140.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.207,1526.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,8.094,1358.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,0.768,2864.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.112,1767.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.110,1768.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.405,3130.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,5.223,2105.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,5.226,2104.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,0.832,2642.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.936,1397.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.935,1397.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.515,2903.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.764,1625.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,6.756,1627.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.338,1644.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.267,1940.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.347,1633.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.819,2418.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.162,1893.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,3.750,1466.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.175,1317.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.041,2155.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.932,1586.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.424,1481.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.317,1737.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.098,1252.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.211,1135.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.542,2028.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.925,1428.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.200,1250.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.219,2507.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,0.994,1383.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,0.993,1384.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.376,2925.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.505,1827.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.504,1828.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.238,2310.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.213,1134.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.213,1133.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.406,2707.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.892,1453.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.893,1452.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.378,1453.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.587,1872.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.443,1241.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.505,2179.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.334,1646.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.136,1210.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.421,968.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.540,2036.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.902,1445.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.171,1266.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.096,1426.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.359,958.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.390,881.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.147,1865.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.568,1210.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.640,1074.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.070,1958.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.366,938.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.367,937.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.103,2671.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.498,1379.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.499,1377.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.077,1796.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.428,803.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.428,802.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.112,2444.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.600,1146.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.600,1146.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.126,1088.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.159,1724.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.176,783.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.148,1859.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,5155.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.107,1280.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.397,867.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.536,642.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.144,1904.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.579,1187.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.715,962.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask.png b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask.png index e489063f2..796fbe84c 100755 Binary files a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask.png and b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask_det_overhead.png b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask_det_overhead.png index ea8d4ec8f..afc29218a 100755 Binary files a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask_det_overhead.png and b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left.png b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left.png index 3b04cfa2f..52a7d825c 100755 Binary files a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left.png and b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left_det_overhead.png b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left_det_overhead.png index e6166003e..7d496f75e 100755 Binary files a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left_det_overhead.png and b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/h200/llama3.1_20260820_060811.csv b/benchmark/attention_training/results/llama3.1/h200/llama3.1_20260821_061315.csv similarity index 61% rename from benchmark/attention_training/results/llama3.1/h200/llama3.1_20260820_060811.csv rename to benchmark/attention_training/results/llama3.1/h200/llama3.1_20260821_061315.csv index 521ef6f89..1ccafb773 100644 --- a/benchmark/attention_training/results/llama3.1/h200/llama3.1_20260820_060811.csv +++ b/benchmark/attention_training/results/llama3.1/h200/llama3.1_20260821_061315.csv @@ -1,89 +1,89 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,58.211,604.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,168.570,522.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,114.635,614.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,326.963,538.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,38.247,920.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,242.446,363.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,242.448,363.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,75.334,934.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,246.345,714.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,245.270,717.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,55.761,631.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,147.872,595.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,153.973,571.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,105.864,665.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,288.655,609.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,294.351,598.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,13.882,634.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,41.608,529.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,48.908,450.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,27.890,631.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,82.453,533.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,94.072,468.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,9.014,976.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,61.765,356.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,61.758,356.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,18.268,963.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,62.521,703.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,61.944,710.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,13.506,651.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,36.038,610.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,38.775,567.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,25.637,686.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,70.546,623.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,72.993,603.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,3.433,641.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,10.278,535.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,11.953,460.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,6.497,677.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,19.451,565.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,22.869,481.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,2.318,949.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,16.053,343.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,16.057,342.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,4.399,1000.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,15.484,710.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,15.596,705.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,3.356,655.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,8.622,638.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,9.607,572.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,6.077,724.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,17.335,634.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,18.127,607.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.908,606.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,2.752,500.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,3.145,437.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.641,670.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.688,586.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,5.568,494.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.634,867.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,4.287,321.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,4.297,320.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.125,978.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.141,664.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,4.143,663.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.907,607.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,2.386,576.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,2.777,495.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.551,709.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.205,654.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,4.556,603.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,57.849,608.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,166.855,527.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,114.012,617.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,327.512,537.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,38.301,919.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,242.491,363.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,242.430,363.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,74.860,940.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,244.934,718.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,245.203,717.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,55.740,631.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,146.259,601.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,153.851,572.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,105.313,668.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,282.983,622.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,290.471,606.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,14.038,627.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,40.955,537.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,48.592,453.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,28.109,626.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,81.792,538.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,93.608,470.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,9.128,964.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,61.750,356.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,61.747,356.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,17.948,980.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,61.818,711.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,61.638,714.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,13.403,656.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,36.223,607.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,38.768,567.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,25.732,684.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,71.173,618.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,73.493,598.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,3.376,651.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,10.331,532.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,12.080,455.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,6.310,697.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,19.471,565.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,22.775,483.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,2.290,961.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,16.048,343.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,16.049,343.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,4.294,1024.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,15.602,705.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,15.613,704.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,3.299,667.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,8.612,638.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,9.720,566.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,5.861,750.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,17.416,631.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,17.818,617.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.899,612.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,2.747,501.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,3.142,438.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.604,686.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.733,581.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,5.586,492.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.633,869.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,4.289,321.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,4.287,321.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.122,980.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.144,663.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,4.144,663.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.897,613.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,2.385,576.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,2.785,494.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.507,730.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.217,652.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,4.549,604.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.260,529.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.838,410.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.917,375.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.430,640.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.918,374.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.420,655.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.304,527.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.482,464.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.487,462.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.193,713.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,1.214,283.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,1.211,284.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.307,895.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.159,593.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.162,592.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,1.213,283.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,1.214,283.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.306,898.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.162,591.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.164,590.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,2141.000 llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.268,513.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.723,475.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.901,382.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.422,651.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.175,585.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.350,509.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.722,476.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.904,380.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.415,663.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.171,587.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.348,510.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask.png b/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask.png index 0bbf61ef7..d208cf68e 100755 Binary files a/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask.png and b/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask.png differ diff --git a/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask_det_overhead.png b/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask_det_overhead.png index 23f703063..5e001a33b 100755 Binary files a/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask_det_overhead.png and b/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left.png b/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left.png index 754fc5529..d2ea17624 100755 Binary files a/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left.png and b/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left.png differ diff --git a/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left_det_overhead.png b/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left_det_overhead.png index cf782ddab..01bb81930 100755 Binary files a/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left_det_overhead.png and b/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260820_061143.csv b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260820_061143.csv deleted file mode 100644 index c40043b97..000000000 --- a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260820_061143.csv +++ /dev/null @@ -1,87 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,91.737,384.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,330.979,266.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,183.695,383.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,598.644,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,63.509,554.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,124.009,567.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,88.041,400.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,255.876,344.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,359.122,245.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,176.917,398.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,498.736,353.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,697.586,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,106.010,332.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,301.045,292.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,189.432,371.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,618.479,284.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,22.651,388.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,83.633,263.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,45.578,386.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,150.977,291.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,15.378,572.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,30.278,581.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,21.283,413.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,65.320,337.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,94.202,233.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,43.143,408.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,125.193,351.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,176.513,249.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,25.852,340.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,75.182,293.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,46.794,376.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,153.307,287.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,5.635,390.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,21.811,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,20.788,264.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,10.788,408.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,37.658,292.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,42.373,259.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,4.011,548.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,7.390,595.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,5.381,409.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,16.869,326.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,25.059,219.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,10.374,424.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,31.833,345.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,45.536,241.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,6.651,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,19.198,286.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,11.353,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,38.160,288.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.626,338.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,6.053,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,458.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,5.647,243.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.757,399.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,9.984,275.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,10.740,256.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,1.098,501.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.905,577.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,936.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.538,358.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,4.917,280.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,7.491,184.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.719,404.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,8.324,330.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,12.219,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.859,296.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,5.219,263.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,460.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.905,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,9.916,277.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.448,307.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.835,187.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,1.660,207.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.782,352.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.787,247.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,2.960,232.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.346,398.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,904.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.543,506.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,904.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.450,306.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.604,214.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,2.368,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.765,359.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.421,284.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,3.608,190.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.534,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.531,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.829,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.673,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260821_055222.csv b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260821_055222.csv new file mode 100644 index 000000000..2d7e20958 --- /dev/null +++ b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260821_055222.csv @@ -0,0 +1,87 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,91.501,385.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,331.150,266.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,183.359,384.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,597.515,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,63.397,555.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,933.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,123.831,568.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,933.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,87.883,400.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,253.325,347.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,357.140,246.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,176.430,399.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,496.088,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,694.814,253.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,106.094,332.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,300.802,292.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,188.850,373.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,616.291,285.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,22.783,386.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,83.554,263.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,45.683,385.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,150.985,291.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,15.714,560.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,933.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,30.620,575.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,933.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,21.569,408.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,64.824,339.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,459.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,93.291,236.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,43.470,405.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,124.580,353.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,175.522,251.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,25.956,339.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,75.163,293.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,47.041,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,458.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,153.514,286.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,5.633,390.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,21.806,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,20.791,264.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,450.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,10.815,407.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,38.000,289.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,460.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,42.373,259.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,4.010,548.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,933.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,7.410,593.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,892.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,5.378,409.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,16.675,330.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,24.610,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,449.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,10.443,421.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,31.462,349.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,45.147,244.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,462.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,6.647,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,19.191,286.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,11.366,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,38.253,287.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,462.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.548,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,6.045,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,5.640,244.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.757,399.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,9.978,275.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,10.728,256.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,1.178,467.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,889.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.048,537.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,892.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.554,354.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,4.725,291.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,7.279,189.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.873,383.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,8.137,338.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,12.003,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.898,290.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,5.219,263.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.991,368.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,9.914,277.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.481,286.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.933,178.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,1.732,198.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.790,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.782,247.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,2.966,232.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.350,393.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,889.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.549,501.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,889.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.455,302.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.608,214.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,2.406,143.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.775,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.494,275.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,3.731,184.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.542,254.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.627,211.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.838,328.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.668,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask.png b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask.png index 7c65a16ed..7541b2216 100755 Binary files a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask.png and b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask.png differ diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask_det_overhead.png b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask_det_overhead.png index 79fc8d66b..19e90cd53 100755 Binary files a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask_det_overhead.png and b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left.png b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left.png index 26b14dfc9..aa14e146c 100755 Binary files a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left.png and b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left.png differ diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left_det_overhead.png b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left_det_overhead.png index da05f5a54..72e9cde8a 100755 Binary files a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left_det_overhead.png and b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/ltx2/gb200/ltx2_20260819_050934.csv b/benchmark/attention_training/results/ltx2/gb200/ltx2_20260819_050934.csv deleted file mode 100644 index 6e3148dca..000000000 --- a/benchmark/attention_training/results/ltx2/gb200/ltx2_20260819_050934.csv +++ /dev/null @@ -1,36 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.443,1395.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.209,1279.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.382,1119.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.472,1309.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.588,1051.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.118,1383.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.282,1206.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.878,1561.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,5.267,1391.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,6.432,1139.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.960,1496.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,2.527,1160.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.888,1499.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,5.211,1406.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,3.011,1677.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,9.298,1358.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,11.814,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,3.208,1574.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,4.195,1204.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,8.159,1547.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,8.761,1441.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,8.789,1705.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,29.537,1268.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,37.417,1001.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,10.004,1498.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,12.476,1201.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,25.162,1489.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,26.277,1425.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,15.355,1511.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,44.523,1303.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,52.690,1101.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,15.332,1513.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,18.692,1241.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,37.123,1563.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,42.311,1371.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/ltx2/gb200/ltx2_20260821_050806.csv b/benchmark/attention_training/results/ltx2/gb200/ltx2_20260821_050806.csv new file mode 100644 index 000000000..39b7b9da7 --- /dev/null +++ b/benchmark/attention_training/results/ltx2/gb200/ltx2_20260821_050806.csv @@ -0,0 +1,36 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.448,1381.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.210,1277.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.398,1106.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.475,1302.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.597,1036.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.177,1314.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.272,1216.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.838,1595.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,5.288,1386.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,6.924,1058.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,2.007,1460.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,2.515,1166.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,5.082,1442.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,5.319,1378.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,3.049,1656.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,9.417,1341.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,12.775,988.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,3.201,1577.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,4.178,1209.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,8.352,1511.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,8.915,1416.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,8.938,1676.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,29.098,1287.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,33.514,1118.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,9.409,1592.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,12.619,1187.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,25.640,1461.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,27.061,1384.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,15.940,1456.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,45.231,1282.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,53.825,1078.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,15.772,1471.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,1830.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,18.814,1233.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,38.318,1514.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,42.759,1357.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask.png b/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask.png index 170ef47c4..2c8484bae 100755 Binary files a/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask.png and b/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask.png differ diff --git a/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask_det_overhead.png b/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask_det_overhead.png index c75244aee..e08fce9f1 100755 Binary files a/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask_det_overhead.png and b/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/ltx2/gb300/ltx2_20260819_050819.csv b/benchmark/attention_training/results/ltx2/gb300/ltx2_20260821_050857.csv similarity index 50% rename from benchmark/attention_training/results/ltx2/gb300/ltx2_20260819_050819.csv rename to benchmark/attention_training/results/ltx2/gb300/ltx2_20260821_050857.csv index 2f1c23ff7..593b01248 100644 --- a/benchmark/attention_training/results/ltx2/gb300/ltx2_20260819_050819.csv +++ b/benchmark/attention_training/results/ltx2/gb300/ltx2_20260821_050857.csv @@ -1,36 +1,36 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.346,1787.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.103,1402.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.217,1270.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.374,1654.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.348,1780.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.053,1468.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.233,1254.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.493,1964.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.836,1515.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,5.821,1259.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.630,1798.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.481,1980.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.501,1628.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,4.944,1482.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.457,2055.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,8.260,1528.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,9.937,1270.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.679,1885.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.430,2078.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,7.604,1660.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,8.572,1473.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.221,2075.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,24.622,1521.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,29.838,1255.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.914,1893.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.201,2081.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,23.584,1588.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,25.098,1492.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,11.290,2055.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,38.525,1506.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,42.773,1356.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,12.277,1890.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,10.986,2112.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,35.825,1619.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,41.351,1403.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.338,1831.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.103,1402.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.217,1271.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.371,1665.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.338,1828.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.049,1473.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.241,1246.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.463,2004.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.827,1518.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,5.800,1263.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.578,1858.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.485,1974.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.498,1629.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,4.955,1479.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.420,2087.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,8.269,1527.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,9.933,1271.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.597,1944.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.422,2085.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,7.575,1667.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,8.624,1464.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.186,2085.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,25.362,1477.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,30.169,1242.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.689,1948.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.187,2085.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,23.221,1613.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,25.078,1494.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,11.197,2072.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,39.002,1487.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,42.984,1349.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,12.008,1932.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,10.999,2110.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,36.069,1608.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,41.292,1405.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask.png b/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask.png index 3e50bd353..7506e977b 100755 Binary files a/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask.png and b/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask.png differ diff --git a/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask_det_overhead.png b/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask_det_overhead.png index c38261771..da4f0b9c7 100755 Binary files a/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask_det_overhead.png and b/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/ltx2/h200/ltx2_20260820_052328.csv b/benchmark/attention_training/results/ltx2/h200/ltx2_20260821_052519.csv similarity index 62% rename from benchmark/attention_training/results/ltx2/h200/ltx2_20260820_052328.csv rename to benchmark/attention_training/results/ltx2/h200/ltx2_20260821_052519.csv index fa835b0db..6c08ca210 100644 --- a/benchmark/attention_training/results/ltx2/h200/ltx2_20260820_052328.csv +++ b/benchmark/attention_training/results/ltx2/h200/ltx2_20260821_052519.csv @@ -1,30 +1,30 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.935,662.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,2.626,589.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,3.078,502.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.875,707.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.912,678.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,2.627,589.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,3.080,502.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.846,731.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,2.322,666.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,2.510,616.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,4.771,614.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,12.673,578.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,15.097,485.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,4.069,720.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,10.664,687.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,11.197,654.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,8.207,615.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,22.482,562.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,26.198,482.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,7.001,721.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,19.879,635.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,20.448,617.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,25.591,585.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,68.034,551.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,79.961,468.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,21.431,699.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,59.000,635.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,60.185,622.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,36.679,633.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,106.628,544.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,31.893,728.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,92.903,624.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,95.284,609.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,4.603,637.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,12.348,593.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,14.753,497.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,3.883,755.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,10.581,693.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,11.277,650.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,7.946,636.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,22.248,567.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,26.271,481.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,6.704,753.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,19.163,659.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,20.275,623.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,25.619,585.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,67.182,558.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,78.634,476.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,21.511,696.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,58.294,643.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,60.389,620.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,36.836,630.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,106.326,546.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,33.489,693.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,92.014,630.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,94.412,614.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask.png b/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask.png index 5c159baa9..d1585fa21 100755 Binary files a/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask.png and b/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask.png differ diff --git a/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask_det_overhead.png b/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask_det_overhead.png index 4ef726f15..46e91d577 100755 Binary files a/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask_det_overhead.png and b/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260820_052617.csv b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260820_052617.csv deleted file mode 100644 index ab51bbc6a..000000000 --- a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260820_052617.csv +++ /dev/null @@ -1,39 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.802,343.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,5.532,279.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,6.458,239.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.629,380.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,4.432,349.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,6.725,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.902,325.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,7.233,214.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,7.189,408.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,24.554,298.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,460.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,28.757,255.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,463.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,6.909,424.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,20.273,361.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,455.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,30.332,242.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,458.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,7.741,379.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,32.806,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,12.672,399.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,49.698,254.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,49.089,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,12.060,419.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,35.462,356.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,51.553,245.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,13.514,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,56.339,224.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,38.813,386.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,146.085,256.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,36.850,407.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,105.417,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,151.303,248.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,40.308,372.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,164.884,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,60.281,385.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,195.414,297.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,57.306,405.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,162.603,357.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,230.632,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,63.450,366.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,254.345,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 diff --git a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260821_050903.csv b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260821_050903.csv new file mode 100644 index 000000000..396ecb3a3 --- /dev/null +++ b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260821_050903.csv @@ -0,0 +1,39 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.821,340.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,5.528,280.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,6.456,240.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.758,352.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,4.396,352.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,6.666,232.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.950,317.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,7.241,214.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,7.242,405.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,24.571,298.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,448.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,28.746,255.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,459.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,6.976,420.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,20.012,366.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,30.158,243.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,460.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,7.744,379.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,32.867,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,460.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,12.756,396.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,49.694,254.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,49.109,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,463.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,12.091,418.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,35.397,357.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,51.371,246.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,13.520,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,448.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,56.180,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,39.431,380.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,449.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,146.013,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,36.882,406.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,450.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,104.742,358.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,151.015,248.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,40.349,371.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,164.926,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,60.553,383.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,195.217,297.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,57.664,402.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,162.309,357.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,230.228,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,63.439,366.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,254.118,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 diff --git a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask.png b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask.png index 7e5385f94..e68d96925 100755 Binary files a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask.png and b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask.png differ diff --git a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask_det_overhead.png b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask_det_overhead.png index a4cde670a..e5ebbceee 100755 Binary files a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask_det_overhead.png and b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/qwen35/gb200/qwen35_20260819_050942.csv b/benchmark/attention_training/results/qwen35/gb200/qwen35_20260819_050942.csv deleted file mode 100644 index b8c8d8ccf..000000000 --- a/benchmark/attention_training/results/qwen35/gb200/qwen35_20260819_050942.csv +++ /dev/null @@ -1,31 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,8.980,1959.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,42.795,1028.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,43.239,1017.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,9.048,1944.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,11.829,1487.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,49.625,886.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.312,1902.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,9.792,1123.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,9.939,1106.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.383,1846.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,3.116,1412.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,11.754,936.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.622,1767.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,2.565,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,2.579,1066.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.647,1700.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.127,975.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,3.263,843.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.186,1477.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,0.776,886.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,0.776,886.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.197,1394.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.465,591.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.406,489.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.058,1182.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.264,651.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.265,648.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.069,991.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2026.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.222,310.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.663,259.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/qwen35/gb200/qwen35_20260821_050813.csv b/benchmark/attention_training/results/qwen35/gb200/qwen35_20260821_050813.csv new file mode 100644 index 000000000..50bde3c1b --- /dev/null +++ b/benchmark/attention_training/results/qwen35/gb200/qwen35_20260821_050813.csv @@ -0,0 +1,31 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,10.071,1747.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,42.095,1045.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,42.873,1026.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,9.565,1839.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,10.780,1632.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,48.381,909.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.294,1918.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,9.892,1112.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,10.315,1066.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.307,1907.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,3.122,1409.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,11.677,942.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.623,1764.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,2.581,1065.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,2.584,1064.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.650,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.127,976.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,3.179,865.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.185,1488.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,0.776,886.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,0.781,880.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.200,1371.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.465,591.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.405,489.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.057,1200.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.264,651.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.265,649.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.070,981.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.221,311.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.662,260.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left.png b/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left.png index a61bc7215..655ad97a3 100755 Binary files a/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left.png and b/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left.png differ diff --git a/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left_det_overhead.png b/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left_det_overhead.png index c9bb2ed81..851817775 100755 Binary files a/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left_det_overhead.png and b/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/qwen35/gb300/qwen35_20260819_050828.csv b/benchmark/attention_training/results/qwen35/gb300/qwen35_20260819_050828.csv deleted file mode 100644 index 8715c8c57..000000000 --- a/benchmark/attention_training/results/qwen35/gb300/qwen35_20260819_050828.csv +++ /dev/null @@ -1,31 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,8.080,2177.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,38.764,1135.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,38.254,1150.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,8.086,2176.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,10.343,1701.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,42.345,1039.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.051,2144.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,8.963,1227.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,8.980,1224.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.078,2117.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,3.087,1425.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,11.005,999.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.537,2049.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,2.405,1143.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,2.406,1143.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.559,1968.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.113,988.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,3.080,893.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.151,1823.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,0.712,966.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,0.712,966.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.164,1679.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.463,594.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.353,508.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.051,1355.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.246,700.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.245,700.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2484.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.059,1166.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.218,315.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.638,269.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/qwen35/gb300/qwen35_20260821_050907.csv b/benchmark/attention_training/results/qwen35/gb300/qwen35_20260821_050907.csv new file mode 100644 index 000000000..26d5a444a --- /dev/null +++ b/benchmark/attention_training/results/qwen35/gb300/qwen35_20260821_050907.csv @@ -0,0 +1,31 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,8.157,2157.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,37.077,1186.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,36.427,1207.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,8.203,2145.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,10.474,1680.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,42.915,1025.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.067,2128.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,9.043,1216.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,9.133,1204.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.095,2099.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,3.100,1419.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,11.054,995.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.541,2031.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,2.423,1134.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,2.421,1135.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.562,1956.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.119,983.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,3.144,874.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.153,1801.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,0.715,961.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,0.716,961.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.167,1643.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.460,598.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.357,506.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.050,1365.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.247,696.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.247,696.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.059,1156.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2316.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.217,317.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.643,267.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left.png b/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left.png index fd8a6da49..a09158852 100755 Binary files a/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left.png and b/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left.png differ diff --git a/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left_det_overhead.png b/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left_det_overhead.png index c9ce7e55c..24beb726d 100755 Binary files a/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left_det_overhead.png and b/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/qwen35/h200/qwen35_20260820_055716.csv b/benchmark/attention_training/results/qwen35/h200/qwen35_20260821_060159.csv similarity index 60% rename from benchmark/attention_training/results/qwen35/h200/qwen35_20260820_055716.csv rename to benchmark/attention_training/results/qwen35/h200/qwen35_20260821_060159.csv index 12da5ca62..0b8603bbc 100644 --- a/benchmark/attention_training/results/qwen35/h200/qwen35_20260820_055716.csv +++ b/benchmark/attention_training/results/qwen35/h200/qwen35_20260821_060159.csv @@ -1,31 +1,31 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,26.389,667.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,124.419,353.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,138.569,317.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,23.780,740.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,141.601,311.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,145.521,302.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,6.212,708.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,30.944,355.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,34.167,322.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,5.714,770.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,34.201,322.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,34.497,319.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.632,674.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,6.966,395.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,7.840,351.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.454,756.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,8.764,314.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,8.984,306.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.455,604.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.924,357.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,2.205,312.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.388,709.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,2.393,287.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,2.480,277.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.143,480.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.607,283.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.664,259.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.113,610.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.699,246.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.770,223.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,26.394,667.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,124.308,354.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,137.414,320.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,24.000,733.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,140.788,312.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,141.822,310.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,6.094,722.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,30.941,355.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,33.888,324.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,5.479,803.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,34.514,319.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,34.572,318.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.609,684.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,7.006,392.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,7.871,349.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.403,784.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,8.810,312.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,9.046,304.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.450,611.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.936,355.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,2.211,311.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.380,724.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,2.403,286.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,2.494,276.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.143,479.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.610,282.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.665,258.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.112,612.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.703,244.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.773,222.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/qwen35/h200/qwen35_top_left.png b/benchmark/attention_training/results/qwen35/h200/qwen35_top_left.png index 3aa34bb05..15ced1cdc 100755 Binary files a/benchmark/attention_training/results/qwen35/h200/qwen35_top_left.png and b/benchmark/attention_training/results/qwen35/h200/qwen35_top_left.png differ diff --git a/benchmark/attention_training/results/qwen35/h200/qwen35_top_left_det_overhead.png b/benchmark/attention_training/results/qwen35/h200/qwen35_top_left_det_overhead.png index d3acde455..b257db794 100755 Binary files a/benchmark/attention_training/results/qwen35/h200/qwen35_top_left_det_overhead.png and b/benchmark/attention_training/results/qwen35/h200/qwen35_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260820_063723.csv b/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260820_063723.csv deleted file mode 100644 index 86e41e5d0..000000000 --- a/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260820_063723.csv +++ /dev/null @@ -1,11 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,107.148,164.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,48.346,364.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,27.191,162.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,462.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,11.463,384.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,6.991,157.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,2.959,372.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,1.867,147.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.859,320.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.554,124.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.308,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 diff --git a/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260821_061615.csv b/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260821_061615.csv new file mode 100644 index 000000000..7051c9d12 --- /dev/null +++ b/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260821_061615.csv @@ -0,0 +1,11 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,107.429,164.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,48.144,365.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,27.158,162.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,458.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,11.473,383.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,6.989,157.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,3.065,359.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,1.998,138.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.868,317.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.567,121.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.312,220.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 diff --git a/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_top_left.png b/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_top_left.png index 343bcd1af..7cdc75a06 100755 Binary files a/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_top_left.png and b/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_top_left.png differ diff --git a/benchmark/attention_training/results/wan22/gb200/wan22_20260819_050958.csv b/benchmark/attention_training/results/wan22/gb200/wan22_20260819_050958.csv deleted file mode 100644 index 139777ab2..000000000 --- a/benchmark/attention_training/results/wan22/gb200/wan22_20260819_050958.csv +++ /dev/null @@ -1,36 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.825,1510.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.273,1371.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.802,1112.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.876,1423.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,1.133,1100.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.151,1448.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.345,1328.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.501,1723.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,10.697,1409.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,13.931,1082.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.794,1590.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,4.930,1223.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.554,1578.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,10.434,1445.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,13.367,1644.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,42.367,1297.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,52.658,1044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,13.950,1576.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,18.317,1200.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,33.841,1624.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,36.862,1491.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,29.530,1622.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,95.292,1257.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,116.025,1032.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,31.037,1543.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,38.993,1228.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,82.779,1447.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,92.986,1288.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,74.740,1566.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,236.015,1240.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,293.295,998.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,80.994,1445.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,101.395,1154.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,199.328,1468.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,236.688,1236.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/wan22/gb200/wan22_20260821_050830.csv b/benchmark/attention_training/results/wan22/gb200/wan22_20260821_050830.csv new file mode 100644 index 000000000..4578e102d --- /dev/null +++ b/benchmark/attention_training/results/wan22/gb200/wan22_20260821_050830.csv @@ -0,0 +1,36 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.836,1490.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.317,1345.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.848,1094.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.888,1403.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,1.137,1096.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.204,1413.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.377,1310.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.584,1683.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,10.501,1436.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,12.775,1180.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.770,1600.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,4.925,1225.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.777,1542.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,11.382,1325.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,13.729,1601.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,39.077,1406.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,52.760,1041.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,14.128,1556.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,17.539,1253.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,33.865,1623.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,36.416,1509.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,30.655,1562.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,95.424,1255.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,119.098,1005.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,30.995,1545.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,39.685,1207.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,81.219,1474.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,93.678,1278.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,79.680,1469.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,240.239,1218.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,295.895,989.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,79.455,1473.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,102.225,1145.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,202.809,1443.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,235.713,1241.000,0.000,10,,True,False,,NVIDIA GB200,1.28.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/wan22/gb200/wan22_no_mask.png b/benchmark/attention_training/results/wan22/gb200/wan22_no_mask.png index a13f9fba1..64bb7c876 100755 Binary files a/benchmark/attention_training/results/wan22/gb200/wan22_no_mask.png and b/benchmark/attention_training/results/wan22/gb200/wan22_no_mask.png differ diff --git a/benchmark/attention_training/results/wan22/gb200/wan22_no_mask_det_overhead.png b/benchmark/attention_training/results/wan22/gb200/wan22_no_mask_det_overhead.png index 3c74d86ff..9f809a9fa 100755 Binary files a/benchmark/attention_training/results/wan22/gb200/wan22_no_mask_det_overhead.png and b/benchmark/attention_training/results/wan22/gb200/wan22_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/wan22/gb300/wan22_20260819_050838.csv b/benchmark/attention_training/results/wan22/gb300/wan22_20260819_050838.csv deleted file mode 100644 index f5d3b9f14..000000000 --- a/benchmark/attention_training/results/wan22/gb300/wan22_20260819_050838.csv +++ /dev/null @@ -1,36 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.653,1907.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.153,1447.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.590,1203.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.718,1735.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.650,1917.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.026,1538.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.229,1397.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,2.883,2092.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.805,1538.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,11.838,1274.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.139,1921.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,2.829,2132.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.082,1660.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,10.425,1446.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,10.354,2123.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,37.632,1460.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,44.118,1245.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,11.519,1908.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,10.356,2122.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,34.170,1608.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,37.460,1467.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,24.395,1963.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,85.900,1394.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,101.617,1178.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,25.583,1872.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,23.522,2036.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,77.964,1536.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,89.666,1335.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,60.489,1935.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,211.692,1382.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,250.297,1169.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,63.447,1845.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,58.654,1996.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,192.854,1517.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,226.112,1294.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/wan22/gb300/wan22_20260821_050920.csv b/benchmark/attention_training/results/wan22/gb300/wan22_20260821_050920.csv new file mode 100644 index 000000000..f7b646a61 --- /dev/null +++ b/benchmark/attention_training/results/wan22/gb300/wan22_20260821_050920.csv @@ -0,0 +1,36 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.650,1917.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.164,1440.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.606,1195.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.703,1772.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.655,1903.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.037,1530.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.257,1380.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,2.900,2079.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.880,1526.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,12.335,1222.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.082,1957.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,2.870,2102.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.072,1662.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,10.298,1464.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,10.441,2105.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,37.326,1472.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,44.016,1248.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,11.108,1979.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,10.225,2150.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,34.884,1575.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,38.251,1437.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,23.954,2000.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,84.382,1419.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,99.721,1201.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,25.556,1874.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,23.865,2007.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,78.366,1528.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,91.283,1312.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,59.577,1965.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,213.240,1372.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,251.807,1162.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,63.627,1840.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,57.917,2021.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,195.708,1495.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,228.477,1281.000,0.000,10,,True,False,,NVIDIA GB300,1.28.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/wan22/gb300/wan22_no_mask.png b/benchmark/attention_training/results/wan22/gb300/wan22_no_mask.png index 81f9528b6..f0dcf7b7e 100755 Binary files a/benchmark/attention_training/results/wan22/gb300/wan22_no_mask.png and b/benchmark/attention_training/results/wan22/gb300/wan22_no_mask.png differ diff --git a/benchmark/attention_training/results/wan22/gb300/wan22_no_mask_det_overhead.png b/benchmark/attention_training/results/wan22/gb300/wan22_no_mask_det_overhead.png index 6f6cc2f9a..dbceff629 100755 Binary files a/benchmark/attention_training/results/wan22/gb300/wan22_no_mask_det_overhead.png and b/benchmark/attention_training/results/wan22/gb300/wan22_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/wan22/h200/wan22_20260820_054310.csv b/benchmark/attention_training/results/wan22/h200/wan22_20260821_054717.csv similarity index 63% rename from benchmark/attention_training/results/wan22/h200/wan22_20260820_054310.csv rename to benchmark/attention_training/results/wan22/h200/wan22_20260821_054717.csv index 76f7b4c58..05025db36 100644 --- a/benchmark/attention_training/results/wan22/h200/wan22_20260820_054310.csv +++ b/benchmark/attention_training/results/wan22/h200/wan22_20260821_054717.csv @@ -1,28 +1,28 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,2.032,613.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,5.210,598.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,6.344,491.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,1.745,714.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,4.626,673.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,1.965,634.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,5.207,598.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,6.358,490.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,1.671,746.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,4.603,677.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,4.900,636.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,9.775,617.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,26.819,562.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,31.261,482.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,8.276,729.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,23.259,648.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,24.075,626.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,38.045,578.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,98.990,555.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,32.189,683.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,87.439,628.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,91.502,601.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,84.037,570.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,218.513,548.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,71.080,674.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,193.642,618.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,198.707,603.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,207.761,563.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,541.366,541.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,176.646,663.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,474.792,616.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,484.420,604.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,9.462,637.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,26.816,562.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,31.498,479.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,7.956,758.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,22.976,656.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,23.950,630.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,37.813,581.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,98.446,558.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,32.014,687.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,87.183,630.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,90.313,608.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,83.776,572.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,218.330,548.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,70.610,678.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,191.649,625.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,197.274,607.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,206.161,568.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,537.122,545.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,176.386,664.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,471.648,620.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,480.717,609.000,0.000,10,,True,False,,NVIDIA H200,1.28.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/wan22/h200/wan22_no_mask.png b/benchmark/attention_training/results/wan22/h200/wan22_no_mask.png index 7a58ef664..37f99d98f 100755 Binary files a/benchmark/attention_training/results/wan22/h200/wan22_no_mask.png and b/benchmark/attention_training/results/wan22/h200/wan22_no_mask.png differ diff --git a/benchmark/attention_training/results/wan22/h200/wan22_no_mask_det_overhead.png b/benchmark/attention_training/results/wan22/h200/wan22_no_mask_det_overhead.png index 917d0edfb..78a91deba 100755 Binary files a/benchmark/attention_training/results/wan22/h200/wan22_no_mask_det_overhead.png and b/benchmark/attention_training/results/wan22/h200/wan22_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260820_064242.csv b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260820_064242.csv deleted file mode 100644 index 53deb97c9..000000000 --- a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260820_064242.csv +++ /dev/null @@ -1,38 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,3.099,402.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,12.620,247.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,12.382,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,453.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,2.996,416.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,8.715,357.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,13.326,234.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,456.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,3.304,377.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,14.173,220.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,463.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,15.385,392.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,452.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,59.335,254.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,59.420,254.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,14.348,420.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,42.680,353.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,61.538,245.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,15.951,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,67.105,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,57.862,380.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,213.048,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,54.550,403.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,154.781,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,223.234,246.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,59.672,368.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,240.278,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,127.074,377.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,465.028,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,119.709,400.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,339.265,353.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,484.709,247.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,131.783,363.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,522.250,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,312.593,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,1140.209,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,297.494,393.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,837.925,349.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,1186.000,247.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,325.581,360.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,1297.782,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,468.000 diff --git a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260821_062130.csv b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260821_062130.csv new file mode 100644 index 000000000..90de1d3ee --- /dev/null +++ b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260821_062130.csv @@ -0,0 +1,38 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,3.116,400.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,12.605,247.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,12.379,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,3.000,415.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,8.643,360.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,13.206,236.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,3.301,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,14.131,220.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,448.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,15.123,399.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,59.258,254.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,59.102,255.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,465.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,14.459,417.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,42.784,352.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,446.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,61.381,246.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,15.958,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,450.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,67.022,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,57.841,380.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,445.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,213.016,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,54.535,403.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,154.795,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,222.782,247.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,59.336,370.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,240.540,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,126.324,379.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,463.647,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,119.379,401.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,337.250,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,483.798,248.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,131.118,365.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,522.912,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,311.371,376.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,1134.292,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,296.086,395.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,831.828,352.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,1160.188,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,323.062,362.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,1272.638,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.28.0,92600.000,466.000 diff --git a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask.png b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask.png index 9d6d3dab1..ce027c2e1 100755 Binary files a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask.png and b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask.png differ diff --git a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask_det_overhead.png b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask_det_overhead.png index 67195f674..ef4cc5ce5 100755 Binary files a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask_det_overhead.png and b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask_det_overhead.png differ diff --git a/benchmark/cutedsl_fusion_kernels/results/GB200/output.txt b/benchmark/cutedsl_fusion_kernels/results/GB200/output.txt index af2cc7607..62d44bc97 100755 --- a/benchmark/cutedsl_fusion_kernels/results/GB200/output.txt +++ b/benchmark/cutedsl_fusion_kernels/results/GB200/output.txt @@ -8,9 +8,9 @@ nvidia-cutlass-dsl: 4.4.1 summary: activation cudnn_ms te_ms te/cudnn -swiglu 1.557 1.948 1.25x -dswiglu 0.848 1.738 2.05x -srelu 0.792 1.073 1.35x -dsrelu 0.841 1.406 1.67x -geglu 1.596 1.940 1.22x -dgeglu 0.889 1.839 2.07x +swiglu 1.585 1.925 1.21x +dswiglu 0.886 1.733 1.96x +srelu 0.787 1.066 1.36x +dsrelu 0.817 1.352 1.65x +geglu 1.579 1.921 1.22x +dgeglu 0.899 1.802 2.00x diff --git a/benchmark/cutedsl_fusion_kernels/results/GB200/output_verbose.txt b/benchmark/cutedsl_fusion_kernels/results/GB200/output_verbose.txt index ec24d3098..2bf01b057 100755 --- a/benchmark/cutedsl_fusion_kernels/results/GB200/output_verbose.txt +++ b/benchmark/cutedsl_fusion_kernels/results/GB200/output_verbose.txt @@ -10,62 +10,62 @@ timing mode: CUDA graph replay activation: swiglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=8192, fused C input N=None, activation output N=4096 -fused cuDNN avg: 1.557 ms -unfused TE avg: 1.948 ms -speedup vs TE: 1.25x -fused samples: [1.387, 1.386, 1.671, 1.675, 1.666] -TE samples: [2.123, 1.914, 1.902, 1.898, 1.902] +fused cuDNN avg: 1.585 ms +unfused TE avg: 1.925 ms +speedup vs TE: 1.21x +fused samples: [1.413, 1.476, 1.685, 1.709, 1.645] +TE samples: [2.021, 1.923, 1.901, 1.884, 1.897] activation: dswiglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=8192, activation output N=8192 -fused cuDNN avg: 0.848 ms -unfused TE avg: 1.738 ms -speedup vs TE: 2.05x -fused samples: [0.792, 0.793, 0.8, 0.868, 0.985] -TE samples: [2.305, 1.68, 1.573, 1.568, 1.566] +fused cuDNN avg: 0.886 ms +unfused TE avg: 1.733 ms +speedup vs TE: 1.96x +fused samples: [0.805, 0.791, 0.856, 0.984, 0.996] +TE samples: [2.265, 1.676, 1.612, 1.552, 1.56] activation: srelu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=None, activation output N=4096 -fused cuDNN avg: 0.792 ms -unfused TE avg: 1.073 ms -speedup vs TE: 1.35x -fused samples: [0.745, 0.726, 0.77, 0.854, 0.867] -TE samples: [1.196, 1.127, 1.037, 1.036, 0.968] +fused cuDNN avg: 0.787 ms +unfused TE avg: 1.066 ms +speedup vs TE: 1.36x +fused samples: [0.732, 0.716, 0.754, 0.855, 0.877] +TE samples: [1.179, 1.089, 1.039, 1.012, 1.011] activation: dsrelu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=4096, activation output N=4096 -fused cuDNN avg: 0.841 ms -unfused TE avg: 1.406 ms -speedup vs TE: 1.67x -fused samples: [0.74, 0.739, 0.876, 0.908, 0.942] -TE samples: [1.661, 1.532, 1.289, 1.268, 1.281] +fused cuDNN avg: 0.817 ms +unfused TE avg: 1.352 ms +speedup vs TE: 1.65x +fused samples: [0.734, 0.727, 0.833, 0.874, 0.917] +TE samples: [1.605, 1.378, 1.274, 1.256, 1.244] activation: geglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=8192, fused C input N=None, activation output N=4096 -fused cuDNN avg: 1.596 ms -unfused TE avg: 1.940 ms +fused cuDNN avg: 1.579 ms +unfused TE avg: 1.921 ms speedup vs TE: 1.22x -fused samples: [1.371, 1.542, 1.681, 1.705, 1.678] -TE samples: [2.133, 1.898, 1.867, 1.902, 1.901] +fused samples: [1.4, 1.51, 1.642, 1.685, 1.659] +TE samples: [2.065, 1.902, 1.862, 1.881, 1.896] activation: dgeglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=8192, activation output N=8192 -fused cuDNN avg: 0.889 ms -unfused TE avg: 1.839 ms -speedup vs TE: 2.07x -fused samples: [0.817, 0.805, 0.806, 0.999, 1.018] -TE samples: [2.347, 1.738, 1.666, 1.734, 1.711] +fused cuDNN avg: 0.899 ms +unfused TE avg: 1.802 ms +speedup vs TE: 2.00x +fused samples: [0.795, 0.801, 0.912, 0.984, 1.004] +TE samples: [2.279, 1.755, 1.672, 1.659, 1.647] summary: activation cudnn_ms te_ms te/cudnn -swiglu 1.557 1.948 1.25x -dswiglu 0.848 1.738 2.05x -srelu 0.792 1.073 1.35x -dsrelu 0.841 1.406 1.67x -geglu 1.596 1.940 1.22x -dgeglu 0.889 1.839 2.07x +swiglu 1.585 1.925 1.21x +dswiglu 0.886 1.733 1.96x +srelu 0.787 1.066 1.36x +dsrelu 0.817 1.352 1.65x +geglu 1.579 1.921 1.22x +dgeglu 0.899 1.802 2.00x diff --git a/benchmark/cutedsl_fusion_kernels/results/GB200/results.png b/benchmark/cutedsl_fusion_kernels/results/GB200/results.png index bc96a410e..8e55fa592 100755 Binary files a/benchmark/cutedsl_fusion_kernels/results/GB200/results.png and b/benchmark/cutedsl_fusion_kernels/results/GB200/results.png differ diff --git a/benchmark/cutedsl_fusion_kernels/results/GB300/output.txt b/benchmark/cutedsl_fusion_kernels/results/GB300/output.txt index 6b4adbbc9..e0bd6b531 100755 --- a/benchmark/cutedsl_fusion_kernels/results/GB300/output.txt +++ b/benchmark/cutedsl_fusion_kernels/results/GB300/output.txt @@ -8,9 +8,9 @@ nvidia-cutlass-dsl: 4.4.1 summary: activation cudnn_ms te_ms te/cudnn -swiglu 1.323 1.627 1.23x -dswiglu 0.745 1.542 2.07x -srelu 0.674 0.881 1.31x -dsrelu 0.691 1.196 1.73x -geglu 1.324 1.641 1.24x -dgeglu 0.743 1.647 2.22x +swiglu 1.329 1.742 1.31x +dswiglu 0.738 1.535 2.08x +srelu 0.669 0.894 1.34x +dsrelu 0.681 1.159 1.70x +geglu 1.340 1.649 1.23x +dgeglu 0.738 1.636 2.22x diff --git a/benchmark/cutedsl_fusion_kernels/results/GB300/output_verbose.txt b/benchmark/cutedsl_fusion_kernels/results/GB300/output_verbose.txt index 6a5cfa20f..4b245e353 100755 --- a/benchmark/cutedsl_fusion_kernels/results/GB300/output_verbose.txt +++ b/benchmark/cutedsl_fusion_kernels/results/GB300/output_verbose.txt @@ -10,62 +10,62 @@ timing mode: CUDA graph replay activation: swiglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=8192, fused C input N=None, activation output N=4096 -fused cuDNN avg: 1.323 ms -unfused TE avg: 1.627 ms -speedup vs TE: 1.23x -fused samples: [1.198, 1.278, 1.321, 1.41, 1.406] -TE samples: [1.768, 1.644, 1.594, 1.53, 1.599] +fused cuDNN avg: 1.329 ms +unfused TE avg: 1.742 ms +speedup vs TE: 1.31x +fused samples: [1.208, 1.276, 1.365, 1.4, 1.397] +TE samples: [1.769, 1.734, 1.78, 1.677, 1.75] activation: dswiglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=8192, activation output N=8192 -fused cuDNN avg: 0.745 ms -unfused TE avg: 1.542 ms -speedup vs TE: 2.07x -fused samples: [0.704, 0.708, 0.749, 0.752, 0.812] -TE samples: [1.908, 1.479, 1.448, 1.441, 1.435] +fused cuDNN avg: 0.738 ms +unfused TE avg: 1.535 ms +speedup vs TE: 2.08x +fused samples: [0.705, 0.712, 0.713, 0.762, 0.8] +TE samples: [1.877, 1.49, 1.446, 1.435, 1.429] activation: srelu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=None, activation output N=4096 -fused cuDNN avg: 0.674 ms -unfused TE avg: 0.881 ms -speedup vs TE: 1.31x -fused samples: [0.641, 0.642, 0.688, 0.682, 0.718] -TE samples: [0.955, 0.956, 0.853, 0.818, 0.822] +fused cuDNN avg: 0.669 ms +unfused TE avg: 0.894 ms +speedup vs TE: 1.34x +fused samples: [0.644, 0.642, 0.662, 0.691, 0.705] +TE samples: [0.964, 0.955, 0.854, 0.849, 0.849] activation: dsrelu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=4096, activation output N=4096 -fused cuDNN avg: 0.691 ms -unfused TE avg: 1.196 ms -speedup vs TE: 1.73x -fused samples: [0.653, 0.65, 0.662, 0.738, 0.754] -TE samples: [1.379, 1.332, 1.097, 1.088, 1.081] +fused cuDNN avg: 0.681 ms +unfused TE avg: 1.159 ms +speedup vs TE: 1.70x +fused samples: [0.649, 0.648, 0.691, 0.696, 0.723] +TE samples: [1.386, 1.159, 1.097, 1.083, 1.071] activation: geglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=8192, fused C input N=None, activation output N=4096 -fused cuDNN avg: 1.324 ms -unfused TE avg: 1.641 ms -speedup vs TE: 1.24x -fused samples: [1.2, 1.288, 1.322, 1.402, 1.409] -TE samples: [1.771, 1.623, 1.568, 1.633, 1.612] +fused cuDNN avg: 1.340 ms +unfused TE avg: 1.649 ms +speedup vs TE: 1.23x +fused samples: [1.21, 1.302, 1.373, 1.412, 1.404] +TE samples: [1.779, 1.679, 1.607, 1.578, 1.603] activation: dgeglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=8192, activation output N=8192 -fused cuDNN avg: 0.743 ms -unfused TE avg: 1.647 ms +fused cuDNN avg: 0.738 ms +unfused TE avg: 1.636 ms speedup vs TE: 2.22x -fused samples: [0.71, 0.711, 0.767, 0.76, 0.765] -TE samples: [2.033, 1.567, 1.553, 1.543, 1.541] +fused samples: [0.709, 0.715, 0.716, 0.753, 0.799] +TE samples: [1.955, 1.605, 1.549, 1.54, 1.531] summary: activation cudnn_ms te_ms te/cudnn -swiglu 1.323 1.627 1.23x -dswiglu 0.745 1.542 2.07x -srelu 0.674 0.881 1.31x -dsrelu 0.691 1.196 1.73x -geglu 1.324 1.641 1.24x -dgeglu 0.743 1.647 2.22x +swiglu 1.329 1.742 1.31x +dswiglu 0.738 1.535 2.08x +srelu 0.669 0.894 1.34x +dsrelu 0.681 1.159 1.70x +geglu 1.340 1.649 1.23x +dgeglu 0.738 1.636 2.22x diff --git a/benchmark/cutedsl_fusion_kernels/results/GB300/results.png b/benchmark/cutedsl_fusion_kernels/results/GB300/results.png index 379029eb9..ffe2389e4 100755 Binary files a/benchmark/cutedsl_fusion_kernels/results/GB300/results.png and b/benchmark/cutedsl_fusion_kernels/results/GB300/results.png differ